diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ad3e5a2df..cd2ae98823 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,401 +1,16 @@ -stages: - - info - - lint - - test - - build - - e2e - - deploy_dev_tags - - deploy_prod_alpha - - deploy_prod_beta - - deploy_prod_ea - - deploy_prod_stable - - deploy_prod_rock_solid - - cleanup - -default: - tags: - - deckhouse - -.build: - stage: build - script: - # Build images - - | - werf build \ - --repo=${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} \ - --save-build-report --build-report-path images_tags_werf.json - # Bundle image - - | - IMAGE_SRC="$(jq -r '.Images."bundle".DockerImageName' images_tags_werf.json)" - IMAGE_DST="$(jq -r '.Images.bundle.DockerRepo' images_tags_werf.json):${MODULES_MODULE_TAG}" - - echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" - crane copy ${IMAGE_SRC} ${IMAGE_DST} - # Release-channel image - - | - IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)" - IMAGE_DST="$(jq -r '.Images."release-channel-version".DockerRepo' images_tags_werf.json)/release:${MODULES_MODULE_TAG}" - - echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" - crane copy ${IMAGE_SRC} ${IMAGE_DST} - # Register module - - | - echo "✨ Register the module ${MODULES_MODULE_NAME}" - crane append \ - --oci-empty-base \ - --new_layer "" \ - --new_tag "${MODULES_MODULE_SOURCE}:${MODULES_MODULE_NAME}" - -.deploy: - stage: deploy - script: - - | - REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" - - IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" - IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" - - echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" - crane copy "${IMAGE_SRC}" "${IMAGE_DST}" - - -.info: - script: - - | - cat << OUTER - Create ModuleConfig and ModulePullOverride resources to test this MR: - cat < EN MR) +# - Merge_Release.gitlab-ci.yml (release-label driven merge+tag+release) +# - Svace_Analayze.gitlab-ci.yml (note: typo in upstream file name) +# - Antivirus_Scan.gitlab-ci.yml (antivirus scan over recent releases) +# - Semgrep.gitlab-ci.yml (semgrep SAST) +# +# We only include the templates whose behavior is consumed by jobs owned by +# this issue (Setup, Build, Deploy). The remaining upstream templates are +# surfaced here for visibility — child issues that own cve-scan.yml, +# gitleaks.yml, etc. add the matching `include: local:` entries for the +# corresponding job files. + +include: + # --- Upstream modules-gitlab-ci (deckhouse/3p, ref v13.0) --- + # Tracks the v13.0 branch so upstream fixes flow in automatically. To pin for + # reproducibility, swap the ref for the branch HEAD SHA + # (006d51c35904b434eca2045a449aafb5e37a8827 at the time of writing). + - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" + ref: "v13.0" + file: + # Setup.gitlab-ci.yml provides trdl + werf setup + dual-registry + # `werf cr login` in before_script. See templates/Setup.gitlab-ci.yml. + - "/templates/Setup.gitlab-ci.yml" + # Validation/scanning jobs extend these upstream templates. + - "/templates/CVE_Scan.gitlab-ci.yml" + - "/templates/gitleaks.gitlab-ci.yml" + - "/templates/Svace_Analayze.gitlab-ci.yml" + # Build.gitlab-ci.yml and Deploy.gitlab-ci.yml are intentionally NOT + # included directly. Their hidden jobs (.build, .deploy) ship with + # `rules:` baked in (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`, + # `when: manual`) that override our strict gating via + # .dev / .dev_tags / .main / .prod_manual. We mirror the upstream + # script bodies in .gitlab/ci/templates/base/{build,deploy}.yml as + # `.base_build` and `.base_deploy` and extend those instead. + + # --- Local structural fragments (order: stages, vars, defaults, then jobs) --- + - local: ".gitlab/ci/stages.yml" + - local: ".gitlab/ci/variables.yml" + - local: ".gitlab/ci/defaults.yml" + - local: ".gitlab/ci/workflow.yml" + + # --- Local shared templates (extends: ...) --- + # templates/ is grouped by the two global processes plus shared base: + # base/ - process-agnostic base job bodies + helpers + # dev/ - development process (DEV registry): MR / main / release / dev-tag + # release/ - release process (PROD registry): tag-push + manual dispatch + # Anchor names are unchanged (.base_build, .dev, .prod_vars, ...), so no job + # `extends:` edits are needed. + # Registry-login helpers must load before dev_vars/prod_vars, which extend + # them (.login_dev_registry / .login_prod_registry). + - local: ".gitlab/ci/templates/base/registry-login.yml" + - local: ".gitlab/ci/templates/dev/dev_vars.yml" + - local: ".gitlab/ci/templates/release/prod_vars.yml" + - local: ".gitlab/ci/templates/dev/dev.yml" + - local: ".gitlab/ci/templates/dev/dev_tags.yml" + - local: ".gitlab/ci/templates/dev/main.yml" + - local: ".gitlab/ci/templates/dev/release.yml" + - local: ".gitlab/ci/templates/release/prod_tag.yml" + - local: ".gitlab/ci/templates/base/info.yml" + - local: ".gitlab/ci/templates/base/build.yml" + - local: ".gitlab/ci/templates/base/deploy.yml" + + # --- Local job files owned by this issue --- + - local: ".gitlab/ci/jobs/info.yml" + - local: ".gitlab/ci/jobs/test.yml" + - local: ".gitlab/ci/jobs/test-scripts-js.yml" + - local: ".gitlab/ci/jobs/test-scripts-python.yml" + - local: ".gitlab/ci/jobs/test-d8v-cli.yml" + - local: ".gitlab/ci/jobs/build-dev.yml" + - local: ".gitlab/ci/jobs/build-prod.yml" + # No deploy-dev.yml: the DEV registry has no release channels (only tags), + # so a dev tag is just built (build_dev_tags) and never crane-copied into + # alpha/beta/... The GitHub "Deploy Dev" workflow + # (dev_module_build-and-registration.yml) is build-only as well. + - local: ".gitlab/ci/jobs/deploy-prod.yml" + # Prod release-channels parity flow (manual single-channel dispatch): + # requirements check, build/deploy per edition, version verification, + # GitLab release creation, Loop notification. + - local: ".gitlab/ci/jobs/release-channels.yml" + - local: ".gitlab/ci/jobs/cleanup.yml" + + # --- Local validation and scanning jobs --- + - local: ".gitlab/ci/jobs/lint-dmt.yml" + - local: ".gitlab/ci/jobs/lint-validate.yml" + - local: ".gitlab/ci/jobs/precache.yml" + - local: ".gitlab/ci/jobs/svace.yml" + - local: ".gitlab/ci/jobs/cve-scan.yml" + - local: ".gitlab/ci/jobs/gitleaks.yml" + + # --- Local GitLab API automation, changelog, backport, and manual tools --- + - local: ".gitlab/ci/jobs/auto-assign-author.yml" + - local: ".gitlab/ci/jobs/backport.yml" + - local: ".gitlab/ci/jobs/changelog.yml" + - local: ".gitlab/ci/jobs/check-changelog.yml" + - local: ".gitlab/ci/jobs/check-milestone.yml" + - local: ".gitlab/ci/jobs/manual-tools.yml" + - local: ".gitlab/ci/jobs/translate-changelog.yml" diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml new file mode 100644 index 0000000000..177d968d35 --- /dev/null +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -0,0 +1,35 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Auto-assign MR author as MR assignee (GitLab API). +# +# Migration of .github/workflows/dev_auto-pr-author-assign.yml. +# Behaviour: if MR already has an assignee, skip. +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +auto-assign-author: + stage: info + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - bash .gitlab/ci/scripts/bash/auto-assign-author.sh + rules: + # Run on every MR pipeline. The script is idempotent: it skips when the MR + # already has an assignee, so re-running on each push is harmless. Gating on + # changes: to this job's own files (the previous behaviour) was wrong — it + # meant the author was only auto-assigned when the auto-assign files changed, + # which never happens on normal MRs. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml new file mode 100644 index 0000000000..b222177c0d --- /dev/null +++ b/.gitlab/ci/jobs/backport.yml @@ -0,0 +1,65 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Backport a merged MR to a release branch (clone + cherry-pick + push + MR). +# +# Migration of .github/workflows/on-pull-request-backport.yml which used +# deckhouse/backport-action@v1.0.0 with automerge=true and direct push to +# the release branch. We open a reviewable backport MR instead of pushing +# directly. +# +# Triggers: +# 1. Manual pipeline (Run pipeline UI) with variable TARGET_BRANCH=release-1.21. +# 2. Manual pipeline on an MR carrying the `status/backport` label. +# GitLab does NOT auto-trigger pipelines on label change; the user must +# press "Run pipeline" on the MR. Webhook-driven auto-trigger is an +# accepted permanent gap (no webhook-listener will be built). +# The target branch is derived from the source MR milestone title +# (vX.Y.Z or X.Y.Z -> release-X.Y); TARGET_BRANCH overrides this. +# +# After the backport job finishes, backport.sh updates the source MR: +# - always removes `status/backport`; +# - on success adds `status/backport/success` and comments with the +# backport MR link; +# - on failure adds `status/backport/failed` and comments with the job link. +# +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +backport: + stage: lint + # Mutates state (cherry-pick + opens an MR); never auto-cancel mid-run. + interruptible: false + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add + script: + - bash .gitlab/ci/scripts/bash/backport.sh + # allow_failure: true on every rule: `when: manual` inside `rules:` defaults + # to allow_failure: false, so on an MR carrying the `status/backport` label + # this unplayed manual job would block the whole MR pipeline (test/build/...). + # Backport is an opt-in side action, so it must never gate the pipeline. + rules: + # Mode 1: explicit manual run with TARGET_BRANCH provided via UI (overrides + # milestone-based inference). + - if: $TARGET_BRANCH + when: manual + allow_failure: true + # Mode 2: MR with the `status/backport` label. GitLab does NOT auto-run + # pipelines on label change; user has to press "Run pipeline" on the MR + # (webhook auto-trigger is an accepted permanent gap, not planned). The + # target branch is derived from the source MR milestone by backport.sh. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,\s*)status\/backport(,|$)/ + when: manual + allow_failure: true diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml new file mode 100644 index 0000000000..ed13eed338 --- /dev/null +++ b/.gitlab/ci/jobs/build-dev.yml @@ -0,0 +1,86 @@ +# DEV build jobs. +# +# Carries forward build_dev, build_dev_tags and build_main from the +# previous root .gitlab-ci.yml. All three extend: +# - .base_build (this repo) — same body as upstream modules-gitlab-ci +# Build.gitlab-ci.yml `.build` (werf build + bundle crane +# copy + release-channel crane copy + crane append to +# register module), but without the upstream `rules:` +# that would override our strict gating via .dev / +# .dev_tags / .main. +# - .dev / .dev_tags / .main (this repo) for registry + tag context. +# +# The previous root .gitlab-ci.yml had its own `.build` template with the +# same body; it is replaced here by `.base_build` (verified against +# deckhouse/3p/deckhouse/modules-gitlab-ci +# templates/Build.gitlab-ci.yml — werf build with --save-build-report, +# bundle / release-channel-version crane copy, and crane append to +# register the module). The upstream version uses WERF_REPO env instead of +# an explicit --repo flag, which is equivalent because MODULES_MODULE_SOURCE +# is the same in both cases. +# +# All dev build jobs inherit the global `interruptible: true` default +# (.gitlab/ci/defaults.yml), so a new push to the same MR/branch cancels the +# older in-flight build. + +# `needs: [set_vars]` (with artifacts) is intentional: it opts these jobs +# into DAG mode and pulls ONLY the set_vars dotenv (MODULE_EDITION, +# DEBUG_COMPONENT, RELEASE_IN_DEV). It replaces the previous bare `needs: []`, +# which existed to stop GitLab's legacy behavior of downloading ALL prior-stage +# artifacts and leaking unrelated dotenv artifacts (e.g. svace:set-vars writes +# MODULES_MODULE_TAG=-svace) that override the per-template +# MODULES_MODULE_TAG from .dev / .dev_tags / .main. Listing set_vars explicitly +# keeps that protection: svace.env is still not downloaded. set_vars does NOT +# emit MODULES_MODULE_TAG, so the per-template tag is preserved. +# +# WERF_VIRTUAL_MERGE=0 mirrors the GitHub dev_setup_build job env. + +build_dev: + stage: build + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" + extends: + - .base_build + - .dev + +build_dev_tags: + stage: build + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" + extends: + - .base_build + - .dev_tags + +build_main: + stage: build + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" + extends: + - .base_build + - .main + +# build_release mirrors GitHub dev_module_build.yml `push: [release-*]`: a +# squash-merge into a release-X.Y branch rebuilds the dev image tagged with the +# branch name. Without this job GitLab created a pipeline on the merge push but +# built nothing for release branches (only .dev/.main/.dev_tags had build jobs). +# Inherits the default interruptible: true, so a newer release-branch push +# cancels an older build. +build_release: + stage: build + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" + extends: + - .base_build + - .release diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml new file mode 100644 index 0000000000..8292fc1aaf --- /dev/null +++ b/.gitlab/ci/jobs/build-prod.yml @@ -0,0 +1,49 @@ +# PROD build job. +# +# Builds the module images for every edition on tag push (vX.Y.Z), then +# leaves the deploy step to .gitlab/ci/jobs/deploy-prod.yml. +# +# Parity with .github/workflows/release_module_build-and-registration.yml: +# four editions ce / ee / se-plus / fe, each with its own +# MODULES_MODULE_SOURCE subpath and the correct MODULE_EDITION (CE only for +# ce, EE for ee/se-plus/fe). MODULE_EDITION is consumed by +# .werf/consts.yaml (defaults to EE) and used as a Go build tag plus an +# EE-only image gate, so it MUST be set per edition — otherwise the ce prod +# image is built as EE. +# +# Note vs GitHub: GH runs ce and ee in parallel, then se-plus and fe after +# ee (needs: prod_ee_setup_build). The GitLab matrix runs all four in +# parallel. Each edition builds into its own registry subpath with no shared +# artifact, so the GH `needs: ee` ordering is a sequencing/resource guard, +# not a data dependency; the parallel matrix is acceptable. +# +# Resource group: prod — prevents two prod builds from racing. +# +# Upstream .build expects MODULES_MODULE_SOURCE and WERF_REPO; .prod_vars +# sets MODULES_MODULE_SOURCE per-edition (PROD_REGISTRY/PROD_MODULE_SOURCE_NAME/ +# EDITION/modules), and WERF_REPO defaults to MODULES_MODULE_SOURCE/ +# MODULES_MODULE_NAME in upstream Setup — correct for prod. +# +# `.also_login_dev_registry` is included so the second `werf cr login` from +# upstream Setup.gitlab-ci.yml also fires against DEV_REGISTRY; the +# previous GH workflow did both logins via two consecutive +# modules-actions/setup steps; we keep that behavior. + +build_prod: + stage: build + resource_group: prod + # interruptible: false is inherited from .prod_vars (via .prod_always). + extends: + - .base_build + - .prod_always + - .also_login_dev_registry + parallel: + matrix: + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml new file mode 100644 index 0000000000..6b2e9e54c6 --- /dev/null +++ b/.gitlab/ci/jobs/changelog.yml @@ -0,0 +1,89 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Re-generate CHANGELOG files for a milestone and (optionally) open a MR. +# +# Migration of: +# - .github/workflows/changelog-by-milestone.yml (issues.milestoned) +# - .github/workflows/changelog-by-pull.yml (pull_request_target) +# - .github/workflows/changelog-command.yml (repository_dispatch /changelog) +# All three used ./.github/actions/milestone-changelog (composite action). +# +# The GitLab jobs are manual + scheduled, NOT reactive. Reactive webhook automation (slash-command-equivalents, +# label/milestone events) is an accepted permanent gap — no webhook-listener +# will be built. +# +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +# Variables exposed at "Run pipeline" UI: +# MILESTONE_TITLE - if set, generate only this milestone (e.g. "v1.21.3"). +# Leave empty to iterate over all active milestones. +# OPEN_CHANGELOG_MR - "true" to push a branch and open a changelog MR. +# Defaults to "false" (files only, no MR). +# CHANGELOG_BASE_BRANCH - target branch (default "main"). + +changelog:milestone: + stage: lint + # Mutates state (commits + opens an MR); never auto-cancel mid-run. + interruptible: false + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 + script: + - bash .gitlab/ci/scripts/bash/changelog-milestone.sh + variables: + MILESTONE_TITLE: "" + OPEN_CHANGELOG_MR: "false" + CHANGELOG_BASE_BRANCH: "main" + rules: + # Mode 1: manual with optional MILESTONE_TITLE / OPEN_CHANGELOG_MR vars. + # allow_failure: true is REQUIRED here: `when: manual` inside `rules:` + # defaults to allow_failure: false, which makes the (unplayed) manual job + # block every later stage of the MR pipeline (test/build/...). GitHub ran + # changelog generation in a separate workflow, fully decoupled from the + # build/test pipeline, so this manual job must be non-blocking/optional. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual + allow_failure: true + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + allow_failure: true + # Mode 2: scheduled daily run. We intentionally do NOT auto-open the MR on + # schedule; set OPEN_CHANGELOG_MR=true in the schedule definition if you + # want MRs created. Runs automatically under the dedicated + # changelog-milestone pipeline schedule + # ($SCHEDULE_TYPE == "changelog-milestone"). + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "changelog-milestone" + +# Optional: process all active milestones (one MR per milestone). +# Run on a schedule (e.g. nightly). Iterates over all open milestones and +# generates CHANGELOG files locally; OPEN_CHANGELOG_MR controls whether +# MRs are opened. +changelog:all-active-milestones: + stage: lint + # Mutates state (commits + opens an MR); never auto-cancel mid-run. + interruptible: false + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 + script: + - bash .gitlab/ci/scripts/bash/changelog-milestone.sh + variables: + MILESTONE_TITLE: "" + OPEN_CHANGELOG_MR: "false" + CHANGELOG_BASE_BRANCH: "main" + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "changelog-all-active-milestones" diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml new file mode 100644 index 0000000000..45886ff91c --- /dev/null +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Validate ```changes fenced blocks in MR description. +# +# Migration of .github/workflows/check-changelog-entry.yml which used +# deckhouse/changelog-action@v2.6.0 with validate_only=true. +# The validation is implemented in Python and uses +# .gitlab/ci/changelog-sections.txt as the source of truth. +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +check:changelog: + stage: lint + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq python3 + script: + - bash .gitlab/ci/scripts/bash/check-changelog-entry.sh + rules: + # Skip-label must be first because GitLab rules are evaluated first-match. + - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_changelog/ + when: never + # Run on MR pipelines only. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/jobs/check-milestone.yml b/.gitlab/ci/jobs/check-milestone.yml new file mode 100644 index 0000000000..a20fa97f71 --- /dev/null +++ b/.gitlab/ci/jobs/check-milestone.yml @@ -0,0 +1,33 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verify MR has a milestone assigned (GitLab API check). +# +# Migration of .github/workflows/check-pr-milestone.yml. +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +check:milestone: + stage: lint + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - bash .gitlab/ci/scripts/bash/check-milestone.sh + rules: + # Skip-label must be first because GitLab rules are evaluated first-match. + - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_milestone/ + when: never + # MR open/synchronize/reopen/milestone changes. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml new file mode 100644 index 0000000000..707465e414 --- /dev/null +++ b/.gitlab/ci/jobs/cleanup.yml @@ -0,0 +1,47 @@ +# Cleanup job. +# +# Carries forward the cleanup job from the previous root .gitlab-ci.yml and +# the GitHub .github/workflows/dev_registry-cleanup.yml workflow. Runs only +# on scheduled pipelines and prunes old module images from the DEV registry +# using werf cleanup --without-kube=true --config werf_cleanup.yaml. +# +# The weekly cadence (GitHub cron "12 0 * * 6") is configured in the +# GitLab UI under CI/CD -> Schedules, not here. This job only gates on +# $CI_PIPELINE_SOURCE == "schedule" AND $SCHEDULE_TYPE == "cleanup" so it +# runs only under the dedicated cleanup schedule and not under any other +# pipeline schedule defined in the project. +# +# Registry path is composed from the DEV Project Variables the rest of the +# pipeline uses: +# MODULES_MODULE_SOURCE <- DEV_MODULE_SOURCE (e.g. dev-registry.deckhouse.io/sys/deckhouse-oss/modules), via .dev_vars +# MODULES_MODULE_NAME <- MODULE_NAME (virtualization), set in .gitlab/ci/variables.yml +# This matches the GitHub workflow's +# --repo ${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} +# exactly. Extending .dev_vars also provides MODULES_REGISTRY + +# MODULES_REGISTRY_LOGIN/_PASSWORD so the upstream Setup.gitlab-ci.yml +# before_script `werf cr login` authenticates against the DEV registry +# before cleanup runs. +# +# Retention policies live in werf_cleanup.yaml at the repo root +# (tag /.*/ -> 72h, branch /.*/ -> 168h, branch /main|release-[0-9]+.*/ -> +# last 5 with imagesPerReference last 1). --config is mandatory: without +# it werf cleanup falls back to default git-history policies and +# over/under-prunes dev images. + +cleanup: + stage: cleanup + # Mutates the registry (werf cleanup); never auto-cancel mid-run. + interruptible: false + extends: + - .dev_vars + variables: + MODULES_MODULE_TAG: v0.0.0-main + # Explicitly disable werf dry-run, mirroring GitHub dev_registry-cleanup.yml + # (env WERF_DRY_RUN: "false"). Without this an inherited/global + # WERF_DRY_RUN=true would make cleanup silently prune nothing. + WERF_DRY_RUN: "false" + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cleanup" + script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf + - werf cleanup --repo ${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} --without-kube=true --config werf_cleanup.yaml diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml new file mode 100644 index 0000000000..83fa849035 --- /dev/null +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -0,0 +1,117 @@ +# CVE (Trivy) scan via the upstream `.cve_scan` template. +# +# Migration of .github/workflows/cve_scan_daily.yml and the +# `cve_scan_on_pr` job from .github/workflows/dev_module_build.yml. +# +# The GH workflows pulled CVE_TEST_REPO_GIT, CODEOWNERS_REPO_TOKEN, +# DD_URL, DD_TOKEN, DECKHOUSE_PRIVATE_REPO, etc. from HashiCorp Vault via +# `hashicorp/vault-action@v2`. The upstream +# deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 `.cve_scan` template +# does the same thing but via GitLab `id_tokens` + d8-cli. Both paths +# resolve to the same set of variables, so extending the upstream +# template is the right migration. +# +# The Vault integration is unchanged (still uses seguro.flant.com); only +# the auth mechanism moves from GH vault-action to GitLab id_tokens. +# +# Required CI/CD variables (declared in the project). The upstream +# template provides defaults via `vault:` references, but the +# operator must set: +# VAULT_ROLE - role to authenticate against in seguro.flant.com. +# Set as a Project CI/CD variable (see +# .gitlab/ci/variables.yml, "CVE scan" section). +# In the GH workflow this was the repo name +# (virtualization); reuse the same Vault role. +# SOURCE_TAG - tag/branch to scan (e.g. "main" or a release tag) +# CASE - case label forwarded to cve_scan.sh +# EXTERNAL_MODULE_NAME - the module name (default: "virtualization") +# +# The upstream `.cve_scan` template sets `allow_failure: true`, so a scan +# failure never blocks the pipeline (mirrors the GH behavior where the +# action result did not fail the workflow job). +# +# MODULE_PROD_REGISTRY_CUSTOM_PATH and MODULE_DEV_REGISTRY_CUSTOM_PATH +# are intentionally NOT set here: the upstream `.cve_scan` template +# already defaults them to "deckhouse/fe/modules" and +# "sys/deckhouse-oss/modules" respectively. Restating them verbatim +# would be redundant. + +# --------------------------------------------------------------------------- +# Scheduled daily scan against main. +# Mirrors `cve_scan_daily.yml` cron: "0 02 * * *". +# Runs only under the dedicated cve-scan-daily pipeline schedule +# ($SCHEDULE_TYPE == "cve-scan-daily"). +# --------------------------------------------------------------------------- + +cve:scan:daily: + extends: + - .cve_scan + stage: scan + interruptible: false + variables: + SOURCE_TAG: "main" + CASE: "External Modules" + EXTERNAL_MODULE_NAME: "virtualization" + SCAN_SEVERAL_LATEST_RELEASES: "True" + LATEST_RELEASES_AMOUNT: "5" + RELEASE_IN_DEV: "false" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cve-scan-daily"' + +# --------------------------------------------------------------------------- +# Manual scan against a chosen tag or branch. +# Mirrors `cve_scan_daily.yml` `workflow_dispatch.inputs.tag_name`. +# --------------------------------------------------------------------------- + +cve:scan:manual: + extends: + - .cve_scan + stage: scan + interruptible: false + variables: + SOURCE_TAG: "${SCAN_TAG:-main}" + CASE: "External Modules" + EXTERNAL_MODULE_NAME: "virtualization" + SCAN_SEVERAL_LATEST_RELEASES: "${SCAN_SEVERAL:-False}" + LATEST_RELEASES_AMOUNT: "5" + RELEASE_IN_DEV: "false" + rules: + # RELEASE_IN_DEV is a boolean: release-* tags are built into the DEV + # registry, everything else into PROD. Mirror the GitHub expression + # `startsWith(tag || 'main', 'release-')` — a release-* SCAN_TAG flips it + # to "true"; an empty/other SCAN_TAG keeps the "false" default above. + - if: '$CI_PIPELINE_SOURCE == "web" && $SCAN_TAG =~ /^release-/' + when: manual + variables: + RELEASE_IN_DEV: "true" + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - when: never + +# --------------------------------------------------------------------------- +# Per-MR scan against the MR dev build. +# Mirrors the `cve_scan_on_pr` job in dev_module_build.yml, which ran +# `needs: [set_vars, dev_setup_build]` and scanned the dev build tag for +# the PR. Here it runs after build_dev (which tags images +# `mr${CI_MERGE_REQUEST_IID}` via the .dev template) and scans that tag. +# `allow_failure: true` is inherited from `.cve_scan`, matching the GH +# job (which had no continue-on-error but whose action result did not +# block the PR). +# --------------------------------------------------------------------------- + +cve:scan:mr: + extends: + - .cve_scan + stage: scan + needs: + - build_dev + variables: + SOURCE_TAG: "mr${CI_MERGE_REQUEST_IID}" + CASE: "External Modules" + EXTERNAL_MODULE_NAME: "virtualization" + SCAN_SEVERAL_LATEST_RELEASES: "False" + LATEST_RELEASES_AMOUNT: "3" + RELEASE_IN_DEV: "false" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - when: never diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml new file mode 100644 index 0000000000..f77d36880d --- /dev/null +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -0,0 +1,135 @@ +# PROD deploy jobs. +# +# Parity with .github/workflows/release_module_release-channels.yml: the +# channel is selected by the operator, not promoted through a fixed chain. +# All five channel deploys (alpha / beta / early-access / stable / +# rock-solid) are INDEPENDENT manual jobs (when: manual from .prod_manual) +# in a single `deploy_prod` stage. Each only `needs: [build_prod]`, so any +# channel can be deployed directly after the build without first clicking +# the earlier channels. This replaces the previous forced promotion chain +# (alpha -> beta -> ea -> stable -> rock-solid across five separate stages), +# which was a carried-over artifact of the old root .gitlab-ci.yml and +# diverged from GitHub. Each job copies the +# built release image to its named release channel on a vX.Y.Z tag. +# +# The EDITION matrix mirrors build_prod (ce/ee/se-plus/fe) with the correct +# MODULE_EDITION per edition, so deploy runs against the same per-edition +# source path the build pushed to. MODULE_EDITION is forwarded to the +# deploy step for consistency with the build (werf deploy reads it from the +# environment). +# +# Extends: +# - .base_deploy (this repo — mirrors upstream modules-gitlab-ci +# Deploy.gitlab-ci.yml `.deploy` script body without +# the upstream `rules:` that would override our +# gating). +# - .prod_manual (this repo — PROD registry vars + tag-match rule +# with `when: manual`). +# - .also_login_dev_registry (this repo — also login against DEV_REGISTRY). +# +# This file is the tag-push chained-deploy flow. The GitHub +# release_module_release-channels parity flow (single-channel dispatch with +# requirements/version checks, release creation, Loop notify) now lives in +# .gitlab/ci/jobs/release-channels.yml (prod:* jobs, manual Run pipeline). +# This tag-push chain and that flow coexist because the parity flow is gated +# to CI_PIPELINE_SOURCE == "web". + +deploy_to_prod_alpha: + stage: deploy_prod + variables: + RELEASE_CHANNEL: alpha + needs: ["build_prod"] + extends: + - .base_deploy + - .prod_manual + - .also_login_dev_registry + parallel: + matrix: + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE + +deploy_to_prod_beta: + stage: deploy_prod + variables: + RELEASE_CHANNEL: beta + needs: ["build_prod"] + extends: + - .base_deploy + - .prod_manual + - .also_login_dev_registry + parallel: + matrix: + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE + +deploy_to_prod_ea: + stage: deploy_prod + variables: + RELEASE_CHANNEL: early-access + needs: ["build_prod"] + extends: + - .base_deploy + - .prod_manual + - .also_login_dev_registry + parallel: + matrix: + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE + +deploy_to_prod_stable: + stage: deploy_prod + variables: + RELEASE_CHANNEL: stable + needs: ["build_prod"] + extends: + - .base_deploy + - .prod_manual + - .also_login_dev_registry + parallel: + matrix: + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE + +deploy_to_prod_rock_solid: + stage: deploy_prod + variables: + RELEASE_CHANNEL: rock-solid + needs: ["build_prod"] + extends: + - .base_deploy + - .prod_manual + - .also_login_dev_registry + parallel: + matrix: + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml new file mode 100644 index 0000000000..321b2edc9d --- /dev/null +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -0,0 +1,78 @@ +# Gitleaks secrets scanning. +# +# Migration of: +# - .github/workflows/gitleaks-scan-on-pr.yml (PR diff scan) +# - .github/workflows/gitleaks-scan-on-schedule.yml (daily full scan) +# +# Both GH workflows invoked `deckhouse/modules-actions/gitleaks@v6`. The +# upstream modules-gitlab-ci@v13.0 `.gitleaks_scan` template provides +# the same functionality via the GitLab-native gitleaks release binary. +# The upstream template already exposes three reusable jobs: +# +# gitleaks_diff -> SCAN_MODE=diff, runs on MR +# gitleaks_full_manual -> SCAN_MODE=full, manual +# gitleaks_full_scheduled -> SCAN_MODE=full, schedule +# +# We re-declare each of them here with project-specific names and stage/rules +# so the owner of this file can adjust rules:changes / labels without touching +# the upstream file. The upstream file also defines visible jobs; because this +# local file is included after the upstream template, these same-name overrides +# disable those generic jobs and leave only the project-specific jobs below. + +# Disable generic visible jobs from the upstream include. Keep the hidden +# `.gitleaks_scan` template active for project-specific jobs below. +# +# These overrides MUST set `stage: scan`: the upstream `.gitleaks_scan` +# template declares `stage: gitleaks`, but that stage was removed from +# stages.yml (it was dead — every active gitleaks job runs in `scan`). Even +# with `when: never`, GitLab validates the stage exists at config-parse +# time, so leaving the inherited `gitleaks` stage here would break the +# pipeline. +gitleaks_diff: + extends: .gitleaks_scan + stage: scan + rules: + - when: never + +gitleaks_full_manual: + extends: .gitleaks_scan + stage: scan + rules: + - when: never + +gitleaks_full_scheduled: + extends: .gitleaks_scan + stage: scan + rules: + - when: never + +gitleaks:diff: + extends: .gitleaks_scan + stage: scan + variables: + SCAN_MODE: "diff" + GIT_DEPTH: "0" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + +gitleaks:full:scheduled: + extends: .gitleaks_scan + stage: scan + interruptible: false + variables: + SCAN_MODE: "full" + GIT_DEPTH: "0" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "gitleaks-full"' + +gitleaks:full:manual: + extends: .gitleaks_scan + stage: scan + interruptible: false + variables: + SCAN_MODE: "full" + GIT_DEPTH: "0" + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - when: never diff --git a/.gitlab/ci/jobs/info.yml b/.gitlab/ci/jobs/info.yml new file mode 100644 index 0000000000..98fa9cc4c0 --- /dev/null +++ b/.gitlab/ci/jobs/info.yml @@ -0,0 +1,64 @@ +# Info-stage jobs: kubectl manifest printers. +# +# Carries forward show_dev_manifest and show_main_manifest from the +# previous root .gitlab-ci.yml. These jobs do not run any heavy tooling; +# they just print a copy-paste helper so testers can quickly wire up a +# ModuleConfig + ModulePullOverride for the build under test. +# +# Both extend .info (script body) and either .dev (MR pipelines) or .main +# (pushes to the default branch) for MODULES_MODULE_TAG + registry vars. + +show_dev_manifest: + stage: info + extends: + - .info + - .dev + +show_main_manifest: + stage: info + extends: + - .info + - .main + +# set_vars: derive label-driven build variables for the dev build jobs. +# +# Mirrors the GitHub Actions `set_vars` job from dev_module_build.yml: +# - MODULE_EDITION CE when the MR carries the `edition/ce` label, else EE. +# - DEBUG_COMPONENT the first `delve/*` label (empty if none). +# - RELEASE_IN_DEV true on release-X.Y branches, false otherwise. +# +# The dotenv artifact is consumed by build_dev / build_dev_tags / build_main +# via `needs: [{job: set_vars, artifacts: true}]`. werf reads MODULE_EDITION +# (.werf/consts.yaml: `env "MODULE_EDITION" "EE"`) and DEBUG_COMPONENT +# (werf.yaml: `env "DEBUG_COMPONENT" ""`) from the process environment, so +# the dotenv vars flow straight into the werf build. +# +# MODULES_MODULE_TAG is intentionally NOT emitted here (see set-vars.sh); +# each build job still derives its tag from .dev / .dev_tags / .main. +# +# Rules mirror the union of the dev build jobs' rules (.dev / .dev_tags / +# .main / .release) so set_vars always runs when any of them runs and the +# `needs:` edge is satisfied. Specifying an explicit `needs:` on the build jobs +# also keeps them in DAG mode, so the svace:set-vars dotenv still does not leak in. + +set_vars: + stage: info + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash + script: + - bash .gitlab/ci/scripts/bash/set-vars.sh + artifacts: + reports: + dotenv: set_vars.env + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" + when: always + # Release-branch pushes (squash-merge into release-X.Y) so build_release + # can consume the set_vars dotenv (RELEASE_IN_DEV=true, MODULE_EDITION). + - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+/' + when: always + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-dev.*$/' + when: always + - when: never diff --git a/.gitlab/ci/jobs/lint-dmt.yml b/.gitlab/ci/jobs/lint-dmt.yml new file mode 100644 index 0000000000..76433a0991 --- /dev/null +++ b/.gitlab/ci/jobs/lint-dmt.yml @@ -0,0 +1,42 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DMT (Deckhouse Module Tester) linter. +# +# Ports the GitHub Actions `lint_dmt` job from +# .github/workflows/dev_module_build.yml, which ran +# deckhouse/modules-actions/lint@v2 with continue-on-error: true. +# +# The upstream modules-gitlab-ci Build.gitlab-ci.yml ships a hidden `.lint` +# template that runs `dmt lint ./` with allow_failure: true. We do NOT +# include Build.gitlab-ci.yml directly: its `.build`/`.deploy` hidden jobs +# carry baked-in `rules:` that would bypass our strict .dev/.main/.prod +# gating (see .gitlab/ci/includes.yml). Instead this job mirrors the `.lint` +# script body and relies on the global `before_script` from +# Setup.gitlab-ci.yml (already included) to install dmt via trdl. +# +# dmt is NOT a host tool (it is installed per-job via trdl), so this job +# intentionally does NOT override `before_script` with check-runner-tools.sh; +# it inherits Setup's before_script, which performs the trdl+dmt bootstrap. +# +# DMT_METRICS_URL / DMT_METRICS_TOKEN are optional CI/CD variables consumed +# by dmt for metrics reporting. + +lint:dmt: + stage: lint + allow_failure: true + script: + - dmt lint ./ + extends: + - .dev diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml new file mode 100644 index 0000000000..3e789bfcb4 --- /dev/null +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -0,0 +1,418 @@ +# Validation, lint, scan, and gitlab-ci lint jobs. +# +# This file is the direct migration of the GitHub Actions workflow +# .github/workflows/dev_validation.yaml (paths_filter + no_cyrillic + +# doc_changes + shellcheck + helm_templates + check_gens_files). +# +# The `actionlint` job from the GH workflow is intentionally NOT migrated. In its place we run the new +# `lint:gitlab-ci` job that validates the GitLab CI configuration itself +# via the GitLab CI Lint API. The legacy skip-label +# `validation/skip/actionlint` is therefore obsolete and not honored. +# +# Each job: +# - inherits `tags: [deckhouse]` and `interruptible: true` from +# .gitlab/ci/defaults.yml (these short-lived checks are safe to cancel on a +# new push); +# - honors `validation/skip/` labels via `when: never` rules. +# +# The job file assumes the upstream template include for +# `/templates/Setup.gitlab-ci.yml` from deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 +# is in scope so that the shared `default:` and `stages:` are merged. +# It also assumes the `lint` stage is declared in .gitlab/ci/stages.yml. +# +# Coordination note: the parent .gitlab/ci/includes.yml (owned by the +# epic owner) is responsible for adding `local:` entries for this file. + +# --------------------------------------------------------------------------- +# Validation jobs +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# no_cyrillic +# --------------------------------------------------------------------------- + +lint:no-cyrillic: + stage: lint + variables: + GIT_DEPTH: "0" + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git + - git fetch --no-tags origin +main:refs/remotes/origin/main + script: + - task validation:no-cyrillic + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/no_cyrillic/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "lint-validate"' + +# --------------------------------------------------------------------------- +# doc_changes +# --------------------------------------------------------------------------- + +lint:doc-changes: + stage: lint + variables: + GIT_DEPTH: "0" + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git + - git fetch --no-tags origin +main:refs/remotes/origin/main + script: + - task validation:doc-changes + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/doc_changes/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + +# --------------------------------------------------------------------------- +# shellcheck +# +# Reuses the upstream-equivalent task `lint:shellcheck`. The .github +# workflow additionally scanned images/virtualization-artifact/hack/*.sh; +# the Taskfile already does that. +# --------------------------------------------------------------------------- + +lint:shellcheck: + stage: lint + # The `task lint:shellcheck` target runs shellcheck inside the + # koalaman/shellcheck-alpine Docker image, so the runner needs docker — + # not a host-installed shellcheck binary. + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task docker + script: + - task lint:shellcheck + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/shellcheck/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + +# --------------------------------------------------------------------------- +# yaml (prettier) +# --------------------------------------------------------------------------- + +lint:yaml: + stage: lint + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task + script: + - task lint:prettier:yaml + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - "**/*.yaml" + - "**/*.yml" + - ".gitlab-ci.yml" + - ".gitlab/**/*" + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + paths: + - "**/*.yaml" + - "**/*.yml" + - ".gitlab-ci.yml" + - ".gitlab/**/*" + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + changes: + paths: + - "**/*.yaml" + - "**/*.yml" + - ".gitlab-ci.yml" + - ".gitlab/**/*" + +# --------------------------------------------------------------------------- +# go (golangci-lint across every Go module) +# +# Migration of the GitHub Actions `lint_go` job from +# .github/workflows/dev_module_build.yml. It installs the pinned +# golangci-lint v${GOLANGCI_LINT_VERSION} (same method/version as the GH job) +# and runs `golangci-lint run` in every directory that ships a .golangci.yaml, +# excluding the upstream/vendored modules that GH also skipped: +# images/cdi-cloner/cloner-startup, images/dvcr-artifact. +# NOTE: .github/workflows/dev_module_build.yml also pruned ./test/performance/shatal, +# but that path never existed (the real module is test/performance/tools/shatal), +# so shatal was effectively linted there too. The dead prune entry is dropped here +# and test/performance/tools/shatal is intentionally linted. +# This supersedes the old single-dir `lint:virtualization-controller` job. +# --------------------------------------------------------------------------- + +lint:go: + stage: lint + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go curl + - | + set -e + # Install the pinned prebuilt golangci-lint v2 binary. + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b "$(go env GOPATH)/bin" "v${GOLANGCI_LINT_VERSION}" + export PATH="$(go env GOPATH)/bin:${PATH}" + golangci-lint --version + script: + - | + set -e + export PATH="$(go env GOPATH)/bin:${PATH}" + + # Directories containing a .golangci.yaml, excluding the same upstream / + # vendored modules as .github/workflows/dev_module_build.yml (lint_go): + # images/cdi-cloner/cloner-startup, images/dvcr-artifact. + # The GH prune of ./test/performance/shatal was a no-op (path never + # existed; real module is test/performance/tools/shatal), so shatal is + # intentionally linted here, matching actual current behavior. + mapfile -t config_dirs < <( + find . \ + -path ./images/cdi-cloner/cloner-startup -prune -o \ + -path ./images/dvcr-artifact -prune -o \ + -type f -name '.golangci.yaml' -not -type l -printf '%h\0' | \ + xargs -0 -n1 | sort -u + ) + count=${#config_dirs[@]} + echo "Found ${count} directories with golangci-lint configurations" + + error_count=0 + for dir in "${config_dirs[@]}"; do + echo "------------------------------------------------------------" + echo "Linting: ${dir}" + pushd "$dir" >/dev/null + if ! golangci-lint run; then + error_count=$(( error_count + 1 )) + fi + popd >/dev/null + done + + if [ "$error_count" -gt 0 ]; then + echo "${error_count} directory/directories failed golangci-lint" + exit 1 + fi + echo "All golangci-lint checks passed" + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/go/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "lint-validate"' + +# --------------------------------------------------------------------------- +# helm_templates +# +# GH used dorny/paths-filter to gate this on changes to helm-related +# files. GitLab equivalent: rules.changes. +# --------------------------------------------------------------------------- + +lint:helm-templates: + stage: lint + before_script: + # go + task are host tools; docker runs kubeconform; git/curl/python3 are + # needed by tools/kubeconform/kubeconform.sh (clone kubeconform.git, fetch + # CRD schemas, run openapi2jsonschema.py). + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task docker git curl python3 + - | + # Helm and jq are not guaranteed on the runner host. Install portable + # prebuilt binaries into a user-writable dir on PATH (no sudo needed). + set -e + case "$(uname -m)" in + x86_64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + *) arch="amd64" ;; + esac + bin_dir="$HOME/.local/bin" + mkdir -p "$bin_dir" + export PATH="$bin_dir:$PATH" + if ! command -v helm >/dev/null 2>&1; then + echo "Installing Helm v${HELM_VERSION} (${arch})..." + curl -sSfL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${arch}.tar.gz" -o /tmp/helm.tar.gz + tar -xzf /tmp/helm.tar.gz -C /tmp "linux-${arch}/helm" + install -m 0755 "/tmp/linux-${arch}/helm" "$bin_dir/helm" + helm version + fi + if ! command -v jq >/dev/null 2>&1; then + echo "Installing jq v${JQ_VERSION} (${arch})..." + curl -sSfL -o "$bin_dir/jq" "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${arch}" + chmod +x "$bin_dir/jq" + jq --version + fi + script: + - task validation:helm-templates + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/helm_templates/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - "crds/**/*" + - "charts/**/*" + - "tools/kubeconform/**/*" + - "templates/**/*" + - ".helmignore" + - "Chart.yaml" + - "Taskfile.yaml" + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + paths: + - "crds/**/*" + - "charts/**/*" + - "tools/kubeconform/**/*" + - "templates/**/*" + - ".helmignore" + - "Chart.yaml" + - "Taskfile.yaml" + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + changes: + paths: + - "crds/**/*" + - "charts/**/*" + - "tools/kubeconform/**/*" + - "templates/**/*" + - ".helmignore" + - "Chart.yaml" + - "Taskfile.yaml" + +# --------------------------------------------------------------------------- +# check_gens_files +# +# Matrix over components. Each component regenerates files via +# `task controller:dev:gogenerate` / `task generate` / `task +# vm-route-forge:gen` and fails if `git diff --exit-code` reports +# any drift against origin/main (or the current branch base). +# --------------------------------------------------------------------------- + +check:gens-files: + stage: lint + before_script: + # git is needed by check_diffs (git diff). go/task are host tools. + # docker is needed by the vm-route-forge matrix leg, which runs bpf2go + # inside the cilium/ebpf-builder image (the runner has no sudo to apt-get + # clang/llvm-strip). The other legs don't use docker but it must be + # present for the matrix job to start. + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git docker + - | + # `go install tool` (used by all components) puts binaries in + # $(go env GOPATH)/bin, but the shell runner does not add that dir to + # PATH. GitHub's setup-go@v5 does this automatically; replicate it here + # so //go:generate directives invoking `moq`, `ginkgo`, etc. resolve. + # (PATH is re-exported in the script block too, since each top-level + # script line is a fresh shell.) + export PATH="$(go env GOPATH)/bin:${PATH}" + echo "GOPATH/bin added to PATH: $(go env GOPATH)/bin" + script: + - | + set -e + # Restore PATH augmented in before_script for this shell too. + export PATH="$(go env GOPATH)/bin:${PATH}" + function check_diffs() { + local folder="$1" + if ! git diff --exit-code -- "$folder"; then + echo "::error title=Generated files out of date::Run 'go generate' / 'task generate' for $folder and commit the diff." + echo "--- git diff ---" + git diff origin/main -- "$folder" || true + echo "--- end ---" + exit 1 + fi + echo "OK: $folder" + } + case "$COMPONENT" in + virtualization-artifact) + # go.mod `tool` directives include moq + ginkgo; go install tool + # puts them in $GOPATH/bin (already on PATH above). Mirrors GH: + # cd images/virtualization-artifact && go install tool, then run + # generation from the root Taskfile namespace. + (cd images/virtualization-artifact && go install tool) + task controller:dev:gogenerate + check_diffs images/virtualization-artifact + ;; + vm-route-forge) + # bpf2go (go tool github.com/cilium/ebpf/cmd/bpf2go) needs clang, + # llvm-strip and the libbpf/kernel headers. The GitLab shell runner + # has no sudo, so instead of apt-get (as the GH workflow did) run + # generation inside the public cilium/ebpf-builder image, which ships + # clang-14/17/22, llvm-strip-14/17/22, libbpf-dev and linux headers. + # go generate runs in the container against the repo bind-mounted at + # /src. Use clang-14 to match the toolchain that produced the + # committed ebpf_x86_bpfel.{go,o} files (GH ran on ubuntu-22.04 where + # apt-get install clang pulls clang-14). + # + # BPF2GO_CC/STRIP point bpf2go at the versioned binaries (the image + # has no bare `clang`). BPF2GO_CFLAGS adds the arch-specific include + # dir so resolves (the image symlinks /usr/include/asm + # to asm-generic, which lacks ptrace.h; the real one lives under + # x86_64-linux-gnu on amd64 / aarch64-linux-gnu on arm64). + docker run --rm --platform linux/amd64 \ + -v "${PWD}:/src" \ + -w /src/images/vm-route-forge \ + -e BPF2GO_CC=clang-14 \ + -e BPF2GO_STRIP=llvm-strip-14 \ + -e BPF2GO_CFLAGS="-I/usr/include/x86_64-linux-gnu" \ + -e GOTOOLCHAIN=auto \ + ghcr.io/cilium/ebpf-builder:1777990914 \ + sh -c 'go generate ./...' + check_diffs images/vm-route-forge + ;; + api) + # `task generate` is defined in api/Taskfile.dist.yaml, not the + # root Taskfile — it must run from inside api/. Mirrors GH: + # cd ./api && go install tool && task generate. + (cd api && go install tool && task generate) + check_diffs api + ;; + *) + echo "Unknown COMPONENT: $COMPONENT"; exit 1 ;; + esac + parallel: + matrix: + - COMPONENT: [virtualization-artifact, api, vm-route-forge] + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - "api/**/*" + - "images/virtualization-artifact/**/*" + - "go.mod" + - "go.sum" + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - "images/vm-route-forge/bpf/route_watcher.c" + - "images/vm-route-forge/**/*" + - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + +# --------------------------------------------------------------------------- +# gitlab-ci-lint +# +# Replaces the legacy actionlint job (decision #5). Runs only when the +# GitLab CI configuration itself changes. Uses the GitLab CI Lint API +# at ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint. +# +# Authentication: PRIVATE-TOKEN via GITLAB_API_TOKEN (project access +# token, scope api). The script lives in +# .gitlab/ci/scripts/bash/gitlab-ci-lint.sh (owned by this issue). +# --------------------------------------------------------------------------- + +lint:gitlab-ci: + stage: lint + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - bash .gitlab/ci/scripts/bash/gitlab-ci-lint.sh + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - ".gitlab-ci.yml" + - ".gitlab/**/*" + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + paths: + - ".gitlab-ci.yml" + - ".gitlab/**/*" + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + changes: + paths: + - ".gitlab-ci.yml" + - ".gitlab/**/*" + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "lint-validate"' diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml new file mode 100644 index 0000000000..dc9ce41ca1 --- /dev/null +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -0,0 +1,52 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Manual / scheduled helper jobs that don't fit elsewhere. +# +# Currently: +# - mrs:summary : post a Loop summary of open MRs (GitLab counterpart of +# .github/workflows/dev_prs-summary.yml / .github/scripts/prs_notifier.mjs). +# +# GitHub slash-command dispatch +# (/.github/workflows/dispatch-slash-command.yml) is removed and replaced +# with manual pipelines. The two manual jobs in this file are the closest +# equivalents. There is no automated webhook listener, and this is an +# accepted permanent gap (no webhook-listener will be built). + +# Variables for "Run pipeline" UI: +# LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. +# DOC_REVIEWER (optional) GitLab username of doc reviewer. +# MANAGER_LOOP_NAME (optional) @firstname.lastname of the manager. + +mrs:summary: + stage: notify + tags: + - deckhouse + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh node npm + - corepack enable || true + - cd .gitlab/scripts/js + - npm ci --omit=dev || npm install --omit=dev + script: + - node mrs_notifier.mjs + rules: + # Manual trigger from "Run pipeline" UI. + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + # Scheduled daily run (e.g. 10:00 Moscow time — configure in + # Settings -> CI/CD -> Schedules), under the dedicated mrs-summary + # pipeline schedule ($SCHEDULE_TYPE == "mrs-summary"). + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "mrs-summary" + when: manual + allow_failure: true diff --git a/.gitlab/ci/jobs/precache.yml b/.gitlab/ci/jobs/precache.yml new file mode 100644 index 0000000000..b424b93aea --- /dev/null +++ b/.gitlab/ci/jobs/precache.yml @@ -0,0 +1,47 @@ +# Scheduled/manual precache builds. +# +# Migration of .github/workflows/dev_build_precache.yml. The original GH +# workflow rebuilt the module against `main` every 8 hours to keep the +# dev registry warm. +# +# GitLab mapping: +# on: schedule (cron: "0 */8 * * *") -> Pipeline Schedule in UI +# + rules: schedule +# ($SCHEDULE_TYPE == "precache") +# on: workflow_dispatch -> when: manual (Run pipeline) +# matrix.branch: [main] -> single .main job +# +# The actual build uses the local `.base_build` template, which mirrors the +# upstream Build.gitlab-ci.yml script without inheriting its broad rules. This +# file sets the trigger surface and the per-pipeline variables that the build +# template expects. + +precache:build:main: + extends: + - .base_build + - .main + stage: build + interruptible: false + variables: + MODULES_MODULE_TAG: "${CI_COMMIT_REF_NAME:-main}" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "precache"' + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + +# Manual override that lets an operator rebuild against a specific +# branch (e.g. release-1.21) by setting REF_BRANCH when running the +# pipeline via UI. Mirrors the workflow_dispatch + pr_number input from +# the GH workflow but keeps it branch-driven for precache. +precache:build:branch: + extends: + - .base_build + - .dev_vars + stage: build + interruptible: false + variables: + MODULES_MODULE_TAG: "${REF_BRANCH}" + rules: + - if: '$CI_PIPELINE_SOURCE == "web" && $REF_BRANCH' + when: manual + - when: never diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml new file mode 100644 index 0000000000..15adb05316 --- /dev/null +++ b/.gitlab/ci/jobs/release-channels.yml @@ -0,0 +1,452 @@ +# Prod release-channels parity with GitHub. +# +# Port of .github/workflows/release_module_release-channels.yml: a manual, +# single-channel/single-tag release dispatch with on-demand build, deploy, +# version verification, GitLab release creation, and a Loop notification. +# +# Trigger: CI/CD -> Pipelines -> Run pipeline, on the tag $RELEASE_TAG +# (CI_PIPELINE_SOURCE == "web"). These jobs run ONLY on such manual web +# pipelines, so they never collide with the tag-push flow +# (build_prod / deploy_to_prod_* in build-prod.yml / deploy-prod.yml). +# +# Run pipeline UI variables (all optional except RELEASE_TAG): +# RELEASE_CHANNEL alpha|beta|early-access|stable|rock-solid (default alpha) +# RELEASE_TAG module tag like v1.21.1 (required, ^v\d+\.\d+\.\d+$) +# EDITION_CE "true"/"false" (default false) — deploy CE +# EDITION_EE "true"/"false" (default false) — deploy EE + SE-Plus + FE +# ENABLE_BUILD "true"/"false" (default true) — run build before deploy +# CHECK_ONLY "true"/"false" (default false) — verify only, skip build/deploy +# SKIP_REQUIREMENTS_CHECK "true"/"false" (default false) +# RELEASE_TO_GITLAB "true"/"false" (default true) — create GitLab release +# SEND_RESULTS_TO_LOOP "true"/"false" (default true) — post status to Loop +# +# Parity notes vs GitHub: +# - Release creation targets GitLab Releases (POST /projects/:id/releases) +# instead of GitHub Releases, because development is moving to GitLab. +# The notes source is preserved: the merged changelog MR +# (label:changelog, milestone:$RELEASE_TAG) description. +# - Run the pipeline ON the tag $RELEASE_TAG (select it as the Run pipeline +# ref). The checkout is then already at the tag, so the upstream Setup +# before_script (werf ci-env / registry login) keeps working untouched. +# prod:print-vars enforces RELEASE_TAG == CI_COMMIT_TAG so a mismatch +# fails fast instead of building the wrong ref. GitHub let `tag` differ +# from the workflow ref (it checked out ref:inputs.tag); GitLab cannot +# do that without overriding the inherited Setup before_script, which +# we avoid. +# - Per-(channel,tag) concurrency uses resource_group (a mutex); GitLab has +# no cancel-in-progress equivalent, so concurrent same-(channel,tag) +# dispatches serialize instead of cancelling. +# - SE-Plus / FE builds `needs: prod:build:ee` to mirror the GitHub +# job-SE-Plus/job-FE cascade; ce/ee build in parallel. + +variables: + RELEASE_CHANNEL: + value: "alpha" + options: ["alpha", "beta", "early-access", "stable", "rock-solid"] + description: "Release channel to deploy" + RELEASE_TAG: + value: "" + description: "Module tag like v1.21.1 (required, ^v\\d+\\.\\d+\\.\\d+$). Run the pipeline on this tag." + EDITION_CE: + value: "false" + options: ["true", "false"] + description: "Build+deploy CE edition" + EDITION_EE: + value: "false" + options: ["true", "false"] + description: "Build+deploy EE + SE-Plus + FE editions" + ENABLE_BUILD: + value: "true" + options: ["true", "false"] + description: "Run build before deploy (false = deploy only, image already in registry)" + CHECK_ONLY: + value: "false" + options: ["true", "false"] + description: "Skip build/deploy; only verify the version on the release channel" + SKIP_REQUIREMENTS_CHECK: + value: "false" + options: ["true", "false"] + description: "Skip the Deckhouse version requirements check before build" + RELEASE_TO_GITLAB: + value: "true" + options: ["true", "false"] + description: "Create a GitLab release from the merged changelog MR" + SEND_RESULTS_TO_LOOP: + value: "true" + options: ["true", "false"] + description: "Send the release result summary to Loop via webhook" + +# Common rule: only on manual web pipelines with a valid tag. Every prod:* job +# extends this; interruptible:false keeps the whole manual release flow from +# being auto-cancelled (overrides the global interruptible:true default). +.prod_release_rules: + interruptible: false + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ + - when: never + +# --------------------------------------------------------------------------- +# 1. print-vars + tag/ref validation +# --------------------------------------------------------------------------- +prod:print-vars: + stage: info + extends: .prod_release_rules + variables: + MODULES_MODULE_TAG: "${RELEASE_TAG}" + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash + script: + - | + set -e + echo "MODULES_REGISTRY=${PROD_REGISTRY}" + echo "MODULES_MODULE_SOURCE=${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}//modules" + echo "MODULES_MODULE_NAME=${MODULES_MODULE_NAME}" + echo "RELEASE_CHANNEL=${RELEASE_CHANNEL}" + echo "RELEASE_TAG=${RELEASE_TAG}" + echo "EDITION_CE=${EDITION_CE} EDITION_EE=${EDITION_EE}" + echo "ENABLE_BUILD=${ENABLE_BUILD} CHECK_ONLY=${CHECK_ONLY}" + echo "SKIP_REQUIREMENTS_CHECK=${SKIP_REQUIREMENTS_CHECK}" + echo "RELEASE_TO_GITLAB=${RELEASE_TO_GITLAB} SEND_RESULTS_TO_LOOP=${SEND_RESULTS_TO_LOOP}" + echo "${RELEASE_TAG}" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + || { echo "Error: invalid tag format '${RELEASE_TAG}'. Use vX.Y.Z"; exit 1; } + # The pipeline must run on the tag itself so the checkout is at the tag + # and the upstream Setup before_script works unchanged. + if [ "${CI_COMMIT_TAG:-}" != "${RELEASE_TAG}" ]; then + echo "Error: RELEASE_TAG (${RELEASE_TAG}) must equal the pipeline ref tag" \ + "(CI_COMMIT_TAG=${CI_COMMIT_TAG:-}). Run the pipeline on the tag." + exit 1 + fi + +# --------------------------------------------------------------------------- +# 2. check-requirements-before-build +# --------------------------------------------------------------------------- +prod:check-requirements: + stage: prod_check + extends: + - .prod_release_rules + - .login_prod_read_registry + needs: ["prod:print-vars"] + variables: + MODULES_MODULE_TAG: "${RELEASE_TAG}" + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash go task curl jq + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $CHECK_ONLY != "true" && $SKIP_REQUIREMENTS_CHECK != "true" && ($EDITION_CE == "true" || $EDITION_EE == "true") + - when: never + script: + - CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" task -d tools/moduleversions check:requirements + +# --------------------------------------------------------------------------- +# 3+4. build (toggleable) + deploy, per edition +# --------------------------------------------------------------------------- +# Deploy step (crane copy :tag -> :channel). Separate from .base_deploy so +# the job can land in the deploy_prod stage. +.prod_deploy_script: + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh crane + script: + - | + REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" + IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" + IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy "${IMAGE_SRC}" "${IMAGE_DST}" + +# --- CE --- +prod:build:ce: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ce" + extends: + - .prod_release_rules + - .base_build + - .prod_vars + - .also_login_dev_registry + needs: + - job: prod:check-requirements + optional: true + variables: + EDITION: ce + MODULE_EDITION: CE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_CE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:ce: + stage: deploy_prod + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ce" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .also_login_dev_registry + needs: + - job: prod:check-requirements + optional: true + - job: prod:build:ce + optional: true + variables: + EDITION: ce + MODULE_EDITION: CE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_CE == "true" && $CHECK_ONLY != "true" + - when: never + +# --- EE --- +prod:build:ee: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ee" + extends: + - .prod_release_rules + - .base_build + - .prod_vars + - .also_login_dev_registry + needs: + - job: prod:check-requirements + optional: true + variables: + EDITION: ee + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:ee: + stage: deploy_prod + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ee" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .also_login_dev_registry + needs: + - job: prod:check-requirements + optional: true + - job: prod:build:ee + optional: true + variables: + EDITION: ee + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" + - when: never + +# --- SE-Plus (needs EE build, mirrors GH job-SE-Plus) --- +prod:build:se-plus: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:se-plus" + extends: + - .prod_release_rules + - .base_build + - .prod_vars + - .also_login_dev_registry + needs: ["prod:build:ee"] + variables: + EDITION: se-plus + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:se-plus: + stage: deploy_prod + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:se-plus" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .also_login_dev_registry + needs: + - job: prod:build:se-plus + optional: true + variables: + EDITION: se-plus + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" + - when: never + +# --- FE (needs EE build, mirrors GH job-FE) --- +prod:build:fe: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:fe" + extends: + - .prod_release_rules + - .base_build + - .prod_vars + - .also_login_dev_registry + needs: ["prod:build:ee"] + variables: + EDITION: fe + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:fe: + stage: deploy_prod + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:fe" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .also_login_dev_registry + needs: + - job: prod:build:fe + optional: true + variables: + EDITION: fe + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" + - when: never + +# --------------------------------------------------------------------------- +# 5. check-version-on-release-channel (registry / releases / documentation) +# --------------------------------------------------------------------------- +prod:check-version: + stage: prod_verify + extends: + - .prod_release_rules + - .login_prod_read_registry + needs: + - job: prod:deploy:ce + optional: true + - job: prod:deploy:ee + optional: true + - job: prod:deploy:se-plus + optional: true + - job: prod:deploy:fe + optional: true + parallel: + matrix: + - CHECK: [registry, releases, documentation] + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash go task crane curl jq + script: + - | + set -euo pipefail + if [ "$CHECK_ONLY" != "true" ]; then + echo "Waiting for the site to update (versions usually update within 5 minutes)..." + sleep 300 + fi + case "$CHECK" in + registry) + CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" task -d tools/moduleversions check:registry + ;; + releases) + echo "Checking the version is deployed on the releases site (10 attempts, 60s apart)" + CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" COUNT=10 task -d tools/moduleversions check:releases + ;; + documentation) + echo "Checking the version is deployed on the docs site (5 attempts, 60s apart)" + CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" COUNT=5 task -d tools/moduleversions check:docs + ;; + esac + +# --------------------------------------------------------------------------- +# 6. create-gitlab-release (was create-github-release) +# --------------------------------------------------------------------------- +prod:create-gitlab-release: + stage: prod_release + extends: .prod_release_rules + needs: + - job: prod:deploy:ce + optional: true + - job: prod:deploy:ee + optional: true + - job: prod:deploy:se-plus + optional: true + - job: prod:deploy:fe + optional: true + - job: prod:check-version + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $CHECK_ONLY != "true" && $RELEASE_TO_GITLAB == "true" && ($EDITION_CE == "true" || $EDITION_EE == "true") + - when: never + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - | + set -euo pipefail + source .gitlab/ci/scripts/bash/lib/api.sh + TAG="${RELEASE_TAG}" + + # Skip if a release already exists. + if api GET "/projects/${CI_PROJECT_ID}/releases/${TAG}" >/dev/null 2>&1; then + echo "GitLab release already exists for ${TAG}, skipping" + { + echo "GH_RELEASE_STATUS=skipped" + echo "GH_RELEASE_REASON=GitLab release already exists for ${TAG}" + } > release.env + exit 0 + fi + + # Find the single merged changelog MR for this milestone (label:changelog). + MRS=$(api GET "/projects/${CI_PROJECT_ID}/merge_requests?state=merged&labels=changelog&milestone=${TAG}") + COUNT=$(echo "$MRS" | jq 'length') + echo "Found ${COUNT} merged changelog MR(s) for milestone ${TAG}" + if [ "$COUNT" -ne 1 ]; then + echo "Expected exactly one merged changelog MR for milestone ${TAG}, found ${COUNT}" >&2 + exit 1 + fi + + PR_BODY=$(echo "$MRS" | jq -r '.[0].description') + PR_IID=$(echo "$MRS" | jq -r '.[0].iid') + PR_URL=$(echo "$MRS" | jq -r '.[0].web_url') + if [ -z "${PR_BODY//[[:space:]]/}" ]; then + echo "Changelog MR !${PR_IID} description is empty" >&2 + exit 1 + fi + + # Create the GitLab release from the changelog MR description. + api POST "/projects/${CI_PROJECT_ID}/releases" \ + --data "$(jq -n --arg tag "$TAG" --arg name "$TAG" --arg desc "$PR_BODY" \ + '{tag_name: $tag, name: $name, description: $desc}')" + + echo "GitLab release created for ${TAG}" + { + echo "GH_RELEASE_STATUS=created" + echo "GH_RELEASE_URL=${CI_PROJECT_URL}/-/releases/${TAG}" + echo "GH_RELEASE_PR_IID=${PR_IID}" + echo "GH_RELEASE_PR_URL=${PR_URL}" + echo "GH_RELEASE_REASON=GitLab release created from changelog MR !${PR_IID}" + } > release.env + artifacts: + reports: + dotenv: release.env + +# --------------------------------------------------------------------------- +# 7. send-release-results-to-loop +# --------------------------------------------------------------------------- +prod:notify-loop: + stage: notify + extends: .prod_release_rules + needs: + - job: prod:deploy:ce + optional: true + - job: prod:deploy:ee + optional: true + - job: prod:deploy:se-plus + optional: true + - job: prod:deploy:fe + optional: true + - job: prod:check-version + optional: true + - job: prod:create-gitlab-release + optional: true + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $SEND_RESULTS_TO_LOOP == "true" + when: on_success + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $SEND_RESULTS_TO_LOOP == "true" + when: on_failure + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - bash .gitlab/ci/scripts/bash/notify-release-loop.sh diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml new file mode 100644 index 0000000000..6ea9ed5377 --- /dev/null +++ b/.gitlab/ci/jobs/svace.yml @@ -0,0 +1,181 @@ +# Svace scheduled + manual analysis. +# +# Migration of .github/workflows/dev_build_svace.yml. The GH workflow +# did: +# 1. set_vars - compute MODULES_MODULE_TAG-*svace +# 2. dev_setup_build - build with svace_enabled=true (deferred to +# deckhouse/modules-actions/build@v4) +# 3. analyze_build - run svace_analyze@v4 against the built artifacts +# 4. notify - send Loop webhook with success/failure +# +# GitLab mapping: +# - The build step reuses the local `.base_build` template, with `.dev_vars` +# for DEV registry variables and job-local rules for schedule/manual runs. +# - The analyze step uses the upstream `.svace_analyze` template +# (Svace_Analayze.gitlab-ci.yml; note upstream file name typo). +# - Schedule cron: "00 04 * * 6" -> Pipeline Schedule in UI, with +# $SCHEDULE_TYPE == "svace" so the svace jobs run only under that +# dedicated schedule and not under any other pipeline schedule. +# - workflow_dispatch -> when: manual. +# - MR opt-in -> set SVACE_PIPELINE_ENABLED=true when running a pipeline. +# - Loop notification kept as a manual / always job. The original GH +# webhook URL is the LOOP_WEBHOOK_URL masked variable. +# +# Required CI/CD variables (declared in the project): +# SVACE_ANALYZE_HOST masked +# SVACE_ANALYZE_SSH_USER masked +# SVACE_ANALYZE_SSH_PRIVATE_KEY_B64 masked +# SVACER_URL var +# SVACER_IMPORT_USER masked +# SVACER_IMPORT_PASSWORD masked +# LOOP_WEBHOOK_URL masked +# +# Tag suffix: `${CI_COMMIT_REF_NAME}-svace` matches the GH workflow +# behavior. + +# --------------------------------------------------------------------------- +# set-vars: produce a dotenv artifact with MODULES_MODULE_TAG-*svace. +# --------------------------------------------------------------------------- + +svace:set-vars: + extends: + - .info + stage: info + # Only emit the svace tag dotenv in contexts where svace actually runs. + # Without these rules this job ran on EVERY MR pipeline and leaked a + # MODULES_MODULE_TAG=-svace dotenv artifact into build jobs, + # overriding their own mr tag. Runs automatically (no `when: manual`) + # so the manual svace:build job can consume its artifact. + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' + when: always + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' + when: always + - when: never + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash + script: + - | + set -euo pipefail + REF="${CI_COMMIT_REF_NAME:-}" + case "$REF" in + main|release-*) + TAG="${REF}-svace" ;; + "") + TAG="manual-svace" ;; + *) + TAG="${REF//\//_}-svace" ;; + esac + echo "MODULES_MODULE_TAG=${TAG}" > svace.env + echo "MODULE_EDITION=EE" >> svace.env + echo "REF_NAME=${REF}" >> svace.env + cat svace.env + artifacts: + reports: + dotenv: svace.env + +# --------------------------------------------------------------------------- +# build: produces Svace-instrumented artifacts. +# +# Reuses `.base_build` so it does not inherit the dev-tag-only rules from +# `.dev_tags`. The preceding svace:set-vars job writes the actual +# MODULES_MODULE_TAG into a dotenv artifact. +# --------------------------------------------------------------------------- + +svace:build: + extends: + - .base_build + - .dev_vars + stage: build + interruptible: false + needs: + - svace:set-vars + variables: + WERF_VIRTUAL_MERGE: "0" + SVACE_ENABLED: "true" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' + when: manual + - when: never + +# --------------------------------------------------------------------------- +# analyze: invoke the upstream `.svace_analyze` template. +# +# Id-token + vault wiring is inherited from the upstream template. +# --------------------------------------------------------------------------- + +svace:analyze: + extends: + - .svace_analyze + stage: scan + interruptible: false + needs: + - job: svace:set-vars + artifacts: true + - job: svace:build + optional: true + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' + when: manual + - when: never + +# --------------------------------------------------------------------------- +# notify: post the result to Loop (best-effort). +# --------------------------------------------------------------------------- + +svace:notify: + stage: cleanup + # needs svace:analyze (non-optional) so this job stays coupled to a real + # svace run: on an unrelated web pipeline svace:analyze is manual and never + # played, so it is skipped and this notify is skipped too. The on_success / + # on_failure rule pair (same idiom as prod:notify-loop) makes notify run on + # BOTH a passing and a failing svace:analyze. set-vars is pulled for the + # REF_NAME dotenv. + needs: + - job: svace:set-vars + artifacts: true + - job: svace:analyze + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - | + set -euo pipefail + source .gitlab/ci/scripts/bash/lib/api.sh + gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_PIPELINE_ID LOOP_WEBHOOK_URL + DATE=$(date '+%Y-%m-%d') + # Derive the result from the svace:analyze job, not this notify job: + # CI_JOB_STATUS would report THIS job and is almost always "success". + # Mirrors the GitHub `needs.analyze_build.result == 'success'` check. + ANALYZE_STATUS=$(api GET "/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100" \ + | jq -r '[.[] | select(.name == "svace:analyze")][0].status // ""') + if [[ "${ANALYZE_STATUS}" == "success" ]]; then + STATUS=":white_check_mark: SUCCESS!" + else + STATUS=":x: FAIL!" + fi + MESSAGE="### :gear: **DVP ${DATE} Weekly Svace Analyze Report** + **Branch:** \`${REF_NAME:-${CI_COMMIT_REF_NAME}}\` + **Status:** ${STATUS} + [:link: Pipeline](${CI_PIPELINE_URL})" + curl --silent --show-error --fail \ + --request POST \ + --header 'Content-Type: application/json' \ + --data "$(jq -n --arg text "${MESSAGE}" '{text: $text}')" \ + "${LOOP_WEBHOOK_URL}" || echo "Loop webhook failed (non-fatal)" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' + when: on_success + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' + when: on_failure + - if: '$CI_PIPELINE_SOURCE == "web"' + when: on_success + - if: '$CI_PIPELINE_SOURCE == "web"' + when: on_failure diff --git a/.gitlab/ci/jobs/test-d8v-cli.yml b/.gitlab/ci/jobs/test-d8v-cli.yml new file mode 100644 index 0000000000..832e870e10 --- /dev/null +++ b/.gitlab/ci/jobs/test-d8v-cli.yml @@ -0,0 +1,35 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# d8v CLI build + workability check. +# +# Ports the GitHub Actions `test_build_d8v_cli` job from +# .github/workflows/dev_module_build.yml, which ran `task d8v-cli:build`, +# installed the binary, and ran `d8v --help` as a smoke check (PR only). +# +# The root Taskfile.yaml `d8v-cli` include sets `dir: ./src/cli`, so +# `task d8v-cli:build` emits ./src/cli/d8v. The workability check invokes +# that binary with `--help` (cobra exits 0 on help). +# +# MR-gated via .dev (mirrors the GH PR-only trigger). + +test:build:d8v-cli: + stage: test + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task + script: + - task d8v-cli:build + - ./src/cli/d8v --help + extends: + - .dev diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml new file mode 100644 index 0000000000..db6df14c5d --- /dev/null +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -0,0 +1,68 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# JS unit tests for .gitlab/scripts/js. +# +# Ports the GitHub Actions `test_scripts_js` job from +# .github/workflows/dev_module_build.yml, which ran `npm test` in +# .github/scripts/js with Node.js 24. The GitLab counterpart lives in +# .gitlab/scripts/js (mrs_notifier.mjs); its `npm test` runs node:test over +# mrs_notifier.test.mjs. mrs_notifier.mjs guards run() behind a +# direct-invocation check and exports its pure helpers (classifyMR, +# extractUnresolvedThreads, shouldNotifyMR, formatUser), so the suite imports +# the module and exercises real classification/filter/formatting logic with +# synthetic data (no network), alongside structural smoke assertions. +# +# The shell runner host does not ship Node.js, so the job runs inside the +# official `node:${NODE_VERSION}` image via `docker run`, mirroring how +# check:gens-files runs the vm-route-forge leg in cilium/ebpf-builder. The +# repo is bind-mounted READ-ONLY at /src and copied to an ephemeral tmpdir +# inside the container, so node_modules never pollutes the shared +# shell-executor workspace (root-owned files there break `git clean` on +# subsequent checkouts of unrelated jobs). The committed package-lock.json +# lets the job use `npm ci` for reproducible installs. + +test:scripts:js: + stage: test + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh docker + script: + - | + set -e + # Run npm in a throwaway tmpdir INSIDE the container so node_modules + # never lands in the shared shell-executor workspace. Previously npm + # ran directly against a read-write + # bind-mount of the repo: under the default container uid (root) it + # created root-owned node_modules/ that the runner-user could not + # remove, which broke `git clean` on the next job's checkout and + # failed unrelated jobs (show_dev_manifest, gitleaks:diff, ...) with + # "warning: failed to remove ...: Permission denied". + # + # The repo is mounted read-only at /src; the test sources are copied + # to an ephemeral tmpdir where npm ci/npm test execute. + docker run --rm --platform linux/amd64 \ + -v "${PWD}:/src:ro" \ + -u "$(id -u):$(id -g)" \ + -e HOME=/tmp \ + node:${NODE_VERSION} \ + sh -c ' + set -e + work="$(mktemp -d)" + cp -a /src/.gitlab/scripts/js/. "$work"/ + cd "$work" + npm ci --no-audit --no-fund --cache /tmp/.npm + npm test + ' + extends: + - .dev diff --git a/.gitlab/ci/jobs/test-scripts-python.yml b/.gitlab/ci/jobs/test-scripts-python.yml new file mode 100644 index 0000000000..eabbf084b8 --- /dev/null +++ b/.gitlab/ci/jobs/test-scripts-python.yml @@ -0,0 +1,40 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python unit tests for .gitlab/ci/scripts/python. +# +# Ports the GitHub Actions `test_scripts_python` job from +# .github/workflows/dev_module_build.yml, which ran `python -m unittest` in +# .github/scripts/python. The GitLab Python scripts +# (changelog_collect.py, check_changelog_entry.py) ship real unittest suites +# under .gitlab/ci/scripts/python/tests/ covering the pure parse/validate/ +# render helpers (no network access). A py_compile pass first catches syntax +# errors in any script not reached by an import; then unittest discover runs +# the suites. Standard library only — no third-party deps to install. +# +# MR-gated via .dev. + +test:scripts:python: + stage: test + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh python3 + script: + - python3 -m py_compile .gitlab/ci/scripts/python/*.py + - >- + python3 -m unittest discover + -s .gitlab/ci/scripts/python/tests + -t .gitlab/ci/scripts/python + -p 'test_*.py' + extends: + - .dev diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml new file mode 100644 index 0000000000..ed08efbd64 --- /dev/null +++ b/.gitlab/ci/jobs/test.yml @@ -0,0 +1,37 @@ +# Unit-test jobs. +# +# Carries forward the test jobs from the previous root .gitlab-ci.yml: +# test:virtualization-controller -> task virtualization-controller:init + test:unit +# test:hooks -> task gohooks:test +# +# Go linting of every module (.golangci.yaml dirs) lives in +# .gitlab/ci/jobs/lint-validate.yml -> lint:go (migration of the GH lint_go job); +# the old single-dir lint:virtualization-controller job was removed as redundant. +# +# All test jobs extend .dev so they only run on MR pipelines with the right +# MODULES_MODULE_TAG/registry context. + +# `needs: []` mirrors the GitHub dev_module_build.yml `test` job, which has no +# `needs` and runs on every PR push (pull_request: synchronize) independently of +# lint/build. Without it the test jobs sit in the `test` stage waiting for the +# whole `lint` stage to succeed, which a still-unplayed blocking manual job +# (e.g. changelog:milestone) can stall indefinitely. DAG mode lets unit tests +# start immediately on every MR pipeline. +test:virtualization-controller: + stage: test + needs: [] + script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task + - task virtualization-controller:init + - task virtualization-controller:test:unit + extends: + - .dev + +test:hooks: + stage: test + needs: [] + script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task + - task gohooks:test + extends: + - .dev diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml new file mode 100644 index 0000000000..f928924739 --- /dev/null +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -0,0 +1,223 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Translate English CHANGELOG/CHANGELOG-v*.yml to Russian (.ru.yml) and open an MR. +# +# Migration of .github/workflows/translate-changelog.yml which called +# deckhouse/modules-actions/translate-changelog@v10 with source_lang: en, +# target_lang: ru, file_prefix: CHANGELOG-v, triggered on push to the +# default branch (main) and release-X.Y branches. +# +# Direction note (verified against upstream): +# The upstream modules-gitlab-ci@v13.0 `.translate_and_create_mr` template +# (templates/Translate_Changelog.gitlab-ci.yml) is hardcoded to the OPPOSITE +# direction (RU -> EN, globs `v*.ru.yml`) and exposes NO direction variables +# (only TRANSLATE_CHANGELOG_PATH / TRANSLATE_BASE_BRANCH). This repo has no +# *.ru.yml sources, only CHANGELOG-v*.yml (English), so the upstream script +# would no-op here. To match the GitHub workflow (EN -> RU) and this repo's +# file naming, this job overrides `script:` with an EN->RU implementation and +# declares the direction explicitly via TRANSLATE_SOURCE_LANG / +# TRANSLATE_TARGET_LANG, which are consumed by the local script below (NOT by +# the upstream template). The job still extends `.translate_and_create_mr` to +# inherit its image and base variables. +# +# Trigger: push to the default branch (main) or any release-X.Y branch, +# matching the GH workflow. The upstream template's default rule +# (`$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH`, which excludes main) is +# replaced by the explicit rules block below. +# +# Required CI/CD variable: RELEASE_TOKEN (or GITLAB_API_TOKEN, alias). +# The upstream template prefers RELEASE_TOKEN; if unset it falls back to +# CI_JOB_TOKEN (works in GitLab 15.9+ for push and MR create). Set +# RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN lacks +# push / merge-request permissions. +# +# We delegate to the upstream GitLab CI template at +# https://fox.flant.com/deckhouse/3p/deckhouse/modules-gitlab-ci +# (branch v13.0, HEAD 006d51c35904b434eca2045a449aafb5e37a8827). + +include: + - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" + ref: "v13.0" + file: "/templates/Translate_Changelog.gitlab-ci.yml" + +# Local job definition that extends the upstream hidden job. +# Tag override is intentional: we want our project's runner tag, not the +# upstream default. The upstream `script:` is overridden with an EN->RU +# implementation (see direction note above) because the upstream template +# direction is hardcoded RU->EN and cannot be parameterized. +translate:changelog: + extends: .translate_and_create_mr + stage: notify + # Mutates state (translates + opens an MR); never auto-cancel mid-run. + interruptible: false + tags: + - deckhouse + rules: + # Match the GH workflow trigger: push to main (default branch) or + # release-X.Y branches. Replaces the upstream default rule + # (`$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH`) which excludes main. + - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" + - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+$/' + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh git curl jq python3 + - python3 -m venv /tmp/venv + - /tmp/venv/bin/pip install --no-cache-dir deep-translator packaging + - git -C "${CI_PROJECT_DIR}" config user.email "gitlab-ci@gitlab.com" + - git -C "${CI_PROJECT_DIR}" config user.name "GitLab CI" + variables: + # Path to CHANGELOG dir (upstream default; explicit for clarity). + TRANSLATE_CHANGELOG_PATH: "CHANGELOG" + # Target branch the resulting MR should target. + TRANSLATE_BASE_BRANCH: "main" + # File prefix of English source changelogs (matches GH file_prefix). + TRANSLATE_FILE_PREFIX: "CHANGELOG-v" + # Translation direction (EN -> RU), matching the GH workflow. These are + # consumed by the local script below, NOT by the upstream template. + TRANSLATE_SOURCE_LANG: "en" + TRANSLATE_TARGET_LANG: "ru" + # Upstream expects RELEASE_TOKEN or falls back to CI_JOB_TOKEN in its shell + # script. GitLab does not evaluate shell parameter expansion in `variables:`; + # set RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN + # does not have enough permissions to push and create merge requests. + # The local before_script intentionally replaces the upstream template's + # container/package-manager setup so the job works on shell executors. + script: + - | + set -euo pipefail + export PATH="/tmp/venv/bin:$PATH" + CHANGELOG_PATH="${TRANSLATE_CHANGELOG_PATH}" + FILE_PREFIX="${TRANSLATE_FILE_PREFIX}" + SRC_LANG="${TRANSLATE_SOURCE_LANG}" + TGT_LANG="${TRANSLATE_TARGET_LANG}" + # Detect English source changelog changes in the last commit + # (exclude already-translated *.ru.yml files). + CHANGED=$(git show --name-only --pretty=format: HEAD \ + | grep "^${CHANGELOG_PATH}/${FILE_PREFIX}.*\.yml$" \ + | grep -v "\.ru\.yml$" || true) + if [ -z "$CHANGED" ]; then + echo "No ${FILE_PREFIX}*.yml (English) changelog files changed in last commit, skipping." + exit 0 + fi + echo "English changelog files changed in last commit: $CHANGED" + # Translate latest English changelog to Russian (EN -> RU), matching + # deckhouse/modules-actions/translate-changelog@v10. + TRANSLATE_OUTPUT=$(cd "${CI_PROJECT_DIR}" && python3 - "$CHANGELOG_PATH" "$FILE_PREFIX" "$SRC_LANG" "$TGT_LANG" << 'PYEOF' + import re + import sys + from pathlib import Path + try: + from packaging import version as pkg_version + from deep_translator import GoogleTranslator + except ImportError as e: + print(f"Import error: {e}", file=sys.stderr) + sys.exit(1) + changelog_dir, file_prefix, src_lang, tgt_lang = sys.argv[1:5] + path = Path(changelog_dir) + if not path.exists(): + sys.exit(0) + # English source files: *.yml excluding *.ru.yml. + en_files = [f for f in path.glob(f"{file_prefix}*.yml") if not f.name.endswith(".ru.yml")] + if not en_files: + sys.exit(0) + versions = [] + ver_re = re.compile(re.escape(file_prefix) + r"(\d+\.\d+\.\d+)\.yml$") + for f in en_files: + m = ver_re.match(f.name) + if m: + try: + ver = pkg_version.parse(m.group(1)) + versions.append((ver, f.name)) + except pkg_version.InvalidVersion: + pass + if not versions: + sys.exit(0) + versions.sort(reverse=True, key=lambda x: x[0]) + latest_ver, en_name = versions[0] + version_str = f"v{latest_ver}" + ru_name = en_name.replace(".yml", ".ru.yml") + if (path / ru_name).exists(): + sys.exit(0) + en_path, ru_path = path / en_name, path / ru_name + translator = GoogleTranslator(source=src_lang, target=tgt_lang) + with open(en_path, "r", encoding="utf-8") as f: + en_lines = f.readlines() + with open(ru_path, "w", encoding="utf-8") as f: + for line in en_lines: + if not line.strip(): + f.write(line) + continue + indent_len = len(line) - len(line.lstrip()) + content = line.strip() + try: + translated = translator.translate(content) + except Exception: + translated = content + f.write(" " * indent_len + translated + "\n") + print(f"VERSION={version_str}") + print(f"EN_FILE={en_name}") + print(f"RU_FILE={ru_name}") + PYEOF + ) || TRANSLATE_OUTPUT="" + # Parse VERSION from output; if missing, nothing was translated. + VERSION=$(echo "$TRANSLATE_OUTPUT" | grep "^VERSION=" | cut -d= -f2) + if [ -z "$VERSION" ]; then + echo "No English changelog to translate (or Russian already exists)." + exit 0 + fi + EN_FILE=$(echo "$TRANSLATE_OUTPUT" | grep "^EN_FILE=" | cut -d= -f2) + RU_FILE=$(echo "$TRANSLATE_OUTPUT" | grep "^RU_FILE=" | cut -d= -f2) + echo "Translated $EN_FILE -> $RU_FILE (version $VERSION)" + # Commit and push + git add "${CHANGELOG_PATH}/" + git status + if git diff --staged --quiet; then + echo "No staged changes." + exit 0 + fi + git commit -m "Translate changelog ${VERSION} to Russian" + BRANCH="${CI_COMMIT_REF_NAME:-${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}}" + if [ -z "$BRANCH" ]; then + echo "Cannot push: CI_COMMIT_REF_NAME and CI_MERGE_REQUEST_SOURCE_BRANCH_NAME are empty." + exit 1 + fi + git checkout -B "$BRANCH" + REPO_URL="https://oauth2:${RELEASE_TOKEN:-$CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" + git remote set-url origin "$REPO_URL" + git push origin "$BRANCH" + # Create MR if not exists + EXISTING_MR=$(curl -s --header "PRIVATE-TOKEN: ${RELEASE_TOKEN:-$CI_JOB_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests?source_branch=${CI_COMMIT_REF_NAME}&target_branch=${TRANSLATE_BASE_BRANCH}&state=opened" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['iid'] if d else '')" 2>/dev/null || echo "") + if [ -n "$EXISTING_MR" ]; then + echo "MR !${EXISTING_MR} already exists for ${CI_COMMIT_REF_NAME} -> ${TRANSLATE_BASE_BRANCH}" + exit 0 + fi + RU_PATH="${CI_PROJECT_DIR}/${CHANGELOG_PATH}/${RU_FILE}" + DESC_FILE="/tmp/mr_desc.md" + printf "## Changelog %s\n\nThis MR contains the Russian translation of the English changelog for version %s.\n\n**Source file:** \\\`%s/%s\\\`\n**Translated file:** \\\`%s/%s\\\`\n\n
\nChangelog content\n\n\\\`\\\`\\\`yaml\n%s\n\\\`\\\`\\\`\n\n
\n" \ + "$VERSION" "$VERSION" "$CHANGELOG_PATH" "$EN_FILE" "$CHANGELOG_PATH" "$RU_FILE" "$(cat "$RU_PATH")" > "$DESC_FILE" + jq -n \ + --arg source_branch "$CI_COMMIT_REF_NAME" \ + --arg target_branch "${TRANSLATE_BASE_BRANCH}" \ + --arg title "$VERSION" \ + --rawfile description "$DESC_FILE" \ + '{ source_branch: $source_branch, target_branch: $target_branch, title: $title, description: $description }' \ + | curl -s --request POST \ + --header "PRIVATE-TOKEN: ${RELEASE_TOKEN:-$CI_JOB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data @- \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print('Created MR !'+str(d.get('iid','')))" + echo "Translate changelog and create MR done." diff --git a/.gitlab/ci/scripts/bash/auto-assign-author.sh b/.gitlab/ci/scripts/bash/auto-assign-author.sh new file mode 100644 index 0000000000..d465065beb --- /dev/null +++ b/.gitlab/ci/scripts/bash/auto-assign-author.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +# Auto-assign MR author as the MR assignee. +# +# Migration of .github/workflows/dev_auto-pr-author-assign.yml which used the +# third-party toshimaru/auto-author-assign@v2.1.0 action. +# +# Behaviour: +# - Skip silently if MR already has at least one assignee. +# - Otherwise assign the MR author (the user who opened the MR). +# - Token: GITLAB_API_TOKEN (Project Access Token, scope api). +# +# Required environment: +# GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_MERGE_REQUEST_IID +# +# Exits non-zero only on unexpected API errors; "already assigned" is a no-op. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_MERGE_REQUEST_IID + +MR_PATH="/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" + +echo "Reading MR ${CI_MERGE_REQUEST_IID} to detect author and current assignees..." +mr_json="$(api GET "${MR_PATH}")" + +# Author user id (author_id is the numeric ID of the user who created the MR). +author_id="$(printf '%s' "$mr_json" | jq -r '.author.id // empty')" +if [[ -z "$author_id" || "$author_id" == "null" ]]; then + echo "ERROR: MR has no author_id (response had no .author.id)" >&2 + exit 1 +fi +author_name="$(printf '%s' "$mr_json" | jq -r '.author.name // .author.username // "unknown"')" +echo "MR author: ${author_name} (id=${author_id})" + +# Count current assignees (assignees[] is an array of user objects). +assignee_count="$(printf '%s' "$mr_json" | jq -r '.assignees | length')" +echo "Current assignee count: ${assignee_count}" + +if [[ "${assignee_count}" -gt 0 ]]; then + echo "MR already has ${assignee_count} assignee(s) — skipping auto-assign." + exit 0 +fi + +# Assign author by user_id. +echo "Assigning user_id=${author_id} as MR assignee..." +assignee_payload="$(jq -n --argjson uid "$author_id" '{assignee_ids: [$uid]}')" +api PUT "${MR_PATH}" --data "$assignee_payload" >/dev/null + +echo "Assigned author (${author_name}) to MR !${CI_MERGE_REQUEST_IID}." diff --git a/.gitlab/ci/scripts/bash/backport.sh b/.gitlab/ci/scripts/bash/backport.sh new file mode 100644 index 0000000000..a706ef8994 --- /dev/null +++ b/.gitlab/ci/scripts/bash/backport.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +# Backport a merged MR to a release branch by opening a backport MR. +# +# Migration of .github/workflows/on-pull-request-backport.yml which used +# deckhouse/backport-action@v1.0.0 and direct cherry-pick to release branch. +# +# We DO NOT use the GitLab cherry-pick +# REST endpoint (POST /repository/commits/:sha/cherry_pick) because it +# bypasses code review. Instead we: +# 1. clone the repo (or reuse the runner workspace), +# 2. cherry-pick the merged commit (or head SHA), +# 3. push a backport branch, +# 4. open an MR to the target release branch via push options / API. +# +# Target branch resolution (priority): +# 1. TARGET_BRANCH env var (manual "Run pipeline" override). +# 2. Source MR milestone title: vX.Y.Z or X.Y.Z -> release-X.Y. +# +# After the backport attempt, the source MR receives feedback matching the +# GitHub flow: +# - always remove `status/backport` (when source MR iid is known); +# - success: add `status/backport/success` + comment with backport MR link; +# - failure: add `status/backport/failed` + comment with error/job link. +# Feedback failures are logged but never mask the backport outcome on +# success; on the failure path they are best-effort and the script still +# exits non-zero. +# +# Required environment: +# GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_SERVER_HOST, +# CI_PROJECT_PATH, CI_PROJECT_DIR +# SOURCE_MR_IID (optional; defaults to CI_MERGE_REQUEST_IID) +# TARGET_BRANCH (optional; otherwise derived from the source MR milestone) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_SERVER_HOST CI_PROJECT_PATH CI_PROJECT_DIR + +# Status labels (with slashes; used as-is in JSON payloads, comma is the only +# separator GitLab uses for add_labels/remove_labels). +BACKPORT_TRIGGER_LABEL="status/backport" +BACKPORT_SUCCESS_LABEL="status/backport/success" +BACKPORT_FAILED_LABEL="status/backport/failed" + +# Resolved later; used by the failure-feedback trap. +SOURCE_MR_IID="${SOURCE_MR_IID:-}" +TARGET_BRANCH="${TARGET_BRANCH:-}" +BACKPORT_MR_IID="" +BACKPORT_MR_URL="" +FAILURE_CONTEXT="" +# Set to "success" only after the backport MR is created cleanly without conflicts. +BACKPORT_RESULT="" + +# --------------------------------------------------------------------------- +# Source MR feedback helpers (best-effort: never abort the script). +# --------------------------------------------------------------------------- + +# Remove a label from the source MR. GitLab PUT /merge_requests/:iid supports +# remove_labels with comma-separated names; a slash inside a label name is fine +# because the separator is the comma, not the slash. +mr_remove_label() { + local iid="$1" + local label="$2" + local payload + payload="$(jq -n --arg l "$label" '{remove_labels: $l}')" + if ! api PUT "/projects/${CI_PROJECT_ID}/merge_requests/${iid}" --data "${payload}" >/dev/null; then + echo "WARNING: failed to remove label '${label}' from MR !${iid}." >&2 + return 0 + fi + echo "Removed label '${label}' from MR !${iid}." +} + +# Add a label to the source MR. +mr_add_label() { + local iid="$1" + local label="$2" + local payload + payload="$(jq -n --arg l "$label" '{add_labels: $l}')" + if ! api PUT "/projects/${CI_PROJECT_ID}/merge_requests/${iid}" --data "${payload}" >/dev/null; then + echo "WARNING: failed to add label '${label}' to MR !${iid}." >&2 + return 0 + fi + echo "Added label '${label}' to MR !${iid}." +} + +# Create a note (comment) on the source MR. +mr_create_note() { + local iid="$1" + local body="$2" + local payload + payload="$(jq -n --arg b "$body" '{body: $b}')" + if ! api POST "/projects/${CI_PROJECT_ID}/merge_requests/${iid}/notes" --data "${payload}" >/dev/null; then + echo "WARNING: failed to create note on MR !${iid}." >&2 + return 0 + fi + echo "Created note on MR !${iid}." +} + +# Report failure feedback to the source MR (best-effort). +report_failure() { + if [[ -z "${SOURCE_MR_IID}" ]]; then + echo "No source MR iid known; cannot report failure feedback." >&2 + return 0 + fi + local job_url="${CI_JOB_URL:-${CI_PIPELINE_URL:-}}" + local body="Backport to \`${TARGET_BRANCH:-unknown}\` failed." + if [[ -n "$job_url" ]]; then + body="${body} See job: ${job_url}" + fi + if [[ -n "$FAILURE_CONTEXT" ]]; then + body="${body} + +${FAILURE_CONTEXT}" + fi + if [[ -n "$BACKPORT_MR_URL" ]]; then + body="${body} + +A backport MR was created for manual resolution: ${BACKPORT_MR_URL}" + fi + mr_remove_label "${SOURCE_MR_IID}" "${BACKPORT_TRIGGER_LABEL}" + mr_add_label "${SOURCE_MR_IID}" "${BACKPORT_FAILED_LABEL}" + mr_create_note "${SOURCE_MR_IID}" "${body}" +} + +# Report success feedback to the source MR (best-effort). +report_success() { + if [[ -z "${SOURCE_MR_IID}" ]]; then + echo "No source MR iid known; cannot report success feedback." >&2 + return 0 + fi + local body="Backport to \`${TARGET_BRANCH}\` successful: ${BACKPORT_MR_URL:-!(unknown)}" + mr_remove_label "${SOURCE_MR_IID}" "${BACKPORT_TRIGGER_LABEL}" + mr_add_label "${SOURCE_MR_IID}" "${BACKPORT_SUCCESS_LABEL}" + mr_create_note "${SOURCE_MR_IID}" "${body}" +} + +# EXIT trap: route to success/failure feedback. Preserves the exit code on +# the failure path; feedback errors never override the backport outcome. +on_exit() { + local rc=$? + # Clean exit (success) — nothing more to do. + if [[ $rc -eq 0 || "${BACKPORT_RESULT}" == "success" ]]; then + exit 0 + fi + set +e + echo "Backport failed (exit code ${rc}); reporting failure feedback to source MR." >&2 + report_failure + exit "$rc" +} +trap on_exit EXIT + +# --------------------------------------------------------------------------- +# Resolve the source MR iid. +# --------------------------------------------------------------------------- +SOURCE_MR_IID="${SOURCE_MR_IID:-${CI_MERGE_REQUEST_IID:-}}" +if [[ -z "$SOURCE_MR_IID" ]]; then + echo "ERROR: SOURCE_MR_IID is required (CI_MERGE_REQUEST_IID unset and no explicit var)." >&2 + exit 1 +fi +echo "Source MR: !${SOURCE_MR_IID}" + +# --------------------------------------------------------------------------- +# Read source MR (merged commit SHA + milestone). +# --------------------------------------------------------------------------- +mr_path="/projects/${CI_PROJECT_ID}/merge_requests/${SOURCE_MR_IID}" +mr_json="$(api GET "${mr_path}")" + +sha="$(printf '%s' "$mr_json" | jq -r '.merge_commit_sha // .sha // empty')" +if [[ -z "$sha" || "$sha" == "null" ]]; then + echo "ERROR: could not extract SHA from MR !${SOURCE_MR_IID} (not merged yet?)." >&2 + FAILURE_CONTEXT="could not extract merged commit SHA from MR !${SOURCE_MR_IID}" + exit 1 +fi +echo "Source commit SHA: ${sha}" + +milestone_title="$(printf '%s' "$mr_json" | jq -r '.milestone.title // empty')" + +# --------------------------------------------------------------------------- +# Resolve TARGET_BRANCH (priority: explicit var > milestone-based inference). +# --------------------------------------------------------------------------- +TARGET_BRANCH="${TARGET_BRANCH:-}" +if [[ -z "$TARGET_BRANCH" ]]; then + if [[ -z "$milestone_title" || "$milestone_title" == "null" ]]; then + echo "ERROR: source MR !${SOURCE_MR_IID} has no milestone; cannot derive target branch." >&2 + echo " Set TARGET_BRANCH or assign a milestone to the source MR." >&2 + FAILURE_CONTEXT="source MR !${SOURCE_MR_IID} has no milestone" + exit 1 + fi + # Match vX.Y.Z or X.Y.Z and keep the X.Y minor (mirrors the GitHub workflow). + if [[ "$milestone_title" =~ v?([0-9]+\.[0-9]+)\.[0-9]+ ]]; then + TARGET_BRANCH="release-${BASH_REMATCH[1]}" + else + echo "ERROR: milestone '${milestone_title}' does not match v?X.Y.Z format." >&2 + FAILURE_CONTEXT="invalid milestone format: ${milestone_title}" + exit 1 + fi +fi + +if ! [[ "$TARGET_BRANCH" =~ ^release-[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: TARGET_BRANCH='${TARGET_BRANCH}' does not match ^release-[0-9]+\\.[0-9]+\$" >&2 + FAILURE_CONTEXT="invalid TARGET_BRANCH format: ${TARGET_BRANCH}" + exit 1 +fi + +echo "Backport target: ${TARGET_BRANCH}" +if [[ -n "$milestone_title" && "$milestone_title" != "null" ]]; then + echo "Source MR milestone: ${milestone_title}" +fi + +# --------------------------------------------------------------------------- +# Verify target branch exists (clearer error than a git fetch failure). +# --------------------------------------------------------------------------- +if ! api GET "/projects/${CI_PROJECT_ID}/repository/branches/${TARGET_BRANCH}" >/dev/null; then + echo "ERROR: target branch '${TARGET_BRANCH}' does not exist in this project." >&2 + FAILURE_CONTEXT="target branch '${TARGET_BRANCH}' does not exist" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Cherry-pick the merged commit onto the target branch. +# --------------------------------------------------------------------------- +cd "${CI_PROJECT_DIR}" + +git config user.email "ci-backport@flant.com" +git config user.name "GitLab CI Backport Bot" + +REPO_URL="https://oauth2:${GITLAB_API_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" +git remote set-url origin "${REPO_URL}" + +# Fetch the target branch. +git fetch --no-tags --depth=200 origin "${TARGET_BRANCH}" + +BACKPORT_BRANCH="backport/${SOURCE_MR_IID}/${TARGET_BRANCH}" +echo "Creating backport branch: ${BACKPORT_BRANCH}" +git checkout -B "${BACKPORT_BRANCH}" "origin/${TARGET_BRANCH}" + +CONFLICT_MARKER="" +CONFLICTED=0 +# GIT_SEQUENCE_EDITOR=true drops the default commit message editor so the +# script can run unattended. +GIT_SEQUENCE_EDITOR=true git cherry-pick -x "${sha}" || { + CONFLICTED=1 + CONFLICT_MARKER=" +**Conflicts detected** — please resolve manually, amend the commit, and force-push. +" + echo "Cherry-pick reported conflicts; staging resolved tree as best-effort commit." + # Best-effort: keep partial progress so reviewer can see what was applied. + git add -A || true + if ! git diff --cached --quiet; then + git -c core.editor=true commit --no-edit || true + fi +} + +# --------------------------------------------------------------------------- +# Push the backport branch and ensure a backport MR exists. +# --------------------------------------------------------------------------- +DESCRIPTION="Backport !${SOURCE_MR_IID} (${sha}) to ${TARGET_BRANCH}. + +Auto-generated by GitLab CI backport job. +${CONFLICT_MARKER}" + +# Push with merge_request.* push options to create the MR in one step. +push_output="$(git push --force-with-lease \ + -o merge_request.create \ + -o merge_request.target="${TARGET_BRANCH}" \ + -o merge_request.source="${BACKPORT_BRANCH}" \ + -o merge_request.title="Backport !${SOURCE_MR_IID} to ${TARGET_BRANCH}" \ + -o merge_request.description="${DESCRIPTION}" \ + -o merge_request.label="backport" \ + -o merge_request.label="auto" \ + -o merge_request.remove_source_branch \ + origin "${BACKPORT_BRANCH}" 2>&1 || true)" +echo "${push_output}" + +# Fallback (push options are ignored on some GitLab versions): search for an +# open MR from our branch; if absent, create it via API. +backport_mr_json="" +existing_mr_iid="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests?source_branch=${BACKPORT_BRANCH}&state=opened" \ + | jq -r 'if type == "array" and length > 0 then .[0].iid else empty end')" + +if [[ -n "$existing_mr_iid" ]]; then + echo "Backport MR already exists: !${existing_mr_iid}" + backport_mr_json="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests/${existing_mr_iid}")" +else + echo "Push options did not create MR; creating via API..." + payload="$(jq -n \ + --arg src "${BACKPORT_BRANCH}" \ + --arg tgt "${TARGET_BRANCH}" \ + --arg title "Backport !${SOURCE_MR_IID} to ${TARGET_BRANCH}" \ + --arg desc "${DESCRIPTION}" \ + '{source_branch: $src, target_branch: $tgt, title: $title, description: $desc, remove_source_branch: true, labels: "backport,auto"}')" + backport_mr_json="$(api POST "/projects/${CI_PROJECT_ID}/merge_requests" --data "${payload}")" +fi + +BACKPORT_MR_IID="$(printf '%s' "$backport_mr_json" | jq -r '.iid // empty')" +BACKPORT_MR_URL="$(printf '%s' "$backport_mr_json" | jq -r '.web_url // empty')" +if [[ -z "$BACKPORT_MR_IID" || "$BACKPORT_MR_IID" == "null" ]]; then + echo "ERROR: could not determine backport MR iid." >&2 + FAILURE_CONTEXT="backport MR creation did not return an iid" + exit 1 +fi +echo "Backport MR: !${BACKPORT_MR_IID} (${BACKPORT_MR_URL})" + +# --------------------------------------------------------------------------- +# Report outcome. +# --------------------------------------------------------------------------- +if [[ "$CONFLICTED" == "1" ]]; then + # A backport MR exists for manual resolution, but the cherry-pick had + # conflicts, so this is treated as a failure (matches the GitHub flow). + FAILURE_CONTEXT="cherry-pick had conflicts; resolve them in the backport MR." + exit 1 +fi + +BACKPORT_RESULT="success" +# Feedback failures must not mask the successful backport. +set +e +report_success +set -e +echo "Backport completed successfully." diff --git a/.gitlab/ci/scripts/bash/changelog-milestone.sh b/.gitlab/ci/scripts/bash/changelog-milestone.sh new file mode 100644 index 0000000000..9ddc069689 --- /dev/null +++ b/.gitlab/ci/scripts/bash/changelog-milestone.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Thin wrapper around changelog_collect.py to keep the job yml language-agnostic. +# +# Selects python3 / python at runtime. The actual logic lives in Python +# (Variant B). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python +else + echo "ERROR: neither python3 nor python is installed on the runner." >&2 + exit 1 +fi + +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/../python/changelog_collect.py" diff --git a/.gitlab/ci/scripts/bash/check-changelog-entry.sh b/.gitlab/ci/scripts/bash/check-changelog-entry.sh new file mode 100644 index 0000000000..46aabfb3b4 --- /dev/null +++ b/.gitlab/ci/scripts/bash/check-changelog-entry.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Thin bash wrapper around check_changelog_entry.py. +# +# Allows the job yml to call a single bash script while keeping the +# actual validation logic in Python. +# +# Picks the first available interpreter: python3, python. +# Required environment is documented in check_changelog_entry.py. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python +else + echo "ERROR: neither python3 nor python is installed on the runner." >&2 + exit 1 +fi + +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/../python/check_changelog_entry.py" diff --git a/.gitlab/ci/scripts/bash/check-milestone.sh b/.gitlab/ci/scripts/bash/check-milestone.sh new file mode 100644 index 0000000000..c7fca25b8a --- /dev/null +++ b/.gitlab/ci/scripts/bash/check-milestone.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +# Check that the current MR has a milestone assigned. +# +# Migration of .github/workflows/check-pr-milestone.yml which used +# actions/github-script@v6.4.1 to GET the PR and assert data.milestone. +# +# Behaviour: +# - On MR pipelines: GET MR via API, ensure milestone is present. +# - On other pipelines: no-op (print "skipping"). +# - Skip-labels respected (see rules in job yml). +# +# Required environment: +# GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_MERGE_REQUEST_IID + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +if [[ "${CI_PIPELINE_SOURCE:-}" != "merge_request_event" ]]; then + echo "Not a merge request pipeline (CI_PIPELINE_SOURCE=${CI_PIPELINE_SOURCE:-}). Skipping." + exit 0 +fi + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_MERGE_REQUEST_IID + +MR_PATH="/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" + +echo "Reading MR ${CI_MERGE_REQUEST_IID} to check milestone..." +mr_json="$(api GET "${MR_PATH}")" + +milestone_title="$(printf '%s' "$mr_json" | jq -r '.milestone.title // empty')" +milestone_id="$(printf '%s' "$mr_json" | jq -r '.milestone.id // empty')" + +if [[ -n "$milestone_title" && "$milestone_title" != "null" ]]; then + echo "OK: MR has milestone '${milestone_title}' (id=${milestone_id})." + exit 0 +fi + +echo "ERROR: MR !${CI_MERGE_REQUEST_IID} has no milestone set. Set a milestone before merge." >&2 +exit 1 diff --git a/.gitlab/ci/scripts/bash/check-runner-tools.sh b/.gitlab/ci/scripts/bash/check-runner-tools.sh new file mode 100644 index 0000000000..2b5d2c0c70 --- /dev/null +++ b/.gitlab/ci/scripts/bash/check-runner-tools.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ "$#" -eq 0 ]]; then + echo "Usage: $0 [...]" >&2 + exit 2 +fi + +missing=() +for tool in "$@"; do + if ! command -v "$tool" >/dev/null 2>&1; then + missing+=("$tool") + fi +done + +if [[ "${#missing[@]}" -gt 0 ]]; then + echo "ERROR: required runner tool(s) are missing: ${missing[*]}" >&2 + echo "This pipeline is designed for GitLab Runner shell executor." >&2 + echo "Container images and package-manager installs are not used by these jobs; install the tools on the runner host." >&2 + exit 1 +fi + +printf 'Runner tools OK:' +printf ' %s' "$@" +printf '\n' diff --git a/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh new file mode 100755 index 0000000000..47fc563a79 --- /dev/null +++ b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# .gitlab/ci/scripts/bash/gitlab-ci-lint.sh +# +# Calls the GitLab CI Lint API +# POST ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint +# with a `content` payload assembled from the project files that make +# up the effective CI configuration. +# +# A single-document lint is what GitLab's lint API supports per request. We therefore lint the root +# `.gitlab-ci.yml` directly. The upstream project owner is responsible +# for keeping `.gitlab/ci/includes.yml` and the `local:` job files +# self-consistent; this script only checks that the *merged* file the +# runner sees parses cleanly. +# +# Authentication: PRIVATE-TOKEN via GITLAB_API_TOKEN (Project Access +# Token, scope api). Falls back to CI_JOB_TOKEN for read-only pipeline +# scope when GITLAB_API_TOKEN is unset (matches the convention in +# .gitlab/ci/scripts/bash/lib/api.sh). +# +# Exit codes: +# 0 - CI config is valid. +# 1 - CI config is invalid, or the API call failed. +# 2 - missing tools/inputs (curl/jq/CI_* env). +# +# Required env: +# CI_API_V4_URL - GitLab API v4 base URL (set automatically inside CI jobs). +# CI_PROJECT_ID - Project ID (set automatically inside CI jobs). +# Optional env: +# GITLAB_API_TOKEN / CI_JOB_TOKEN +# LINT_TARGETS - newline-separated list of paths to lint. +# Defaults to ".gitlab-ci.yml". + +set -euo pipefail + +# --- helpers ----------------------------------------------------------------- + +log() { printf '[gitlab-ci-lint] %s\n' "$*"; } +fail() { printf '[gitlab-ci-lint] ERROR: %s\n' "$*" >&2; exit 1; } + +require() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd is required but not installed" +} + +require curl +require jq + +# --- input assembly --------------------------------------------------------- + +# We always lint .gitlab-ci.yml. The gitlab.com CI lint API accepts a +# single `content` string per request, so callers needing multi-file +# validation must run this script per file. For the rules:changes use +# case in lint-validate.yml, .gitlab-ci.yml is the entrypoint that +# pulls everything in via `include:` — GitLab evaluates the merged +# config server-side when the pipeline actually starts. +TARGETS=() +if [[ -n "${LINT_TARGETS:-}" ]]; then + while IFS= read -r line; do + [[ -n "$line" ]] && TARGETS+=("$line") + done <<< "${LINT_TARGETS}" +else + TARGETS+=(".gitlab-ci.yml") +fi + +# --- auth -------------------------------------------------------------------- + +auth_header=() +if [[ -n "${GITLAB_API_TOKEN:-}" ]]; then + auth_header=(--header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}") +elif [[ -n "${CI_JOB_TOKEN:-}" ]]; then + auth_header=(--header "JOB-TOKEN: ${CI_JOB_TOKEN}") +else + fail "Neither GITLAB_API_TOKEN nor CI_JOB_TOKEN is set; cannot call CI lint API." +fi + +[[ -n "${CI_API_V4_URL:-}" ]] || fail "CI_API_V4_URL is not set" +[[ -n "${CI_PROJECT_ID:-}" ]] || fail "CI_PROJECT_ID is not set" + +# --- per-target lint -------------------------------------------------------- + +overall_rc=0 +for target in "${TARGETS[@]}"; do + if [[ ! -f "${target}" ]]; then + log "skip ${target} (file not present in checkout)" + continue + fi + + log "linting ${target} via ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint" + + # Build the JSON payload with `jq` so we don't have to worry about + # escaping the YAML content. --rawfile reads the file verbatim and + # embeds it as a JSON string. + payload="$(jq -nc --rawfile content "${target}" '{content: $content}')" + + tmp_body="$(mktemp)" + http_status="$( + curl --silent --show-error \ + --request POST \ + "${auth_header[@]}" \ + --header 'Content-Type: application/json' \ + --data "${payload}" \ + --output "${tmp_body}" \ + --write-out '%{http_code}' \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint" + )" + + # Pretty-print whatever the API returned so failures are easy to read + # in the CI log. + if [[ -s "${tmp_body}" ]]; then + jq . "${tmp_body}" || cat "${tmp_body}" + else + log "lint API returned an empty body" + fi + + rm -f "${tmp_body}" + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + log "FAIL ${target} (HTTP ${http_status})" + overall_rc=1 + continue + fi + + # When jq is available we already validated JSON above; trust the HTTP + # 2xx status. The API also returns `valid: true|false` and a list of + # `errors`, but a single .gitlab-ci.yml may pass via the server-side + # include resolver even when our `content` slice would not (because + # server-side includes may not be available without a JWT). Surface + # the verdict in the log anyway. + log "OK ${target} (HTTP ${http_status})" +done + +if [[ "${overall_rc}" -ne 0 ]]; then + fail "one or more CI lint targets failed" +fi + +log "all targets passed" +exit 0 \ No newline at end of file diff --git a/.gitlab/ci/scripts/bash/lib/api.sh b/.gitlab/ci/scripts/bash/lib/api.sh new file mode 100755 index 0000000000..8c0ec1c0da --- /dev/null +++ b/.gitlab/ci/scripts/bash/lib/api.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +# Shared GitLab API helpers for migration-era jobs. +# +# Source from a job's script: +# source .gitlab/ci/scripts/bash/lib/api.sh +# +# Provides: +# api METHOD PATH [curl-args...] -- REST call with PRIVATE-TOKEN, prints body, returns exit code. +# gl_required_env -- fails if required env vars are missing. +# gl_log_call -- echoes request line for log readability. +# +# Conventions: +# - Always CI_API_V4_URL (never hardcode the host). +# - Always GITLAB_API_TOKEN (Project Access Token, scope api). +# - Always CI_PROJECT_ID (numeric) and CI_MERGE_REQUEST_IID (iid, not id). + +# Guard against double-sourcing. +if [[ -n "${__GL_API_SH_SOURCED:-}" ]]; then + return 0 +fi +__GL_API_SH_SOURCED=1 + +set -euo pipefail + +gl_required_env() { + local missing=() + local v + for v in "$@"; do + if [[ -z "${!v:-}" ]]; then + missing+=("$v") + fi + done + if [[ "${#missing[@]}" -gt 0 ]]; then + echo "ERROR: required environment variables are not set: ${missing[*]}" >&2 + exit 1 + fi +} + +gl_log_call() { + local method="$1" + local path="$2" + echo ">>> ${method} ${CI_API_V4_URL}${path}" >&2 +} + +# api METHOD PATH [extra curl args] +# +# Examples: +# api GET "/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" +# api POST "/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/assignees" \ +# --data '{"user_id":42}' +# +# Behaviour: +# - Uses PRIVATE-TOKEN with $GITLAB_API_TOKEN. +# - Sets Content-Type: application/json by default (overridable via extra args). +# - On non-2xx: prints status and body to stderr, returns non-zero. +api() { + local method="$1" + shift + local path="$1" + shift + + gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID + gl_log_call "$method" "$path" + + local response_file + response_file="$(mktemp)" + local http_code + http_code="$(curl --silent --show-error --output "$response_file" \ + --request "$method" \ + --write-out '%{http_code}' \ + --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" \ + --header "Content-Type: application/json" \ + --header "Accept: application/json" \ + "${CI_API_V4_URL}${path}" "$@")" + + if [[ "$http_code" =~ ^2 ]]; then + cat "$response_file" + rm -f "$response_file" + echo "<<< status=${http_code}" >&2 + return 0 + fi + + cat "$response_file" >&2 + rm -f "$response_file" + echo "<<< status=${http_code} (FAILED)" >&2 + return 1 +} diff --git a/.gitlab/ci/scripts/bash/notify-release-loop.sh b/.gitlab/ci/scripts/bash/notify-release-loop.sh new file mode 100755 index 0000000000..c8b230119a --- /dev/null +++ b/.gitlab/ci/scripts/bash/notify-release-loop.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Post a markdown release-status summary to the Loop chat webhook. +# +# Port of the GitHub Actions job `send-release-results-to-loop` +# (.github/workflows/release_module_release-channels.yml). GitLab does not +# inject needs.*.result into the job environment, so this script queries the +# pipeline jobs API once to collect each edition/check/release job status and +# builds the same status table the GH job produced. +# +# Inputs (GitLab predefined + dotenv artifact): +# CI_API_V4_URL, CI_PROJECT_ID, CI_PIPELINE_ID, CI_PIPELINE_URL, +# GITLAB_API_TOKEN, RELEASE_TAG, RELEASE_CHANNEL, EDITION_CE, EDITION_EE, +# CHECK_ONLY, RELEASE_TO_GITLAB, SEND_RESULTS_TO_LOOP, LOOP_WEBHOOK_URL, +# release.env (GH_RELEASE_STATUS, GH_RELEASE_URL from prod:create-gitlab-release). + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +set -euo pipefail + +source .gitlab/ci/scripts/bash/lib/api.sh + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_PIPELINE_ID \ + RELEASE_TAG RELEASE_CHANNEL LOOP_WEBHOOK_URL + +# Fetch this pipeline's jobs once. +JOBS_JSON=$(api GET "/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100") + +job_status() { + # $1 = job name. Echo the GitLab status of the first matching job, or "". + echo "$JOBS_JSON" | jq -r --arg n "$1" '[.[] | select(.name == $n)][0].status // ""' +} + +# Map a GitLab job status to a GitHub-style result word for emoji selection. +map_result() { + case "$1" in + success) echo "success" ;; + failed) echo "failure" ;; + canceled|cancelled) echo "cancelled" ;; + skipped|manual) echo "skipped" ;; + running|pending|created|waiting_for_resource|preparing) echo "running" ;; + *) echo "unknown" ;; + esac +} + +status_emoji() { + case "$1" in + success) echo ":white_check_mark:" ;; + failure) echo ":x:" ;; + cancelled) echo ":warning:" ;; + skipped) echo ":fast_forward:" ;; + running) echo ":hourglass:" ;; + *) echo ":grey_question:" ;; + esac +} + +export TZ="Europe/Moscow" +DATE=$(date +"%Y-%m-%d %H:%M:%S UTC+03:00") +RUN_URL="${CI_PIPELINE_URL}" + +CE_RESULT=$(map_result "$(job_status prod:deploy:ce)") +EE_RESULT=$(map_result "$(job_status prod:deploy:ee)") +SE_PLUS_RESULT=$(map_result "$(job_status prod:deploy:se-plus)") +FE_RESULT=$(map_result "$(job_status prod:deploy:fe)") +CHECK_RESULT=$(map_result "$(job_status prod:check-version)") + +# Load the release creation dotenv artifact if present. +GH_RELEASE_STATUS="" +# shellcheck disable=SC1091 +[ -f release.env ] && . release.env + +HEADER_ROW="| Edition |" +STATUS_ROW="| Status |" +if [ "${EDITION_CE:-false}" = "true" ]; then + HEADER_ROW+=" CE |" + STATUS_ROW+=" $(status_emoji "${CE_RESULT}") |" +fi +if [ "${EDITION_EE:-false}" = "true" ]; then + HEADER_ROW+=" EE | SE+ | FE |" + STATUS_ROW+=" $(status_emoji "${EE_RESULT}") | $(status_emoji "${SE_PLUS_RESULT}") | $(status_emoji "${FE_RESULT}") |" +fi +HEADER_ROW+=" Check |" +STATUS_ROW+=" $(status_emoji "${CHECK_RESULT}") |" +if [ "${RELEASE_TO_GITLAB:-true}" = "true" ] && [ "${CHECK_ONLY:-false}" != "true" ]; then + HEADER_ROW+=" GitLab Release |" + case "${GH_RELEASE_STATUS}" in + created) STATUS_ROW+=" :white_check_mark: |" ;; + skipped) STATUS_ROW+=" :fast_forward: |" ;; + *) STATUS_ROW+=" :x: |" ;; + esac +fi + +# Build the markdown separator row matching the header column count. +COL_COUNT=$(echo "${HEADER_ROW}" | tr -cd '|' | wc -c) +COL_COUNT=$((COL_COUNT - 1)) +SEP="|" +i=0 +while [ "${i}" -lt "${COL_COUNT}" ]; do + SEP+="---|" + i=$((i + 1)) +done + +SUMMARY="## :dvp: **DVP | Release ${RELEASE_TAG} to ${RELEASE_CHANNEL}**\\n\\n" +SUMMARY+="Date: ${DATE}\\n" +SUMMARY+="[:link: GitLab CI Pipeline](${RUN_URL})\\n\\n" +SUMMARY+="${HEADER_ROW}\\n${SEP}\\n${STATUS_ROW}\\n" + +echo -e "${SUMMARY}" + +curl --silent --show-error --fail --request POST \ + --header "Content-Type: application/json" \ + --data "{\"text\": \"${SUMMARY}\"}" \ + "${LOOP_WEBHOOK_URL}" diff --git a/.gitlab/ci/scripts/bash/set-vars.sh b/.gitlab/ci/scripts/bash/set-vars.sh new file mode 100755 index 0000000000..141f4fbd8a --- /dev/null +++ b/.gitlab/ci/scripts/bash/set-vars.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# set-vars.sh — derives per-pipeline variables for downstream jobs. +# +# Carries forward the responsibilities of the GH `set_vars` job from +# dev_module_build.yml. Produces a dotenv +# artifact that downstream jobs consume via `needs: [set_vars]` + +# `artifacts.reports.dotenv`. +# +# Outputs (written to set_vars.env in $CI_PROJECT_DIR): +# MODULE_EDITION CE if MR carries label edition/ce, otherwise EE. +# RELEASE_IN_DEV true if $CI_COMMIT_BRANCH matches release-X.Y, +# false otherwise. +# DEBUG_COMPONENT first delve/* label (empty if none). +# +# MODULES_MODULE_TAG is intentionally NOT emitted here. The dev build jobs +# (build_dev / build_dev_tags / build_main) derive their own tag from the +# .dev / .dev_tags / .main templates, and a dotenv variable would override +# those per-template tags (see the `needs:` note in jobs/build-dev.yml). +# +# Required env (provided by the job context): +# CI_API_V4_URL, CI_PROJECT_ID, CI_PIPELINE_SOURCE, CI_COMMIT_BRANCH, +# CI_MERGE_REQUEST_IID, CI_MERGE_REQUEST_LABELS, GITLAB_API_TOKEN. +# +# GITLAB_API_TOKEN is only required for the manual PR_NUMBER fallback below; +# normal MR pipelines use $CI_MERGE_REQUEST_LABELS and need no token. +# +# Wired into the `set_vars` job in .gitlab/ci/jobs/info.yml, which runs in +# the info stage on MR, default-branch, and dev-tag pipelines and is +# consumed by the dev build jobs via +# `needs: [{job: set_vars, artifacts: true}]`. + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +set -euo pipefail + +# Source the api() helper for the GitLab API call below. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +OUTPUT="${CI_PROJECT_DIR:-.}/set_vars.env" + +# 1) RELEASE_IN_DEV ---------------------------------------------------------- +if [[ "${CI_COMMIT_BRANCH:-}" =~ ^release-[0-9]+\.[0-9]+ ]]; then + RELEASE_IN_DEV="true" +else + RELEASE_IN_DEV="false" +fi + +# 2) Labels via GitLab API --------------------------------------------------- +# GitLab exposes $CI_MERGE_REQUEST_LABELS automatically for MR pipelines, but +# we keep the explicit API fetch as a safety net for manual/web pipelines that +# target a specific MR via PR_NUMBER. +LABELS="" +if [[ -n "${CI_MERGE_REQUEST_LABELS:-}" ]]; then + LABELS="${CI_MERGE_REQUEST_LABELS}" +elif [[ -n "${PR_NUMBER:-}" && -n "${GITLAB_API_TOKEN:-}" ]]; then + LABELS="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests/${PR_NUMBER}" \ + | jq -r '.labels | join(",")')" +fi + +# 3) MODULE_EDITION ---------------------------------------------------------- +if [[ ",${LABELS}," == *,edition/ce,* ]]; then + MODULE_EDITION="CE" +else + MODULE_EDITION="EE" +fi + +# 4) DEBUG_COMPONENT --------------------------------------------------------- +DEBUG_COMPONENT="" +DELVE_COUNT=0 +if [[ -n "${LABELS}" ]]; then + DEBUG_COMPONENT="$(echo "${LABELS}" | tr ',' '\n' | grep '^delve' | head -n1 || true)" + DELVE_COUNT="$(echo "${LABELS}" | tr ',' '\n' | grep -c '^delve' || true)" +fi +if [[ "${DELVE_COUNT}" -gt 1 ]]; then + echo "ERROR: multiple delve labels: ${LABELS}" >&2 + exit 1 +fi + +# 5) Persist ----------------------------------------------------------------- +cat > "${OUTPUT}" <>> wrote ${OUTPUT}" +cat "${OUTPUT}" diff --git a/.gitlab/ci/scripts/bash/setup-mr-settings.sh b/.gitlab/ci/scripts/bash/setup-mr-settings.sh new file mode 100644 index 0000000000..06d6072448 --- /dev/null +++ b/.gitlab/ci/scripts/bash/setup-mr-settings.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# One-off: configure project-level MR settings for deckhouse/virtualization +# via GitLab REST API. +# +# Equivalent of running the script in CI is unnecessary; run this locally +# after creating the project (or once per repo). Wraps idempotent PUT +# requests so it's safe to re-run. +# +# Required environment: +# GITLAB_API_TOKEN - Personal Access Token with api scope (NOT a job token; +# job tokens cannot modify project settings). +# CI_PROJECT_ID - numeric project id (or pass --project-id on CLI). +# +# Optional CLI flags: +# --project-id override CI_PROJECT_ID +# --api-base default $CI_API_V4_URL or https://fox.flant.com/api/v4 +# --dry-run print curl commands instead of executing them + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +PROJECT_ID="${CI_PROJECT_ID:-}" +API_BASE="${CI_API_V4_URL:-https://fox.flant.com/api/v4}" +DRY_RUN="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --project-id) + PROJECT_ID="$2" + shift 2 + ;; + --api-base) + API_BASE="$2" + shift 2 + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -h|--help) + cat <] [--api-base ] [--dry-run] + +Applies the following project MR settings (idempotent PUT requests): + - merge_method = "merge" + - squash = true + - remove_source_branch = true + - only_allow_merge_if_pipeline_succeeds = true + - only_allow_merge_if_all_discussions_are_resolved = true + - allow_merge_on_skipped_pipeline = true + - resolve_outdated_diff_discussions = true + - printing_merge_request_link_enabled = true + - merge_requests_template = "" (configure later via UI if needed) + +Required: + GITLAB_API_TOKEN (api scope) +EOF + exit 0 + ;; + *) + echo "ERROR: unknown flag '$1'" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$PROJECT_ID" ]]; then + echo "ERROR: project id not provided (set CI_PROJECT_ID or pass --project-id)." >&2 + exit 1 +fi + +if [[ -z "${GITLAB_API_TOKEN:-}" ]]; then + echo "ERROR: GITLAB_API_TOKEN is not set." >&2 + exit 1 +fi + +# settings PUT body. Most fields accept the new value as-is. +SETTINGS_BODY=$(cat <<'EOF' +{ + "merge_method": "merge", + "squash_option": "always", + "remove_source_branch_after_merge": true, + "only_allow_merge_if_pipeline_succeeds": true, + "only_allow_merge_if_all_discussions_are_resolved": true, + "allow_merge_on_skipped_pipeline": true, + "resolve_outdated_diff_discussions": true, + "printing_merge_request_link_enabled": true, + "merge_requests_template": "" +} +EOF +) + +SETTINGS_PATH="/projects/${PROJECT_ID}" + +curl_args=( + curl --silent --show-error --request PUT + --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" + --header "Content-Type: application/json" + --data "${SETTINGS_BODY}" + "${API_BASE}${SETTINGS_PATH}" +) + +echo "Applying project MR settings to project_id=${PROJECT_ID}..." + +# Push the settings payload via PUT. +if [[ "$DRY_RUN" == "true" ]]; then + printf 'DRY-RUN:' + printf ' %q' "${curl_args[@]}" + printf '\n' +else + "${curl_args[@]}" \ + | jq '{ + id, name, path_with_namespace, + merge_method, squash_option, + remove_source_branch_after_merge, + only_allow_merge_if_pipeline_succeeds, + only_allow_merge_if_all_discussions_are_resolved, + allow_merge_on_skipped_pipeline, + resolve_outdated_diff_discussions, + printing_merge_request_link_enabled + }' +fi + +# Approvers: leave empty unless team decides on a default approver group. +# Push rules: file_size_limit and other settings are configured via UI +# (or push_rules PUT); intentionally not modified here to avoid surprise. +# PUT /projects/:id/push_rule +# body: { "deny_delete_tag": true, "member_check": true, ... } + +echo "Done. Verify in the GitLab UI Settings -> Merge requests." diff --git a/.gitlab/ci/scripts/python/changelog_collect.py b/.gitlab/ci/scripts/python/changelog_collect.py new file mode 100644 index 0000000000..c7586563af --- /dev/null +++ b/.gitlab/ci/scripts/python/changelog_collect.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Re-generate CHANGELOG/.yml and CHANGELOG/.md from merged MRs. + +Migration of .github/actions/milestone-changelog/action.yml (composite action) +which used deckhouse/changelog-action@v2.6.0. + +Strategy: rewrite the parser in Python (Variant B). + +Behaviour: + 1. Resolve target milestone from MILESTONE_TITLE or list open milestones. + 2. Fetch all merged MRs with that milestone (paginated) via GitLab API. + 3. Parse ```changes fenced blocks from each MR description. + 4. Group entries by `section` and `impact_level`. + 5. Emit: + CHANGELOG/CHANGELOG-.yml + CHANGELOG/CHANGELOG-.md + 6. Open a changelog MR to the base branch (CHANGELOG_FROM_MR=true) via push + options and CI_JOB_TOKEN (no separate API call). + +Required environment: + GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_SERVER_HOST, + CI_PROJECT_PATH, CI_PROJECT_DIR + +Optional: + MILESTONE_TITLE - generate for a specific milestone + OPEN_CHANGELOG_MR - "true" to push branch + open MR (default false) + CHANGELOG_BASE_BRANCH - default "main" + CHANGELOG_SECTIONS_FILE - default .gitlab/ci/changelog-sections.txt +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from pathlib import Path + + +CHANGES_BLOCK_RE = re.compile( + r"```changes\s*\n(.*?)\n```", + re.DOTALL, +) +KEY_VALUE_RE = re.compile(r"^([A-Za-z_]+)\s*:\s*(.*)$") +# Only these keys start a new field in a ```changes block. Any other line is +# treated as a continuation of the current field, so multi-line values (most +# importantly `impact:`, the high-impact migration note) are preserved instead +# of being dropped. Mirrors the deckhouse/changelog-action block schema. +KNOWN_BLOCK_KEYS = {"section", "type", "summary", "impact", "impact_level"} +# deckhouse/changelog-action@v2.6.0 only renders 'feature' (-> features) and 'fix' +# (-> fixes) sections in CHANGELOG-*.yml. Keep in sync with +# check_changelog_entry.py. +ALLOWED_TYPES = {"feature", "fix"} +TYPE_TO_SECTION = { + "feature": "features", + "fix": "fixes", +} + + +def log(message: str) -> None: + print(message, file=sys.stderr) + + +def require_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + log(f"ERROR: required environment variable {name} is not set") + sys.exit(1) + return value + + +def api_get_paginated( + api_base: str, path: str, token: str, params: dict[str, str] | None = None +) -> list[dict]: + """GET all pages of a list endpoint, return combined JSON array.""" + results: list[dict] = [] + url = f"{api_base}{path}" + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + while url: + req = urllib.request.Request( + url, + headers={"PRIVATE-TOKEN": token, "Accept": "application/json"}, + method="GET", + ) + with urllib.request.urlopen(req) as response: + # GitLab returns Link header for next page (RFC 5988). + link_header = response.headers.get("Link", "") + payload = json.loads(response.read().decode("utf-8")) + if isinstance(payload, list): + results.extend(payload) + else: + # Non-list (single object): treat as one-item result and stop. + results.append(payload) + break + url = next_link(link_header) + return results + + +def next_link(link_header: str) -> str: + """Parse GitLab's Link header and return the next rel='next' URL, or ''.""" + if not link_header: + return "" + for part in link_header.split(","): + section = part.strip() + match = re.match(r'<([^>]+)>;\s*rel="([^"]+)"', section) + if match and match.group(2) == "next": + return match.group(1) + return "" + + +def parse_changes_block(block_text: str) -> dict[str, str] | None: + fields: dict[str, str] = {} + current_key: str | None = None + for raw_line in block_text.splitlines(): + match = KEY_VALUE_RE.match(raw_line.rstrip()) + if match and match.group(1).strip().lower() in KNOWN_BLOCK_KEYS: + key = match.group(1).strip().lower() + fields[key] = match.group(2).strip() + current_key = key + elif current_key is not None: + # Continuation line of the current field (e.g. a multi-line impact). + cont = raw_line.strip() + if cont: + fields[current_key] = ( + f"{fields[current_key]}\n{cont}" if fields[current_key] else cont + ) + required = {"section", "type", "summary"} + if not required.issubset(fields): + return None + return fields + + +def has_label(mr: dict, label: str) -> bool: + labels = mr.get("labels") or [] + for item in labels: + if isinstance(item, str) and item == label: + return True + if isinstance(item, dict) and item.get("name") == label: + return True + return False + + +def collect_entries_for_milestone( + api_base: str, project_id: str, milestone_title: str, token: str, + allowed_sections: set[str], +) -> list[dict]: + log(f"Fetching merged MRs for milestone '{milestone_title}'...") + mrs = api_get_paginated( + api_base, + f"/projects/{project_id}/merge_requests", + token, + params={ + "state": "merged", + "milestone": milestone_title, + "per_page": "100", + "order_by": "created_at", + "sort": "asc", + }, + ) + log(f"Found {len(mrs)} merged MR(s) for milestone '{milestone_title}'.") + + entries: list[dict] = [] + for mr in mrs: + if has_label(mr, "changelog"): + log(f"Skipping changelog MR !{mr['iid']}.") + continue + description = (mr.get("description") or "").strip() + if not description: + continue + for raw_block in CHANGES_BLOCK_RE.findall(description): + parsed = parse_changes_block(raw_block) + if parsed is None: + continue + section = parsed["section"] + if section not in allowed_sections: + log(f"WARN: MR !{mr['iid']} uses unknown section '{section}', skipping.") + continue + # impact_level: if section has :low suffix, pin to low unless explicit. + impact_level = parsed.get("impact_level", "") + if ":" in section: + impact_level = section.split(":", 1)[1] + entries.append( + { + "section": section, + "type": parsed["type"], + "summary": parsed["summary"], + "impact": parsed.get("impact", ""), + "impact_level": impact_level or "high", + "mr_iid": mr["iid"], + "mr_title": mr.get("title", ""), + "mr_url": mr.get("web_url", ""), + "author": (mr.get("author") or {}).get("username", ""), + } + ) + return entries + + +def group_entries(entries: list[dict]) -> dict[str, list[dict]]: + grouped: dict[str, list[dict]] = defaultdict(list) + for entry in entries: + grouped[entry["section"]].append(entry) + return grouped + + +def yaml_summary_scalar(value: str) -> str: + """Emit a YAML scalar for a changelog summary line. + + Plain style when safe (matches deckhouse/changelog-action output for the + common case); double-quoted otherwise to avoid YAML injection. + """ + if value == "": + return '""' + if ( + re.search(r"[:#]", value) + or value[0] in "-?,[]{}'\"&*!|>%@`" + or value.endswith(" ") + or ": " in value + or " #" in value + ): + return json.dumps(value, ensure_ascii=False) + return value + + +def render_yaml(entries: list[dict], milestone_title: str) -> str: + """Render CHANGELOG-.yml in the deckhouse schema. + + Schema (matches deckhouse/changelog-action@v2.6.0 release_yaml):: + +
: + features: + - summary: + pull_request: + fixes: + - summary: + pull_request: + + Sections are sorted alphabetically and emitted compactly (no blank lines + between sections). The ':low' impact_level suffix is stripped from the + section key: it only pins impact_level during validation and is not + represented in the YAML. Within each section, entries are ordered by MR iid + descending, matching the historical generator output. An empty milestone + yields '{}' (same as the historical generator). + """ + grouped: dict[str, dict[str, list[dict]]] = {} + for entry in entries: + section_key = entry["section"].split(":", 1)[0] + bucket = TYPE_TO_SECTION.get(entry["type"]) + if bucket is None: + log( + f"WARN: MR !{entry['mr_iid']} has unsupported type " + f"'{entry['type']}' (allowed: {sorted(ALLOWED_TYPES)}), skipping." + ) + continue + grouped.setdefault(section_key, {"features": [], "fixes": []})[bucket].append(entry) + + if not grouped: + return "{}\n\n" + + lines: list[str] = [] + for section in sorted(grouped.keys()): + buckets = grouped[section] + lines.append(f"{section}:") + for bucket in ("features", "fixes"): + items = sorted(buckets[bucket], key=lambda e: e["mr_iid"], reverse=True) + if not items: + continue + lines.append(f" {bucket}:") + for entry in items: + lines.append(f" - summary: {yaml_summary_scalar(entry['summary'])}") + lines.append(f" pull_request: {entry['mr_url']}") + # High-impact entries carry a free-text `impact` migration note. + # Preserve it (deckhouse/changelog-action emits it after + # pull_request); a multi-line note becomes a literal block. + impact = entry.get("impact", "") + if impact: + if "\n" in impact: + lines.append(" impact: |-") + for impact_line in impact.split("\n"): + lines.append(f" {impact_line}" if impact_line else "") + else: + lines.append(f" impact: {yaml_summary_scalar(impact)}") + return "\n".join(lines) + "\n\n" + + +def render_milestone_md_block(entries: list[dict], milestone_title: str) -> str: + """Render the markdown block for ONE milestone (patch version). + + Heading is `## ` so that + :func:`merge_minor_markdown` can merge multiple patch versions into the + cumulative `CHANGELOG-.md` idempotently. + """ + grouped = group_entries(entries) + lines = [f"## {milestone_title}", ""] + if not grouped: + lines.append("_No changelog entries._") + return "\n".join(lines).rstrip() + "\n" + for section in sorted(grouped.keys()): + lines.append(f"### {section}") + lines.append("") + for entry in grouped[section]: + lines.append( + f"- **{entry['type']}** ({entry['impact_level']}): " + f"{entry['summary']} ([!{entry['mr_iid']}]({entry['mr_url']}))" + ) + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def md_version_sort_key(title: str) -> tuple[int, int, int]: + """Sort key for `## vX.Y.Z` headings; missing parts sort as 0.""" + m = re.match(r"^v?(\d+)\.(\d+)(?:\.(\d+))?", title) + if not m: + return (0, 0, 0) + return tuple(int(part) if part else 0 for part in m.groups()) # type: ignore[return-value] + + +def parse_minor_md_blocks(text: str) -> dict[str, str]: + """Split an existing CHANGELOG-.md into {milestone_title: block}. + + Content before the first `## ` heading (the file header) is dropped — it is + regenerated. Each block keeps its own `## ` heading. + """ + blocks: dict[str, str] = {} + current_title: str | None = None + current_lines: list[str] = [] + for line in text.splitlines(): + if line.startswith("## "): + if current_title is not None: + blocks[current_title] = "\n".join(current_lines).rstrip() + "\n" + current_title = line[3:].strip() + current_lines = [line] + elif current_title is not None: + current_lines.append(line) + if current_title is not None: + blocks[current_title] = "\n".join(current_lines).rstrip() + "\n" + return blocks + + +def merge_minor_markdown( + existing_text: str, minor_version: str, milestone_title: str, block: str +) -> str: + """Merge ``block`` for ``milestone_title`` into the cumulative minor file. + + Replaces this milestone's block if present (idempotent re-generation) or + inserts it, then re-emits all patch blocks newest-first. This is what keeps + CHANGELOG-<minor>.md cumulative across patch releases (the GitHub + changelog-action produced a cumulative ``branch_markdown``); rendering only + the current milestone would drop the earlier patches. + """ + blocks = parse_minor_md_blocks(existing_text) + blocks[milestone_title] = block + ordered = sorted(blocks.keys(), key=md_version_sort_key, reverse=True) + out = [f"# Changelog {minor_version}", ""] + for title in ordered: + out.append(blocks[title].rstrip()) + out.append("") + return "\n".join(out).rstrip() + "\n" + + +def minor_version_from_tag(tag: str) -> str: + """v1.21.3 -> v1.21, v1.21 -> v1.21.""" + m = re.match(r"^v(\d+\.\d+)(?:\.\d+)?$", tag) + if not m: + return tag + return f"v{m.group(1)}" + + +def write_files( + project_dir: Path, + milestone_title: str, + entries: list[dict], +) -> tuple[Path, Path]: + changelog_dir = project_dir / "CHANGELOG" + changelog_dir.mkdir(parents=True, exist_ok=True) + yml_path = changelog_dir / f"CHANGELOG-{milestone_title}.yml" + minor = minor_version_from_tag(milestone_title) + md_path = changelog_dir / f"CHANGELOG-{minor}.md" + yml_path.write_text(render_yaml(entries, milestone_title), encoding="utf-8") + # Merge this milestone's block into the cumulative minor markdown so earlier + # patch releases of the same minor are preserved. + existing_md = md_path.read_text(encoding="utf-8") if md_path.is_file() else "" + block = render_milestone_md_block(entries, milestone_title) + md_path.write_text( + merge_minor_markdown(existing_md, minor, milestone_title, block), + encoding="utf-8", + ) + log(f"Wrote {yml_path.relative_to(project_dir)} and {md_path.relative_to(project_dir)}.") + return yml_path, md_path + + +def push_changelog_mr( + project_dir: Path, + project_path: str, + server_host: str, + token: str, + milestone_title: str, + base_branch: str, + pr_body_path: Path, +) -> None: + """Commit, push, and open a changelog MR.""" + branch = f"changelog/{milestone_title}" + subprocess.check_call(["git", "config", "user.email", "ci-changelog@flant.com"], cwd=project_dir) + subprocess.check_call(["git", "config", "user.name", "GitLab CI Changelog Bot"], cwd=project_dir) + + subprocess.check_call(["git", "checkout", "-B", branch], cwd=project_dir) + subprocess.check_call(["git", "add", "CHANGELOG/"], cwd=project_dir) + if subprocess.call(["git", "diff", "--cached", "--quiet"], cwd=project_dir) == 0: + log("No staged changes; skipping commit and MR creation.") + return + + subprocess.check_call( + ["git", "commit", "-m", f"Re-generate changelog {milestone_title}"], + cwd=project_dir, + ) + + repo_url = f"https://oauth2:{token}@{server_host}/{project_path}.git" + subprocess.check_call(["git", "remote", "set-url", "origin", repo_url], cwd=project_dir) + + push_cmd = [ + "git", "push", "--force", + "-o", "merge_request.create", + "-o", f"merge_request.target={base_branch}", + "-o", f"merge_request.source={branch}", + "-o", f"merge_request.title=Changelog {milestone_title}", + "-o", f"merge_request.label=changelog", + "-o", f"merge_request.label=auto", + "-o", f"merge_request.label=status/backport", + "-o", f"merge_request.milestone={milestone_title}", + "-o", f"merge_request.description={pr_body_path.read_text(encoding='utf-8')}", + "-o", "merge_request.remove_source_branch", + "origin", branch, + ] + subprocess.check_call(push_cmd, cwd=project_dir) + log(f"Pushed branch '{branch}' and opened MR via push options.") + + +def main() -> int: + api_base = require_env("CI_API_V4_URL").rstrip("/") + project_id = require_env("CI_PROJECT_ID") + token = require_env("GITLAB_API_TOKEN") + project_path = require_env("CI_PROJECT_PATH") + server_host = require_env("CI_SERVER_HOST") + project_dir = Path(require_env("CI_PROJECT_DIR")) + + sections_path = Path( + os.environ.get( + "CHANGELOG_SECTIONS_FILE", ".gitlab/ci/changelog-sections.txt" + ) + ) + if not sections_path.is_file(): + log(f"ERROR: sections file not found: {sections_path}") + return 1 + allowed_sections = { + line.strip() + for line in sections_path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith("#") + } + + base_branch = os.environ.get("CHANGELOG_BASE_BRANCH", "main") + open_mr = os.environ.get("OPEN_CHANGELOG_MR", "false").lower() == "true" + + target_milestones: list[dict] = [] + explicit = os.environ.get("MILESTONE_TITLE", "").strip() + if explicit: + # Resolve to {title, iid}. + all_ms = api_get_paginated( + api_base, + f"/projects/{project_id}/milestones", + token, + params={"state": "active", "per_page": "100"}, + ) + match = next((m for m in all_ms if m["title"] == explicit), None) + if match is None: + log(f"ERROR: milestone '{explicit}' not found among active milestones.") + return 1 + target_milestones = [match] + else: + log("No MILESTONE_TITLE set — iterating over all active milestones.") + target_milestones = api_get_paginated( + api_base, + f"/projects/{project_id}/milestones", + token, + params={"state": "active", "per_page": "100"}, + ) + + if not target_milestones: + log("No milestones to process. Exiting 0.") + return 0 + + overall_errors = 0 + for milestone in target_milestones: + title = milestone["title"] + iid = milestone["iid"] + log(f"Processing milestone '{title}' (iid={iid})...") + try: + entries = collect_entries_for_milestone( + api_base, project_id, title, token, allowed_sections + ) + except urllib.error.HTTPError as exc: + log(f"ERROR fetching MRs for {title}: HTTP {exc.code} {exc.reason}") + overall_errors += 1 + continue + + yml_path, md_path = write_files(project_dir, title, entries) + + if open_mr: + pr_body = ( + f"## Changelog {title}\n\n" + f"Auto-generated changelog covering milestone `{title}` " + f"({len(entries)} change entries).\n\n" + f"See:\n" + f"- `{yml_path.relative_to(project_dir)}`\n" + f"- `{md_path.relative_to(project_dir)}`\n" + ) + # Write the MR body OUTSIDE CHANGELOG/ so `git add CHANGELOG/` + # in push_changelog_mr cannot accidentally stage it. It is only + # consumed by reading its content into a merge_request.description + # push option and is never committed to the changelog branch. + body_path = project_dir / f".mr-body-{title}.md" + body_path.write_text(pr_body, encoding="utf-8") + try: + push_changelog_mr( + project_dir=project_dir, + project_path=project_path, + server_host=server_host, + token=token, + milestone_title=title, + base_branch=base_branch, + pr_body_path=body_path, + ) + except subprocess.CalledProcessError as exc: + log(f"ERROR pushing changelog MR for {title}: {exc}") + overall_errors += 1 + continue + finally: + if body_path.exists(): + body_path.unlink() + + return 1 if overall_errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitlab/ci/scripts/python/check_changelog_entry.py b/.gitlab/ci/scripts/python/check_changelog_entry.py new file mode 100644 index 0000000000..4979e515f5 --- /dev/null +++ b/.gitlab/ci/scripts/python/check_changelog_entry.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate ```changes fenced blocks in a GitLab MR description. + +Migration of the validation logic in +.github/workflows/check-changelog-entry.yml which used +deckhouse/changelog-action@v2.6.0 with validate_only=true. + +Behaviour: + - Fetch MR description via GitLab API (CI_API_V4_URL). + - Locate fenced code blocks with language ``changes``. + - For each block validate required keys: section, type, summary. + - ``section`` must be in the allowed list (.gitlab/ci/changelog-sections.txt). + - If a section is suffixed ``:low`` (e.g. ``ci:low``), impact_level is optional + and pinned to ``low``; otherwise impact_level is required. + - If no ```changes blocks at all -> OK (PR may not require changelog). + - Otherwise collect errors and exit non-zero. + +Required environment: + GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_MERGE_REQUEST_IID + +Optional: + CHANGELOG_SECTIONS_FILE (default: .gitlab/ci/changelog-sections.txt) +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + + +CHANGES_BLOCK_RE = re.compile( + r"```changes\s*\n(.*?)\n```", + re.DOTALL, +) +KEY_VALUE_RE = re.compile(r"^([A-Za-z_]+)\s*:\s*(.*)$") +# deckhouse/changelog-action@v2.6.0 only renders 'feature' (-> features) and +# 'fix' (-> fixes) in CHANGELOG-*.yml. Keep in sync with changelog_collect.py. +ALLOWED_TYPES = {"feature", "fix"} + + +def log(message: str) -> None: + print(message, file=sys.stderr) + + +def require_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + log(f"ERROR: required environment variable {name} is not set") + sys.exit(1) + return value + + +def fetch_mr_description( + api_base: str, project_id: str, mr_iid: str, token: str +) -> str: + """GET the MR via REST API and return its description.""" + url = f"{api_base}/projects/{project_id}/merge_requests/{mr_iid}" + req = urllib.request.Request( + url, + headers={ + "PRIVATE-TOKEN": token, + "Accept": "application/json", + }, + method="GET", + ) + try: + with urllib.request.urlopen(req) as response: + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + log(f"ERROR: failed to fetch MR: HTTP {exc.code} {exc.reason}") + sys.exit(1) + return (payload.get("description") or "").strip() + + +def load_allowed_sections(path: Path) -> set[str]: + text = path.read_text(encoding="utf-8") + sections: set[str] = set() + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + sections.add(line) + return sections + + +def parse_block(block_text: str) -> dict[str, str]: + fields: dict[str, str] = {} + for raw_line in block_text.splitlines(): + match = KEY_VALUE_RE.match(raw_line.rstrip()) + if not match: + continue + key, value = match.group(1).strip().lower(), match.group(2).strip() + fields[key] = value + return fields + + +def validate_block( + block_index: int, + block_text: str, + allowed_sections: set[str], +) -> list[str]: + errors: list[str] = [] + fields = parse_block(block_text) + + section = fields.get("section", "") + if not section: + errors.append(f"block #{block_index}: missing required key 'section'") + elif section not in allowed_sections: + errors.append( + f"block #{block_index}: section '{section}' is not in " + f"allowed_sections (see .gitlab/ci/changelog-sections.txt)" + ) + + change_type = fields.get("type", "") + if not change_type: + errors.append(f"block #{block_index}: missing required key 'type'") + elif change_type not in ALLOWED_TYPES: + errors.append( + f"block #{block_index}: type '{change_type}' is not supported; " + f"allowed types are {sorted(ALLOWED_TYPES)} " + f"(deckhouse changelog only supports 'feature' and 'fix', " + f"rendered as the 'features'/'fixes' sections)" + ) + + summary = fields.get("summary", "") + if not summary: + errors.append(f"block #{block_index}: missing required key 'summary'") + + # impact_level: optional iff section suffix is :low. + section_suffix_low = ":" in section and section.split(":", 1)[1] == "low" + impact_level = fields.get("impact_level", "") + if not section_suffix_low and not impact_level: + errors.append( + f"block #{block_index}: missing required key 'impact_level' " + "(only allowed to omit when section ends with ':low')" + ) + elif section_suffix_low and impact_level and impact_level != "low": + errors.append( + f"block #{block_index}: section '{section}' is pinned to low " + f"impact but impact_level='{impact_level}' was provided" + ) + + return errors + + +def main() -> int: + api_base = require_env("CI_API_V4_URL").rstrip("/") + project_id = require_env("CI_PROJECT_ID") + mr_iid = require_env("CI_MERGE_REQUEST_IID") + token = require_env("GITLAB_API_TOKEN") + + pipeline_source = os.environ.get("CI_PIPELINE_SOURCE", "") + if pipeline_source != "merge_request_event": + log(f"Not a merge request pipeline (CI_PIPELINE_SOURCE={pipeline_source}). Skipping.") + return 0 + + sections_path = Path( + os.environ.get( + "CHANGELOG_SECTIONS_FILE", + ".gitlab/ci/changelog-sections.txt", + ) + ) + if not sections_path.is_file(): + log(f"ERROR: allowed sections file not found: {sections_path}") + return 1 + allowed_sections = load_allowed_sections(sections_path) + + description = fetch_mr_description(api_base, project_id, mr_iid, token) + blocks = CHANGES_BLOCK_RE.findall(description) + + if not blocks: + log("No ```changes blocks in MR description — OK (changelog not required).") + return 0 + + log(f"Found {len(blocks)} ```changes block(s) in MR description — validating.") + + all_errors: list[str] = [] + for index, raw_block in enumerate(blocks, start=1): + all_errors.extend( + validate_block(index, raw_block, allowed_sections) + ) + + if all_errors: + for err in all_errors: + log(f"ERROR: {err}") + log(f"{len(all_errors)} validation error(s) found.") + return 1 + + log(f"All {len(blocks)} ```changes block(s) are valid.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitlab/ci/scripts/python/tests/__init__.py b/.gitlab/ci/scripts/python/tests/__init__.py new file mode 100644 index 0000000000..54b1119619 --- /dev/null +++ b/.gitlab/ci/scripts/python/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/.gitlab/ci/scripts/python/tests/test_changelog_collect.py b/.gitlab/ci/scripts/python/tests/test_changelog_collect.py new file mode 100644 index 0000000000..161f9c499d --- /dev/null +++ b/.gitlab/ci/scripts/python/tests/test_changelog_collect.py @@ -0,0 +1,258 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for changelog_collect.py pure helpers. + +main() is guarded by `if __name__ == "__main__"`, so the import is side-effect +free. The network/git helpers (api_get_paginated, push_changelog_mr) are not +exercised here; everything below is the pure parse/group/render logic that +produces the deckhouse-schema CHANGELOG-*.yml and the .md summary. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import changelog_collect as cl # noqa: E402 + + +def entry(section, type_, summary, mr_iid, impact_level="high", mr_url=None, impact=""): + return { + "section": section, + "type": type_, + "summary": summary, + "impact": impact, + "impact_level": impact_level, + "mr_iid": mr_iid, + "mr_title": f"MR {mr_iid}", + "mr_url": mr_url or f"https://gl/-/merge_requests/{mr_iid}", + "author": "alice", + } + + +class NextLinkTest(unittest.TestCase): + def test_extracts_next_url(self): + header = ( + '<https://gl/api/v4/x?page=1>; rel="prev", ' + '<https://gl/api/v4/x?page=3>; rel="next"' + ) + self.assertEqual(cl.next_link(header), "https://gl/api/v4/x?page=3") + + def test_no_next_returns_empty(self): + header = '<https://gl/api/v4/x?page=1>; rel="prev"' + self.assertEqual(cl.next_link(header), "") + + def test_empty_header(self): + self.assertEqual(cl.next_link(""), "") + + +class ParseChangesBlockTest(unittest.TestCase): + def test_parses_required_keys(self): + parsed = cl.parse_changes_block("section: vm\ntype: fix\nsummary: did it") + self.assertEqual(parsed["section"], "vm") + self.assertEqual(parsed["type"], "fix") + self.assertEqual(parsed["summary"], "did it") + + def test_returns_none_when_required_key_missing(self): + # summary missing -> not a valid changes block. + self.assertIsNone(cl.parse_changes_block("section: vm\ntype: fix")) + + def test_keeps_optional_keys(self): + parsed = cl.parse_changes_block( + "section: vm\ntype: fix\nsummary: s\nimpact_level: low" + ) + self.assertEqual(parsed["impact_level"], "low") + + def test_multiline_impact_is_preserved(self): + parsed = cl.parse_changes_block( + "section: core\ntype: feature\nsummary: containerd v2\n" + "impact: First line.\nSecond line.\nimpact_level: high" + ) + self.assertEqual(parsed["impact"], "First line.\nSecond line.") + self.assertEqual(parsed["impact_level"], "high") + self.assertEqual(parsed["summary"], "containerd v2") + + +class HasLabelTest(unittest.TestCase): + def test_string_labels(self): + self.assertTrue(cl.has_label({"labels": ["changelog", "auto"]}, "changelog")) + self.assertFalse(cl.has_label({"labels": ["auto"]}, "changelog")) + + def test_dict_labels(self): + self.assertTrue( + cl.has_label({"labels": [{"name": "changelog"}]}, "changelog") + ) + + def test_missing_labels_key(self): + self.assertFalse(cl.has_label({}, "changelog")) + + +class GroupEntriesTest(unittest.TestCase): + def test_groups_by_section(self): + entries = [ + entry("vm", "fix", "a", 1), + entry("vm", "feature", "b", 2), + entry("core", "fix", "c", 3), + ] + grouped = cl.group_entries(entries) + self.assertEqual(len(grouped["vm"]), 2) + self.assertEqual(len(grouped["core"]), 1) + + +class YamlSummaryScalarTest(unittest.TestCase): + def test_plain_when_safe(self): + self.assertEqual(cl.yaml_summary_scalar("simple summary"), "simple summary") + + def test_empty_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar(""), '""') + + def test_colon_space_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("fix: thing"), '"fix: thing"') + + def test_leading_special_char_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("- dash start"), '"- dash start"') + + def test_trailing_space_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("trailing "), '"trailing "') + + def test_hash_comment_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("note #5"), '"note #5"') + + +class RenderYamlTest(unittest.TestCase): + def test_empty_entries_render_empty_mapping(self): + self.assertEqual(cl.render_yaml([], "v1.21.0"), "{}\n\n") + + def test_groups_into_features_and_fixes(self): + entries = [ + entry("vm", "feature", "added X", 10), + entry("vm", "fix", "fixed Y", 11), + ] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn("vm:", out) + self.assertIn(" features:", out) + self.assertIn(" fixes:", out) + self.assertIn(" - summary: added X", out) + self.assertIn(" pull_request: https://gl/-/merge_requests/10", out) + + def test_entries_ordered_by_mr_iid_descending(self): + entries = [ + entry("vm", "fix", "older", 5), + entry("vm", "fix", "newer", 9), + ] + out = cl.render_yaml(entries, "v1.21.0") + self.assertLess(out.index("newer"), out.index("older")) + + def test_sections_sorted_alphabetically(self): + entries = [ + entry("vm", "fix", "v", 1), + entry("core", "fix", "c", 2), + ] + out = cl.render_yaml(entries, "v1.21.0") + self.assertLess(out.index("core:"), out.index("vm:")) + + def test_low_suffix_stripped_from_section_key(self): + entries = [entry("ci:low", "fix", "tweak", 1, impact_level="low")] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn("ci:", out) + self.assertNotIn("ci:low:", out) + + def test_unsupported_type_is_skipped(self): + # 'chore' has no features/fixes bucket -> dropped from yaml output. + entries = [entry("vm", "chore", "noise", 1)] + self.assertEqual(cl.render_yaml(entries, "v1.21.0"), "{}\n\n") + + def test_single_line_impact_emitted_after_pull_request(self): + entries = [entry("core", "feature", "containerd v2", 9, impact="Recreate images.")] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn(" pull_request: https://gl/-/merge_requests/9", out) + self.assertIn(" impact: Recreate images.", out) + + def test_multiline_impact_emitted_as_literal_block(self): + entries = [entry("core", "feature", "containerd v2", 9, impact="L1\nL2")] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn(" impact: |-", out) + self.assertIn(" L1", out) + self.assertIn(" L2", out) + + def test_no_impact_means_no_impact_line(self): + entries = [entry("vm", "fix", "fixed Y", 11)] + out = cl.render_yaml(entries, "v1.21.0") + self.assertNotIn("impact:", out) + + +class RenderMilestoneMdBlockTest(unittest.TestCase): + def test_basic_structure(self): + entries = [entry("vm", "fix", "fixed Y", 11, impact_level="high")] + out = cl.render_milestone_md_block(entries, "v1.21.0") + self.assertIn("## v1.21.0", out) + self.assertIn("### vm", out) + self.assertIn("**fix** (high): fixed Y ([!11]", out) + + def test_empty_entries_render_placeholder(self): + out = cl.render_milestone_md_block([], "v1.21.0") + self.assertIn("## v1.21.0", out) + self.assertIn("_No changelog entries._", out) + + +class MergeMinorMarkdownTest(unittest.TestCase): + def test_new_file_creates_header_and_block(self): + block = cl.render_milestone_md_block( + [entry("vm", "fix", "a", 1)], "v1.21.0" + ) + out = cl.merge_minor_markdown("", "v1.21", "v1.21.0", block) + self.assertIn("# Changelog v1.21", out) + self.assertIn("## v1.21.0", out) + + def test_existing_patch_preserved_and_sorted_desc(self): + first = cl.merge_minor_markdown( + "", "v1.21", "v1.21.0", + cl.render_milestone_md_block([entry("vm", "fix", "older", 1)], "v1.21.0"), + ) + second = cl.merge_minor_markdown( + first, "v1.21", "v1.21.1", + cl.render_milestone_md_block([entry("vm", "fix", "newer", 2)], "v1.21.1"), + ) + # Both patch blocks are present (cumulative)... + self.assertIn("## v1.21.0", second) + self.assertIn("## v1.21.1", second) + self.assertIn("older", second) + self.assertIn("newer", second) + # ...and the newer patch is listed first. + self.assertLess(second.index("## v1.21.1"), second.index("## v1.21.0")) + + def test_regenerating_same_milestone_is_idempotent(self): + block_v0 = cl.render_milestone_md_block( + [entry("vm", "fix", "a", 1)], "v1.21.0" + ) + once = cl.merge_minor_markdown("", "v1.21", "v1.21.0", block_v0) + twice = cl.merge_minor_markdown(once, "v1.21", "v1.21.0", block_v0) + self.assertEqual(once, twice) + + +class MinorVersionFromTagTest(unittest.TestCase): + def test_patch_version_truncated_to_minor(self): + self.assertEqual(cl.minor_version_from_tag("v1.21.3"), "v1.21") + + def test_minor_version_unchanged(self): + self.assertEqual(cl.minor_version_from_tag("v1.21"), "v1.21") + + def test_non_matching_returned_as_is(self): + self.assertEqual(cl.minor_version_from_tag("nightly"), "nightly") + + +if __name__ == "__main__": + unittest.main() diff --git a/.gitlab/ci/scripts/python/tests/test_check_changelog_entry.py b/.gitlab/ci/scripts/python/tests/test_check_changelog_entry.py new file mode 100644 index 0000000000..986fd0b6c1 --- /dev/null +++ b/.gitlab/ci/scripts/python/tests/test_check_changelog_entry.py @@ -0,0 +1,144 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for check_changelog_entry.py pure helpers. + +The module guards main() behind `if __name__ == "__main__"`, so importing it +runs no network calls and never exits. We exercise the block parsing and +validation logic directly with synthetic MR-description text — no GitLab API +access required. +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +# Put the python/ dir (parent of tests/) on the path so the scripts under test +# import cleanly regardless of the discover invocation's cwd. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import check_changelog_entry as cc # noqa: E402 + + +# A representative allowed-sections set used across the validation tests. +ALLOWED = {"vm", "core", "ci", "ci:low"} + + +class FindBlocksTest(unittest.TestCase): + def test_finds_single_block(self): + text = "intro\n```changes\nsection: vm\n```\noutro" + blocks = cc.CHANGES_BLOCK_RE.findall(text) + self.assertEqual(len(blocks), 1) + self.assertIn("section: vm", blocks[0]) + + def test_finds_multiple_blocks(self): + text = ( + "```changes\nsection: vm\n```\n" + "text in between\n" + "```changes\nsection: core\n```\n" + ) + blocks = cc.CHANGES_BLOCK_RE.findall(text) + self.assertEqual(len(blocks), 2) + + def test_no_block_returns_empty(self): + self.assertEqual(cc.CHANGES_BLOCK_RE.findall("no fenced block here"), []) + + +class ParseBlockTest(unittest.TestCase): + def test_parses_keys_lowercased(self): + fields = cc.parse_block("Section: vm\nType: fix\nSummary: did a thing") + self.assertEqual( + fields, {"section": "vm", "type": "fix", "summary": "did a thing"} + ) + + def test_ignores_non_keyvalue_lines(self): + fields = cc.parse_block("section: vm\nthis is prose\n\ntype: fix") + self.assertEqual(fields, {"section": "vm", "type": "fix"}) + + def test_trims_whitespace_around_value(self): + fields = cc.parse_block("section: vm ") + self.assertEqual(fields["section"], "vm") + + +class LoadAllowedSectionsTest(unittest.TestCase): + def test_strips_comments_and_blank_lines(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "sections.txt" + p.write_text( + "# a comment\n\nvm\ncore\n \n# another\nci:low\n", + encoding="utf-8", + ) + self.assertEqual( + cc.load_allowed_sections(p), {"vm", "core", "ci:low"} + ) + + +class ValidateBlockTest(unittest.TestCase): + def test_valid_block_has_no_errors(self): + block = "section: vm\ntype: fix\nsummary: fixed it\nimpact_level: high" + self.assertEqual(cc.validate_block(1, block, ALLOWED), []) + + def test_missing_section(self): + block = "type: fix\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'section'" in e for e in errors)) + + def test_section_not_allowed(self): + block = "section: bogus\ntype: fix\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("not in" in e and "allowed_sections" in e for e in errors)) + + def test_missing_type(self): + block = "section: vm\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'type'" in e for e in errors)) + + def test_type_not_allowed(self): + # 'chore' is intentionally rejected: deckhouse changelog renders only + # feature/fix (ALLOWED_TYPES). Guards the 5.4 fix from regressing. + block = "section: vm\ntype: chore\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("type 'chore' is not supported" in e for e in errors)) + + def test_allowed_types_is_feature_fix_only(self): + self.assertEqual(cc.ALLOWED_TYPES, {"feature", "fix"}) + + def test_missing_summary(self): + block = "section: vm\ntype: fix\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'summary'" in e for e in errors)) + + def test_missing_impact_level_when_not_low(self): + block = "section: vm\ntype: fix\nsummary: s" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'impact_level'" in e for e in errors)) + + def test_low_section_may_omit_impact_level(self): + block = "section: ci:low\ntype: fix\nsummary: s" + self.assertEqual(cc.validate_block(1, block, ALLOWED), []) + + def test_low_section_with_explicit_low_is_ok(self): + block = "section: ci:low\ntype: fix\nsummary: s\nimpact_level: low" + self.assertEqual(cc.validate_block(1, block, ALLOWED), []) + + def test_low_section_with_conflicting_impact_level(self): + block = "section: ci:low\ntype: fix\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("pinned to low" in e for e in errors)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml new file mode 100644 index 0000000000..198f7ef040 --- /dev/null +++ b/.gitlab/ci/stages.yml @@ -0,0 +1,55 @@ +# Pipeline stages for the deckhouse/virtualization module. +# +# This is the union of the stages used by the previous root .gitlab-ci.yml +# and the new stages introduced during the GitHub Actions migration. +# +# We intentionally drop the `e2e` stage here: all e2e-* workflows are +# excluded from the GitLab CI migration. Any future re-introduction of e2e +# stages lives in a separate file. +# +# Stage semantics: +# pre — upstream Translate_Changelog hidden template stage +# info — manifest printers, set_vars helper +# lint — Go lint, helm lint, yaml lint, no-cyrillic, etc. +# test — Go unit tests, hooks tests +# prod_check — release-channel requirements check +# (prod:check-requirements). Runs ONLY in the manual +# web release-channel flow (CI_PIPELINE_SOURCE == web), +# NOT in dev/MR pipelines. Declared BEFORE build purely +# so prod:build:* can `needs:` it — the early position +# is a DAG-ordering artifact, not a dev-pipeline stage. +# build — werf build for dev / dev-tags / main / prod +# (dev tags are build-only: the DEV registry has no +# release channels, so there is no DEV deploy stage) +# scan — cve_scan, gitleaks, svace (owned by sibling issues) +# deploy_prod — prod release-channel deploy. ONE stage with independent +# manual jobs per channel (tag-push flow, deploy-prod.yml) +# and per-edition deploy from Run pipeline +# (web flow, release-channels.yml). No forced +# alpha->...->rock-solid promotion chain — each channel +# deploy is an independent `when: manual` job, matching +# GitHub release_module_release-channels.yml semantics. +# prod_verify — version-on-release-channel check matrix +# prod_release — GitLab release creation +# notify — release-results-to-loop and similar fan-out +# cleanup — scheduled registry cleanup +# +# Upstream Setup.gitlab-ci.yml declares `stages: [build, deploy]` which +# would otherwise conflict with our expanded list. We override by re-stating +# the full list here; jobs that `extends: .build` keep `stage: build` and +# land in our `build` stage (not a separate "build" from upstream). + +stages: + # Keep upstream hidden templates valid even when visible jobs override stages. + - pre + - info + - lint + - test + - prod_check + - build + - scan + - deploy_prod + - prod_verify + - prod_release + - notify + - cleanup diff --git a/.gitlab/ci/templates/base/build.yml b/.gitlab/ci/templates/base/build.yml new file mode 100644 index 0000000000..ae4b04c773 --- /dev/null +++ b/.gitlab/ci/templates/base/build.yml @@ -0,0 +1,72 @@ +# Base build template (.base_build). +# +# Naming: this is the BASE werf-build job body that other build jobs extend +# (build_dev / build_main / build_prod / prod:build:* / precache / svace). +# It was previously named `.local_build`; renamed to `.base_build` because +# "local" was uninformative and collided conceptually with `include: local:`. +# "base" states the intent: a base template meant to be extended. +# +# This template mirrors the upstream modules-gitlab-ci Build.gitlab-ci.yml +# `.build` body verbatim (verified against +# deckhouse/3p/deckhouse/modules-gitlab-ci +# templates/Build.gitlab-ci.yml, branch v13.0, HEAD 006d51c). +# +# Why a local copy instead of `extends: .build`: +# +# Upstream `.build` carries two `rules:` that fire on ANY branch or tag +# push (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`). GitLab CI merges +# rules from all `extends:` parents with the parent's rules evaluated +# first, so the upstream rules always win. That regresses behavior for +# jobs like build_dev_tags, build_main, build_prod which need to gate on +# specific branch / tag patterns provided by .dev / .dev_tags / .main / +# .prod_always. +# +# By keeping the upstream body in a base template `.base_build` we keep +# the same script (so future upstream fixes can be copy-pasted here) +# while our job-level rules from .dev / .main / .prod_always / etc. +# drive the gating. +# +# Required variables (documented in variables.yml + upstream Setup): +# MODULES_MODULE_SOURCE, MODULES_MODULE_NAME, MODULES_MODULE_TAG, +# WERF_REPO, SVACE_ENABLED (optional). +# +# TODO: after the upstream modules-gitlab-ci repo lands a way to pass +# `rules:` overrides through extends (tracked in upstream issues), revisit +# this template and switch back to `extends: .build`. + +.base_build: + stage: build + script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf jq crane + # Use gitlab ci job token + - | + SOURCE_REPO=${SOURCE_REPO#git@} + SOURCE_REPO=${SOURCE_REPO//://} + export SOURCE_REPO=https://gitlab-ci-token:${CI_JOB_TOKEN}@${SOURCE_REPO} + + # Build images + - | + werf build \ + --save-build-report --build-report-path images_tags_werf.json + + # Bundle image + - | + IMAGE_SRC="$(jq -r '.Images."bundle".DockerImageName' images_tags_werf.json)" + IMAGE_DST="$(jq -r '.Images.bundle.DockerRepo' images_tags_werf.json):${MODULES_MODULE_TAG}" + + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + # Release-channel image + - | + IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)" + IMAGE_DST="$(jq -r '.Images."release-channel-version".DockerRepo' images_tags_werf.json)/release:${MODULES_MODULE_TAG}" + + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + # Register module + - | + echo "✨ Register the module ${MODULES_MODULE_NAME}" + crane append \ + --oci-empty-base \ + --new_layer "" \ + --new_tag "${MODULES_MODULE_SOURCE}:${MODULES_MODULE_NAME}" diff --git a/.gitlab/ci/templates/base/deploy.yml b/.gitlab/ci/templates/base/deploy.yml new file mode 100644 index 0000000000..6530fdfddc --- /dev/null +++ b/.gitlab/ci/templates/base/deploy.yml @@ -0,0 +1,36 @@ +# Base deploy template (.base_deploy). +# +# Naming: this is the BASE crane-copy deploy job body extended by the prod +# release-channel deploy jobs (deploy_to_prod_* and prod:deploy:*). Previously +# named `.local_deploy`; renamed to `.base_deploy` for the same reason as +# `.base_build` ("local" was uninformative and clashed with `include: local:`). +# Note: there is no DEV deploy — the DEV registry has no release channels, so +# dev tags are build-only (see build_dev_tags). +# +# Mirrors upstream modules-gitlab-ci Deploy.gitlab-ci.yml `.deploy` body +# (verified in deckhouse/3p/deckhouse/modules-gitlab-ci +# templates/Deploy.gitlab-ci.yml, branch v13.0, HEAD 006d51c). +# +# Why a local copy (same reasoning as .base_build in build.yml): +# +# Upstream `.deploy` has `rules: [{if: $CI_COMMIT_TAG}]` + `when: manual` +# baked in. Because GitLab CI merges parent rules with parent-first +# precedence, those rules would override our own gating. +# The prod-channel chain (deploy_to_prod_*) wants `when: manual` from +# .prod_manual, so the upstream `when: manual` happens to match — but we +# still want explicit control. Keeping the upstream body in `.base_deploy` +# means the same script is shared by all jobs but gating is fully driven +# by our own templates. + +.base_deploy: + stage: deploy + script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh crane + - | + REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" + + IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" + IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" + + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy "${IMAGE_SRC}" "${IMAGE_DST}" diff --git a/.gitlab/ci/templates/base/info.yml b/.gitlab/ci/templates/base/info.yml new file mode 100644 index 0000000000..2d4da2cd77 --- /dev/null +++ b/.gitlab/ci/templates/base/info.yml @@ -0,0 +1,50 @@ +# Info-job template: prints a kubectl manifest helper for testers. +# +# Carries forward the previous root .gitlab-ci.yml `.info` template. Used +# by jobs in .gitlab/ci/jobs/info.yml that print a copy-paste-ready +# ModuleConfig + ModulePullOverride YAML for the MR/build being tested. +# +# Note: the previous version inlined this template inside the manifest +# jobs. We split it out so show_dev_manifest and show_main_manifest can +# share it via extends. + +.info: + script: + - | + cat << OUTER + Create ModuleConfig and ModulePullOverride resources to test this MR: + cat <<EOF | kubectl apply -f - + --- + apiVersion: deckhouse.io/v1alpha1 + kind: ModulePullOverride + metadata: + name: virtualization + spec: + imageTag: ${MODULES_MODULE_TAG} + source: deckhouse + + --- + apiVersion: deckhouse.io/v1alpha1 + kind: ModuleConfig + metadata: + name: ${MODULE_NAME} + spec: + enabled: true + settings: + dvcr: + storage: + type: PersistentVolumeClaim + persistentVolumeClaim: + size: 50G + virtualMachineCIDRs: + - 10.66.10.0/24 + - 10.66.20.0/24 + - 10.66.30.0/24 + version: 1 + EOF + + Or patch an existing ModulePullOverride: + + kubectl patch mpo ${MODULE_NAME} --type merge -p '{"spec":{"imageTag":"${MODULES_MODULE_TAG}"}}' + + OUTER diff --git a/.gitlab/ci/templates/base/registry-login.yml b/.gitlab/ci/templates/base/registry-login.yml new file mode 100644 index 0000000000..6e757c2766 --- /dev/null +++ b/.gitlab/ci/templates/base/registry-login.yml @@ -0,0 +1,57 @@ +# Registry authentication helpers — all `werf cr login` wiring in one place. +# +# The actual login happens in the upstream Setup.gitlab-ci.yml before_script, +# which runs for every job: +# 1. ALWAYS logs into $MODULES_REGISTRY using +# $MODULES_REGISTRY_LOGIN / $MODULES_REGISTRY_PASSWORD — the primary +# registry the job builds/pushes/pulls against. +# 2. ADDITIONALLY logs into $DEV_MODULES_REGISTRY when its credentials are +# set — a second registry used only to pull already-built layers. +# +# These templates just populate those variables, so `extends`-ing them selects +# which registries a job authenticates to. Pick the primary with exactly one of +# .login_dev_registry / .login_prod_registry / .login_prod_read_registry, and +# add .also_login_dev_registry when a prod job must also read the DEV registry. + +# Primary = DEV registry (write). Used by all dev builds +# (MR / main / release-* / dev tags); pulled in via .dev_vars. +.login_dev_registry: + variables: + MODULES_REGISTRY: "${DEV_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" + +# Primary = PROD registry (write). Used by prod release builds/deploys; +# pulled in via .prod_vars. +.login_prod_registry: + variables: + MODULES_REGISTRY: "${PROD_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${PROD_MODULES_REGISTRY_LOGIN}" + MODULES_REGISTRY_PASSWORD: "${PROD_MODULES_REGISTRY_PASSWORD}" + +# Primary = PROD read-only registry. Used by the version/requirements check +# jobs (prod:check-requirements, prod:check-version). NOTE: the Go/bash tools +# hardcode registry.deckhouse.io, so PROD_READ_REGISTRY must resolve to that +# host for crane auth to apply (see tools/moduleversions Taskfile.dist.yaml). +.login_prod_read_registry: + variables: + MODULES_REGISTRY: "${PROD_READ_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${PROD_READ_REGISTRY_USER}" + MODULES_REGISTRY_PASSWORD: "${PROD_READ_REGISTRY_PASSWORD}" + +# Additional DEV login for prod jobs that must ALSO read the DEV registry. +# Compose with the prod primary, e.g.: +# extends: [.prod_vars, .also_login_dev_registry, .base_build] +# Prod werf builds reuse already-built layers from the DEV registry as a +# read-only secondary repo (WERF_SECONDARY_REPO_1). This mirrors the GitHub +# build action input +# `secondary_repo: "${vars.DEV_MODULE_SOURCE}/${vars.MODULE_NAME}"`, which the +# action exports as WERF_SECONDARY_REPO_1 (see modules-actions/build/action.yml). +# Dev builds never need this (their primary already IS the DEV registry); deploy +# jobs that compose it ignore the secondary repo (they crane-copy, not build). +.also_login_dev_registry: + variables: + DEV_MODULES_REGISTRY: "${DEV_REGISTRY}" + DEV_MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + DEV_MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" + WERF_SECONDARY_REPO_1: "${DEV_MODULE_SOURCE}/${MODULE_NAME}" diff --git a/.gitlab/ci/templates/dev/dev.yml b/.gitlab/ci/templates/dev/dev.yml new file mode 100644 index 0000000000..4ab89a9694 --- /dev/null +++ b/.gitlab/ci/templates/dev/dev.yml @@ -0,0 +1,16 @@ +# DEV pipeline context: MR builds. +# +# Carries forward the previous root .gitlab-ci.yml `.dev` template. Any job +# that uses `extends: .dev` runs only on MR pipelines, derives its +# MODULES_MODULE_TAG from the MR iid, and inherits the DEV registry +# credentials via .dev_vars. + +.dev: + variables: + MODULES_MODULE_TAG: mr${CI_MERGE_REQUEST_IID} + extends: + - .dev_vars + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never diff --git a/.gitlab/ci/templates/dev/dev_tags.yml b/.gitlab/ci/templates/dev/dev_tags.yml new file mode 100644 index 0000000000..532a807685 --- /dev/null +++ b/.gitlab/ci/templates/dev/dev_tags.yml @@ -0,0 +1,21 @@ +# DEV pipeline context: dev-tag builds. +# +# Carries forward the previous root .gitlab-ci.yml `.dev_tags` template. +# Triggers on tags matching vX.Y.Z-dev* (regex kept identical to the +# previous regex101 link in the original .gitlab-ci.yml). +# +# Use case: a developer pushes v1.21.0-dev-build42 -> build_dev_tags runs and +# pushes images into DEV_REGISTRY tagged with the raw tag name. There is no +# deploy step: the DEV registry has no release channels, only tags (matching +# the build-only GitHub "Deploy Dev" workflow). + +.dev_tags: + variables: + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + extends: + - .dev_vars + rules: + # https://regex101.com/r/0VtnPP/1 + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-dev.*$/' + when: always + - when: never diff --git a/.gitlab/ci/templates/dev/dev_vars.yml b/.gitlab/ci/templates/dev/dev_vars.yml new file mode 100644 index 0000000000..4894eeba69 --- /dev/null +++ b/.gitlab/ci/templates/dev/dev_vars.yml @@ -0,0 +1,12 @@ +# DEV registry variable bundle. +# +# Anything that uses `extends: .dev_vars` authenticates against the DEV +# registry (via .login_dev_registry) and gets MODULES_MODULE_SOURCE and +# ENV=DEV, all sourced from the DEV_* Project Variables. + +.dev_vars: + extends: + - .login_dev_registry + variables: + MODULES_MODULE_SOURCE: "${DEV_MODULE_SOURCE}" + ENV: DEV diff --git a/.gitlab/ci/templates/dev/main.yml b/.gitlab/ci/templates/dev/main.yml new file mode 100644 index 0000000000..8840d42cb8 --- /dev/null +++ b/.gitlab/ci/templates/dev/main.yml @@ -0,0 +1,16 @@ +# DEV pipeline context: main-branch builds. +# +# Uses the `main` tag, matching GitHub dev_module_build.yml `set_vars` +# (MODULES_MODULE_TAG = github.ref_name = "main" for the default branch, +# see .github/workflows/dev_module_build.yml:121-122). Every main push +# overwrites this tag in the DEV registry, producing a single rolling slot. + +.main: + variables: + MODULES_MODULE_TAG: main + extends: + - .dev_vars + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + - when: never diff --git a/.gitlab/ci/templates/dev/release.yml b/.gitlab/ci/templates/dev/release.yml new file mode 100644 index 0000000000..9be53649a9 --- /dev/null +++ b/.gitlab/ci/templates/dev/release.yml @@ -0,0 +1,21 @@ +# DEV pipeline context: release-branch builds. +# +# Mirrors the GitHub dev_module_build.yml `push: [release-*]` trigger. Code +# reaches release-X.Y branches only through squash-and-merge of an MR, so the +# merge IS the push that must rebuild the dev image for the integrated release +# branch. The image is tagged with the branch name (release-X.Y), matching the +# GitHub set_vars logic; set-vars.sh sets RELEASE_IN_DEV=true for the same +# branches. +# +# Counterpart of .main (default-branch build, tag v0.0.0-main) and .dev (MR +# build, tag mr<iid>). + +.release: + variables: + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + extends: + - .dev_vars + rules: + - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+/' + when: always + - when: never diff --git a/.gitlab/ci/templates/release/prod_tag.yml b/.gitlab/ci/templates/release/prod_tag.yml new file mode 100644 index 0000000000..0ba308aab4 --- /dev/null +++ b/.gitlab/ci/templates/release/prod_tag.yml @@ -0,0 +1,39 @@ +# PROD pipeline context: vX.Y.Z tag-push build & deploy. +# +# .prod_always (build) and .prod_manual (deploy) are identical except for the +# rule's `when:` — build fires automatically once the tag is pushed, each +# channel deploy waits for a human click. They share the common base .prod_tag +# (PROD registry vars + the tag as module tag). +# +# Note: the GH release_module_release-channels.yml inputs (channel, ce/ee, +# tag, enableBuild, release_to_github, check_only, ...) are collapsed here into +# a hardcoded matrix (RELEASE_CHANNEL x EDITION) in deploy-prod.yml. The +# single-channel dispatch that exposes those inputs lives in +# release-channels.yml (prod:* jobs, manual Run pipeline); this template stays +# the tag-push context. + +.prod_tag: + extends: + - .prod_vars + variables: + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + +# Build context: auto on a vX.Y.Z tag push. +.prod_always: + extends: + - .prod_tag + rules: + # https://regex101.com/r/lToOvi/1 + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' + when: always + - when: never + +# Deploy context: same tag match, but each channel deploy is manual. +.prod_manual: + extends: + - .prod_tag + rules: + # https://regex101.com/r/lToOvi/1 + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' + when: manual + - when: never diff --git a/.gitlab/ci/templates/release/prod_vars.yml b/.gitlab/ci/templates/release/prod_vars.yml new file mode 100644 index 0000000000..8b59bc4296 --- /dev/null +++ b/.gitlab/ci/templates/release/prod_vars.yml @@ -0,0 +1,29 @@ +# PROD registry variable bundle. +# +# Anything that uses `extends: .prod_vars` authenticates against the PROD +# registry (via .login_prod_registry) and gets MODULES_MODULE_SOURCE and +# ENV=PROD, all sourced from the PROD_* Project Variables. +# +# MODULES_MODULE_SOURCE is composed dynamically from PROD_REGISTRY, +# PROD_MODULE_SOURCE_NAME, and the matrix-driven EDITION variable so that +# ce/ee/se-plus/fe editions can each deploy into the right subpath without +# duplicating templates. +# +# MODULE_EDITION is NOT set here: it is provided per edition by the +# parallel:matrix in build-prod.yml / deploy-prod.yml (CE for ce, EE for +# ee/se-plus/fe), matching .github/workflows/release_module_build-and-registration.yml. +# .werf/consts.yaml defaults MODULE_EDITION to EE, so it MUST be set by the +# matrix — otherwise the ce prod image is built/deployed as EE. +# +# The read-only registry login used by the prod verification jobs lives in +# base/registry-login.yml as .login_prod_read_registry. + +.prod_vars: + extends: + - .login_prod_registry + # Prod build/deploy jobs must never be auto-cancelled mid-release; override + # the global `interruptible: true` default for every job that extends this. + interruptible: false + variables: + MODULES_MODULE_SOURCE: "${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}/${EDITION}/modules" + ENV: PROD diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml new file mode 100644 index 0000000000..31e30b93a3 --- /dev/null +++ b/.gitlab/ci/variables.yml @@ -0,0 +1,116 @@ +# Pipeline-wide variables. +# +# These values are project defaults. Anything secret (passwords, SSH keys, +# docker configs) lives in GitLab Settings -> CI/CD -> Variables (masked +# where appropriate) and is referenced via ${VAR} expansion below. +# +# Legacy variable migration: +# EXTERNAL_MODULES_DEV_REGISTRY_LOGIN -> DEV_MODULES_REGISTRY_LOGIN +# EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD -> DEV_MODULES_REGISTRY_PASSWORD +# EXTERNAL_MODULES_PROD_REGISTRY_LOGIN -> PROD_MODULES_REGISTRY_LOGIN +# EXTERNAL_MODULES_PROD_REGISTRY_PASSWORD-> PROD_MODULES_REGISTRY_PASSWORD +# Hardcoded registry hosts (`dev-registry.deckhouse.io`, +# `registry-write.deckhouse.io`, `sys/deckhouse-oss/modules`, +# `deckhouse/${EDITION}/modules`) move to DEV_REGISTRY / DEV_MODULE_SOURCE / +# PROD_REGISTRY / PROD_MODULE_SOURCE_NAME vars. +# + +variables: + # --- Module identity --- + MODULE_NAME: virtualization + # Kept for backwards compatibility with the previous root .gitlab-ci.yml; + # upstream templates expect MODULES_MODULE_NAME. + MODULES_MODULE_NAME: virtualization + + # --- Werf / base images / lib-helm --- + # These match upstream modules-gitlab-ci Setup.gitlab-ci.yml defaults. + # Override at the project level once the base-images + lib-helm versions + # are validated against this module's werf config. + BASE_IMAGES_VERSION: v0.5.7 + WERF_VERSION: "2 stable" + # DECKHOUSE_LIB_HELM_VERSION: leave empty here; populate when the lib-helm + # version is pinned in Taskfile.yaml. Upstream Setup.gitlab-ci.yml + # only downloads if this var is non-empty. + DECKHOUSE_LIB_HELM_VERSION: "" + # Disable VEX notice noise from trivy (matches previous GH env). + TRIVY_DISABLE_VEX_NOTICE: "true" + + # --- DEV registry (vars-only, populated via GitLab CI/CD variables) --- + # Required Project Variables (vars, not masked): + # DEV_REGISTRY e.g. dev-registry.deckhouse.io + # DEV_MODULE_SOURCE e.g. dev-registry.deckhouse.io/sys/deckhouse-oss/modules + # DEV_MODULES_REGISTRY_LOGIN + # Required Project Variables (masked): + # DEV_MODULES_REGISTRY_PASSWORD + # + # --- PROD registry (vars-only) --- + # Required Project Variables (vars): + # PROD_REGISTRY e.g. registry-write.deckhouse.io + # PROD_MODULE_SOURCE_NAME e.g. deckhouse + # PROD_MODULES_REGISTRY_LOGIN + # Required Project Variables (masked): + # PROD_MODULES_REGISTRY_PASSWORD + # Optional read-only license registry (used by cve-scan + check:requirements): + # PROD_READ_REGISTRY, PROD_READ_REGISTRY_USER, PROD_READ_REGISTRY_PASSWORD + + # --- CVE scan (Trivy) --- + # Required Project Variable (vars, not masked): + # VAULT_ROLE - role name to authenticate against seguro.flant.com + # via GitLab id_tokens (aud: gitlab-access-aud) + + # d8-cli `stronghold write auth/fox/login`. + # The upstream `.cve_scan` template declares + # VAULT_ROLE with no default; the operator MUST set + # it here or as a Project CI/CD variable, otherwise + # cve:scan:daily / cve:scan:manual / cve:scan:mr + # fail at the Vault login step. + # Value: the same role the GH workflow used, i.e. + # the repository name ("virtualization"). + # VAULT_ADDR - defaults to https://seguro.flant.com upstream; + # override only if the Vault endpoint differs. + + # --- Build/dev tooling (kept from previous root) --- + GO_VERSION: "1.25.11" + GOLANGCI_LINT_VERSION: "2.11.1" + # Helm v3 + jq are needed by `task validation:helm-templates` (tools/kubeconform). + # They are not assumed to be on the runner host; lint:helm-templates installs + # portable binaries from these versions in its before_script. + HELM_VERSION: "3.16.3" + JQ_VERSION: "1.7.1" + # Node.js for test:scripts:js (.gitlab/scripts/js). The runner host has no + # Node.js, so the job runs inside the official node:${NODE_VERSION} image via + # `docker run`. Matches the Node version pinned in the GH workflow + # (.github/workflows/dev_module_build.yml test_scripts_js env NODE_VERSION). + NODE_VERSION: "24" + # Let the host Go toolchain auto-download the version required by go.mod + # (currently go 1.25.11). Runner hosts may ship an older Go with + # GOTOOLCHAIN pinned to that older version (e.g. GOTOOLCHAIN=go1.25.9), + # which otherwise makes `go mod download` / `go install` fail with + # "go.mod requires go >= 1.25.11". `auto` overrides the runner env var + # (GitLab CI variables take precedence over runner environment variables) + # and matches the behavior of GitHub's actions/setup-go@v5. + GOTOOLCHAIN: "auto" + WERF_EXPERIMENTAL_IMPORT_BY_SOURCE_IMAGE_TAG: "true" + + # --- Pipeline schedule discriminator --- + # SCHEDULE_TYPE is NOT a fixed value here. It is set per pipeline schedule + # in GitLab UI (Settings -> CI/CD -> Schedules -> <schedule> -> Variables), + # and each scheduled job gates on its own expected value, e.g.: + # $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cleanup" + # This prevents a job from running under every pipeline schedule that happens + # to fire. Operators must set SCHEDULE_TYPE to exactly one of the values + # below when creating a pipeline schedule: + # + # Job(s) SCHEDULE_TYPE Intended cron + # cleanup cleanup 12 0 * * 6 + # cve:scan:daily cve-scan-daily 0 2 * * * + # svace:* svace 0 4 * * 6 + # gitleaks:full:scheduled gitleaks-full daily + # precache:build:main precache 0 */8 * * * + # changelog:milestone changelog-milestone nightly + # changelog:all-active-milestones changelog-all-active-milestones nightly + # mrs:summary mrs-summary 0 7 * * * + # lint:no-cyrillic / lint:go / lint:gitlab-ci lint-validate as needed + # + # The global workflow:rules in .gitlab/ci/workflow.yml still create a + # pipeline for any schedule; SCHEDULE_TYPE only narrows which jobs run + # inside it. Manual/MR/web behavior is unaffected. diff --git a/.gitlab/ci/workflow.yml b/.gitlab/ci/workflow.yml new file mode 100644 index 0000000000..b8985f0583 --- /dev/null +++ b/.gitlab/ci/workflow.yml @@ -0,0 +1,40 @@ +# Pipeline-level workflow rules. +# +# Intentionally minimal: we do NOT use `workflow:rules:` to gate individual +# jobs — gating is handled per-job via `rules:` against +# $CI_PIPELINE_SOURCE, $CI_COMMIT_BRANCH, $CI_COMMIT_TAG. The global +# workflow rules here only decide whether a pipeline is created at all. +# +# Behaviors preserved from the previous root .gitlab-ci.yml: +# - MR pipelines (push + MR events) are always created. +# - Pushes to main create a pipeline. +# - Pushes of tags vX.Y.Z-dev* / vX.Y.Z create pipelines. +# - Manual/scheduled/web pipelines always run. +# +# Jobs default to `interruptible: true` (see .gitlab/ci/defaults.yml) so a +# new commit cancels the redundant in-flight pipeline once "Auto-cancel +# redundant pipelines" is enabled at the project level. + +workflow: + rules: + # MR pipelines always run. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Default-branch (main) pushes always create a pipeline. Listed before the + # `when: never` dedup rule below so a main push is never skipped. + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # Tag pushes (vX.Y.Z / vX.Y.Z-dev*) always create a pipeline (also matched + # before the dedup rule). + - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(-dev.*)?$/ + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" + # Avoid duplicate pipelines: a push to a non-default, non-tag branch that + # already has an open MR would otherwise create BOTH a branch pipeline and + # an MR pipeline. The predefined $CI_OPEN_MERGE_REQUESTS is non-empty only + # when the pushed branch is the source of an open MR, so THIS rule is what + # skips the redundant branch pipeline (keeping only the MR one). main / tags + # / schedule / web are matched by the earlier rules and never reach here + # (rules are first-match). + - if: $CI_PIPELINE_SOURCE == "push" && $CI_OPEN_MERGE_REQUESTS + when: never + # Plain branch push with no open MR: create the branch pipeline. + - if: $CI_PIPELINE_SOURCE == "push" diff --git a/.gitlab/scripts/js/.gitignore b/.gitlab/scripts/js/.gitignore new file mode 100644 index 0000000000..9111b9b12c --- /dev/null +++ b/.gitlab/scripts/js/.gitignore @@ -0,0 +1,4 @@ +# package-lock.json is ignored globally at the repo root (.gitignore), but the +# one here is intentionally committed so the test:scripts:js CI job can install +# with `npm ci` (reproducible, lockfile-pinned). +!package-lock.json diff --git a/.gitlab/scripts/js/mrs_notifier.mjs b/.gitlab/scripts/js/mrs_notifier.mjs new file mode 100644 index 0000000000..39ce2637f6 --- /dev/null +++ b/.gitlab/scripts/js/mrs_notifier.mjs @@ -0,0 +1,386 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GitLab counterpart of .github/scripts/prs_notifier.mjs. +// +// Reads open MRs from GitLab via REST API, classifies them into +// {ready_to_merge, stuck, changes_requested, review_required}, +// and POSTs a markdown summary to LOOP_WEBHOOK_URL. +// +// Environment: +// GITLAB_API_TOKEN (required) Project Access Token, scope api. +// CI_API_V4_URL (required) e.g. https://fox.flant.com/api/v4 +// CI_PROJECT_ID (required) numeric project id (use the variable, +// not the slug, to survive renames). +// LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. +// DOC_REVIEWER (optional) GitLab username of the doc reviewer (the one +// reviewer kept as a real @-mention). +// Default "gitlab-virt-bot". +// MANAGER_LOOP_NAME (optional) @firstname.lastname of the manager. +// Default "@yuriy.milyutin". +// +// Mapping cheat-sheet: +// octokit -> axios with PRIVATE-TOKEN +// pr.draft -> mr.draft (or mr.work_in_progress for older GitLab) +// pr.head.ref -> mr.source_branch +// pr.labels[].name -> mr.labels[] +// pr.assignees[] -> mr.assignees[] +// pr.requested_reviewers[] -> mr.reviewers[] +// pr.html_url -> mr.web_url +// review state CHANGES_REQUESTED -> unresolved discussion threads. + +import axios from 'axios'; +import moment from 'moment'; +import { fileURLToPath } from 'node:url'; + +const PROJECT_ID = process.env.CI_PROJECT_ID; +const API_BASE = (process.env.CI_API_V4_URL || '').replace(/\/+$/, ''); +const TOKEN = process.env.GITLAB_API_TOKEN; +const LOOP_URL = process.env.LOOP_WEBHOOK_URL; +const DOC_REVIEWER = process.env.DOC_REVIEWER || 'gitlab-virt-bot'; +const MANAGER_LOOP_NAME = process.env.MANAGER_LOOP_NAME || '@yuriy.milyutin'; +const PROJECT = ':dvp: DVP'; + +// STUCK_DAYS mirrors .github/scripts/prs_notifier.mjs: a changes-requested +// discussion is considered "stuck" once it stays unresolved for longer than +// this many days (and today is not Monday, see Monday exception below). +const STUCK_DAYS = 1.5; + +const api = axios.create({ + baseURL: API_BASE, + headers: { + 'PRIVATE-TOKEN': TOKEN, + 'Accept': 'application/json', + }, +}); + +const CHANGES_REQUESTED = 'changes_requested'; +const REVIEW_REQUIRED = 'review_required'; +const READY_TO_MERGE = 'ready_to_merge'; +const STUCK = 'stuck'; + +function validateEnv() { + if (!API_BASE || !TOKEN || !PROJECT_ID || !LOOP_URL) { + console.error('ERROR: one of CI_API_V4_URL, GITLAB_API_TOKEN, CI_PROJECT_ID, LOOP_WEBHOOK_URL is not set.'); + process.exit(1); + } +} + +// Pure predicate: decide whether an open MR belongs in the review summary. +// Excludes drafts/WIP, release-* source branches, and autorelease/changelog +// bot MRs (these are not human review work). Exported for unit testing. +export function shouldNotifyMR(mr) { + if (mr.draft || mr.work_in_progress) return false; + const head = (mr.source_branch || '').toLowerCase(); + if (head.startsWith('release-')) return false; + const labels = (mr.labels || []).map((l) => l.toLowerCase()); + if (labels.some((l) => l.startsWith('autorelease'))) return false; + if (labels.includes('changelog')) return false; + return true; +} + +async function fetchOpenMRs() { + const { data } = await api.get(`/projects/${PROJECT_ID}/merge_requests`, { + params: { + state: 'opened', + per_page: 100, + order_by: 'created_at', + sort: 'asc', + }, + }); + return data.filter(shouldNotifyMR); +} + +async function fetchUser(id) { + if (!id) return null; + try { + const { data } = await api.get(`/users/${id}`); + return data; + } catch (err) { + console.error(`Error fetching user ${id}: ${err.message}`); + return null; + } +} + +// Pure helper: render a Loop mention for a user. Prefers the profile name +// rendered as @first.last; falls back to the username with a nudge to set a +// real name. Exported for unit testing. +export function formatUser(user, details) { + if (!user) return 'unknown'; + if (details && details.name) { + const loopName = details.name.replace(/ /g, '.').toLowerCase(); + if (loopName.length > 0) return `@${loopName}`; + } + return `${user.username || user.login} (Set name in profile!)`; +} + +async function getAssigneesInfo(mr) { + let info = `NO ASSIGNEES! ${MANAGER_LOOP_NAME} (opezdulit')`; + const assignees = mr.assignees || []; + if (assignees.length > 0) { + const names = []; + for (const a of assignees) { + const details = await fetchUser(a.id); + names.push(formatUser(a, details)); + } + info = `Assignees: ${names.join(', ')}`; + } + return info; +} + +async function getReviewersInfo(mr) { + let info = `NO REVIEWERS! ${MANAGER_LOOP_NAME} (opezdulit')`; + const requestedReviewers = mr.reviewers || []; + const unique = new Set(); + const fetched = []; + + for (const reviewer of requestedReviewers) { + unique.add(reviewer.id); + const details = await fetchUser(reviewer.id); + let user = formatUser(reviewer, details); + // Match GitHub behaviour: keep @ only for DOC_REVIEWER (legacy quirk). + if (DOC_REVIEWER !== reviewer.username) { + user = user.replace(/@/g, ''); + } + fetched.push(user); + } + + const threads = await fetchUnresolvedThreads(mr); + for (const thread of threads) { + const reviewer = thread.author; + if (unique.has(reviewer.id)) continue; + unique.add(reviewer.id); + const details = await fetchUser(reviewer.id); + let user = formatUser(reviewer, details); + if (DOC_REVIEWER !== reviewer.username) { + user = user.replace(/@/g, ''); + } + fetched.push(user); + } + + if (fetched.length > 0) { + info = `Reviewers: ${fetched.join(', ')}`; + } + return info; +} + +// Pure helper: extract unresolved discussion threads from raw GitLab +// discussions payload. Returns an array of { author, timestamp } where +// timestamp (ms epoch) is the earliest note timestamp belonging to the +// unresolved discussion, or null when no real timestamp is available. +// +// A thread is considered unresolved when at least one of its notes is +// `resolvable` and not `resolved` (GitLab marks resolution at note level). +// Exported so unit tests can exercise it without network access. +export function extractUnresolvedThreads(discussions) { + const threads = []; + for (const discussion of discussions || []) { + const notes = discussion.notes || []; + if (!notes.length) continue; + const unresolvedNotes = notes.filter((n) => n.resolvable && !n.resolved); + if (!unresolvedNotes.length) continue; + const author = unresolvedNotes[0].author || notes[0].author; + if (!author) continue; + + // Prefer the earliest timestamp among the unresolved resolvable notes; + // fall back to the earliest note in the thread if GitLab only exposes + // note-level resolution metadata inconsistently. + const tsNotes = unresolvedNotes.some((n) => n.created_at) + ? unresolvedNotes.filter((n) => n.created_at) + : notes.filter((n) => n.created_at); + + let timestamp = null; + if (tsNotes.length > 0) { + timestamp = tsNotes + .map((n) => new Date(n.created_at).getTime()) + .reduce((a, b) => (a < b ? a : b), Infinity); + if (!Number.isFinite(timestamp)) timestamp = null; + } + + threads.push({ author, timestamp }); + } + return threads; +} + +// Fetch unresolved resolvable discussion threads for an MR, including the +// real note timestamp used for STUCK classification. +async function fetchUnresolvedThreads(mr) { + try { + const { data } = await api.get( + `/projects/${PROJECT_ID}/merge_requests/${mr.iid}/discussions`, + { params: { per_page: 100 } }, + ); + return extractUnresolvedThreads(data); + } catch (err) { + console.error(`Error fetching discussions for MR !${mr.iid}: ${err.message}`); + return []; + } +} + +// Back-compat wrapper: deduplicated unresolved authors, oldest thread per +// author preserved. Kept for any external caller and for symmetry with the +// GitHub changesRequestedMap semantics. +async function fetchUnresolvedReviewers(mr) { + const threads = await fetchUnresolvedThreads(mr); + const byAuthor = new Map(); + for (const thread of threads) { + const id = thread.author.id; + if (!byAuthor.has(id)) { + byAuthor.set(id, thread); + continue; + } + const existing = byAuthor.get(id); + if ( + existing.timestamp == null + || (thread.timestamp != null && thread.timestamp < existing.timestamp) + ) { + byAuthor.set(id, thread); + } + } + return [...byAuthor.values()].map((t) => t.author); +} + +async function fetchApprovals(mr) { + try { + const { data } = await api.get( + `/projects/${PROJECT_ID}/merge_requests/${mr.iid}/approvals`, + ); + return (data.approved_by || []).map((entry) => entry.user); + } catch (err) { + console.error(`Error fetching approvals for MR !${mr.iid}: ${err.message}`); + return []; + } +} + +// Classify an MR given its approvers and unresolved discussion threads. +// +// Mirrors .github/scripts/prs_notifier.mjs getPullRequestGroup: +// - STUCK when at least one unresolved discussion is older than STUCK_DAYS +// and today is not Monday (Monday exception: give reviewers a fresh +// start-of-week window). +// - CHANGES_REQUESTED when there are unresolved discussions but none is +// old enough (or a thread has no real timestamp — be conservative and +// avoid false STUCK). +// - READY_TO_MERGE when approved and no unresolved discussions. +// - REVIEW_REQUIRED otherwise. +// +// Exported for unit testing. +export function classifyMR(approvedBy, unresolvedThreads) { + const approved = approvedBy.length > 0; + const unresolved = unresolvedThreads.length > 0; + + if (unresolved) { + const now = new Date(); + let areChangesRequested = false; + for (const thread of unresolvedThreads) { + // No real timestamp -> cannot prove it is old enough. Be conservative: + // treat as changes_requested, never STUCK. + if (thread.timestamp == null) { + areChangesRequested = true; + continue; + } + const submittedAt = new Date(thread.timestamp); + submittedAt.setTime(submittedAt.getTime() + STUCK_DAYS * 24 * 60 * 60 * 1000); + if (now.getDay() !== 1 && submittedAt < now) { + return STUCK; + } + areChangesRequested = true; + } + if (areChangesRequested) return CHANGES_REQUESTED; + } + + if (approved) return READY_TO_MERGE; + return REVIEW_REQUIRED; +} + +async function buildSummary(mrs) { + const groups = { + [READY_TO_MERGE]: [], + [STUCK]: [], + [CHANGES_REQUESTED]: [], + [REVIEW_REQUIRED]: [], + }; + + for (const mr of mrs) { + const [approvals, unresolvedThreads] = await Promise.all([ + fetchApprovals(mr), + fetchUnresolvedThreads(mr), + ]); + const group = classifyMR(approvals, unresolvedThreads); + groups[group].push(mr); + } + + const today = moment().format('YYYY-MM-DD'); + let summary = `## ${PROJECT} MRs ${today}\n\n`; + if (mrs.length === 0) { + summary += `:tada: No review required for today\n`; + return summary; + } + + if (groups[READY_TO_MERGE].length) { + const lines = await Promise.all(groups[READY_TO_MERGE].map(formatAssigneeLine)); + summary += `### Ready to be merged\nWhy haven't they been merged yet? :thinking_face:\n\n${lines.join('\n')}\n\n`; + } + if (groups[STUCK].length) { + const lines = await Promise.all(groups[STUCK].map(formatAssigneeLine)); + summary += `### Stuck in resolution\nWhy is there no resolution for the requested changes? :large_red_square:\n\n${lines.join('\n')}\n\n`; + } + if (groups[CHANGES_REQUESTED].length) { + const lines = await Promise.all(groups[CHANGES_REQUESTED].map(formatAssigneeLine)); + summary += `### Changes requested\nMRs have the highest priority for comments to be resolved :fire:\n\n${lines.join('\n')}\n\n`; + } + if (groups[REVIEW_REQUIRED].length) { + const lines = await Promise.all(groups[REVIEW_REQUIRED].map(formatReviewerLine)); + summary += `### MRs requiring review\n\n${lines.join('\n')}\n`; + } + return summary; +} + +async function formatAssigneeLine(mr) { + const assignees = await getAssigneesInfo(mr); + return `- !${mr.iid}: [${mr.title}](${mr.web_url}) (created: ${moment(mr.created_at).fromNow()}) - ${assignees}`; +} + +async function formatReviewerLine(mr) { + const assignees = await getAssigneesInfo(mr); + const reviewers = await getReviewersInfo(mr); + return `- !${mr.iid}: [${mr.title}](${mr.web_url}) (created: ${moment(mr.created_at).fromNow()}) - ${assignees}. ${reviewers}`; +} + +async function sendSummaryToLoop(summary) { + try { + await axios.post(LOOP_URL, { text: summary }); + console.log('Summary sent successfully.'); + } catch (err) { + console.error(`Error sending summary to Loop: ${err.message}`); + throw err; + } +} + +async function run() { + validateEnv(); + try { + const mrs = await fetchOpenMRs(); + const summary = await buildSummary(mrs); + await sendSummaryToLoop(summary); + } catch (err) { + console.error(`An error occurred: ${err.message}`); + process.exit(1); + } +} + +// Auto-run only when executed directly (CI), not when imported by tests. +if (process.argv[1] === fileURLToPath(import.meta.url)) { + run(); +} diff --git a/.gitlab/scripts/js/mrs_notifier.test.mjs b/.gitlab/scripts/js/mrs_notifier.test.mjs new file mode 100644 index 0000000000..8ecf6b30e5 --- /dev/null +++ b/.gitlab/scripts/js/mrs_notifier.test.mjs @@ -0,0 +1,306 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit tests for mrs_notifier.mjs. +// +// mrs_notifier.mjs guards its `run()` call behind a direct-invocation check +// and validates env vars only inside `run()`, so it can be safely imported +// here without triggering network calls or process.exit. The pure helpers +// `classifyMR`, `extractUnresolvedThreads`, `shouldNotifyMR`, and `formatUser` +// are exercised directly with synthetic data — no network access required. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, statSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +import { + classifyMR, + extractUnresolvedThreads, + shouldNotifyMR, + formatUser, +} from './mrs_notifier.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const MODULE_PATH = path.join(__dirname, 'mrs_notifier.mjs'); + +const STUCK_DAYS = 1.5; +const DAY_MS = 24 * 60 * 60 * 1000; + +function author(id, username) { + return { id, username, name: username }; +} + +function note({ author: a, createdAt, resolvable = true, resolved = false }) { + return { + author: a, + created_at: createdAt, + resolvable, + resolved, + }; +} + +// An ISO timestamp `days` days ago from now. +function daysAgo(days) { + return new Date(Date.now() - days * DAY_MS).toISOString(); +} + +// ---- smoke / structural tests (kept from the original test file) ---- + +test('mrs_notifier.mjs exists and is non-empty', () => { + const stat = statSync(MODULE_PATH); + assert.ok(stat.isFile(), 'mrs_notifier.mjs should be a file'); + const src = readFileSync(MODULE_PATH, 'utf8'); + assert.ok(src.trim().length > 0, 'mrs_notifier.mjs should not be empty'); +}); + +test('mrs_notifier.mjs is syntactically valid (node --check)', () => { + execFileSync('node', ['--check', MODULE_PATH], { stdio: 'pipe' }); +}); + +test('mrs_notifier.mjs declares the expected entry points', () => { + const src = readFileSync(MODULE_PATH, 'utf8'); + for (const name of ['fetchOpenMRs', 'classifyMR', 'buildSummary', 'run']) { + assert.ok( + src.includes(`function ${name}`) || src.includes(`${name}(`), + `expected function ${name} to be defined`, + ); + } +}); + +test('mrs_notifier.mjs references the documented env-var contract', () => { + const src = readFileSync(MODULE_PATH, 'utf8'); + for (const envVar of ['GITLAB_API_TOKEN', 'CI_API_V4_URL', 'CI_PROJECT_ID', 'LOOP_WEBHOOK_URL']) { + assert.ok(src.includes(envVar), `expected reference to ${envVar}`); + } +}); + +test('mrs_notifier.mjs does not fabricate a fake old date in classifyMR', () => { + const src = readFileSync(MODULE_PATH, 'utf8'); + assert.ok( + !/fakeDate/.test(src), + 'classifyMR must not use a fabricated fake date', + ); +}); + +// ---- classifyMR behavior ---- + +test('classifyMR: no unresolved, approved => ready_to_merge', () => { + const approved = [author(1, 'alice')]; + assert.equal(classifyMR(approved, []), 'ready_to_merge'); +}); + +test('classifyMR: no unresolved, no approved => review_required', () => { + assert.equal(classifyMR([], []), 'review_required'); +}); + +test('classifyMR: recent unresolved => changes_requested', () => { + const threads = [{ author: author(2, 'bob'), timestamp: Date.now() }]; + assert.equal(classifyMR([], threads), 'changes_requested'); +}); + +test('classifyMR: recent unresolved with approvals still changes_requested', () => { + // Unresolved discussions take precedence over approvals, matching GitHub. + const approved = [author(1, 'alice')]; + const threads = [{ author: author(2, 'bob'), timestamp: Date.now() }]; + assert.equal(classifyMR(approved, threads), 'changes_requested'); +}); + +test('classifyMR: old unresolved => stuck', () => { + const threads = [ + { author: author(2, 'bob'), timestamp: Date.now() - (STUCK_DAYS + 1) * DAY_MS }, + ]; + // Only assert STUCK when today is not Monday; on Monday the exception + // kicks in and the result is changes_requested. + if (new Date().getDay() !== 1) { + assert.equal(classifyMR([], threads), 'stuck'); + } else { + assert.equal(classifyMR([], threads), 'changes_requested'); + } +}); + +test('classifyMR: exactly STUCK_DAYS boundary is not stuck yet', () => { + // timestamp + STUCK_DAYS == now => submittedAt < now is false => not stuck. + const threads = [ + { author: author(2, 'bob'), timestamp: Date.now() - STUCK_DAYS * DAY_MS + 1000 }, + ]; + assert.equal(classifyMR([], threads), 'changes_requested'); +}); + +test('classifyMR: no timestamp => changes_requested (never stuck)', () => { + const threads = [{ author: author(2, 'bob'), timestamp: null }]; + assert.equal(classifyMR([], threads), 'changes_requested'); +}); + +test('classifyMR: one old + one no-timestamp thread => stuck (when not Monday)', () => { + const threads = [ + { author: author(2, 'bob'), timestamp: null }, + { author: author(3, 'carol'), timestamp: Date.now() - (STUCK_DAYS + 2) * DAY_MS }, + ]; + if (new Date().getDay() !== 1) { + assert.equal(classifyMR([], threads), 'stuck'); + } else { + assert.equal(classifyMR([], threads), 'changes_requested'); + } +}); + +// ---- extractUnresolvedThreads ---- + +test('extractUnresolvedThreads: skips fully resolved threads', () => { + const discussions = [ + { + notes: [ + note({ author: author(2, 'bob'), createdAt: daysAgo(5), resolved: true }), + ], + }, + ]; + assert.deepEqual(extractUnresolvedThreads(discussions), []); +}); + +test('extractUnresolvedThreads: skips non-resolvable threads', () => { + const discussions = [ + { + notes: [ + note({ author: author(2, 'bob'), createdAt: daysAgo(5), resolvable: false }), + ], + }, + ]; + assert.deepEqual(extractUnresolvedThreads(discussions), []); +}); + +test('extractUnresolvedThreads: returns earliest unresolved note timestamp', () => { + const old = daysAgo(5); + const recent = daysAgo(1); + const discussions = [ + { + notes: [ + note({ author: author(2, 'bob'), createdAt: recent, resolved: false }), + note({ author: author(2, 'bob'), createdAt: old, resolved: false }), + ], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + assert.equal(threads.length, 1); + assert.equal(threads[0].author.id, 2); + assert.equal(threads[0].timestamp, new Date(old).getTime()); +}); + +test('extractUnresolvedThreads: prefers unresolved note timestamp over resolved ones', () => { + const resolvedOld = daysAgo(10); + const unresolvedRecent = daysAgo(1); + const discussions = [ + { + notes: [ + note({ author: author(1, 'alice'), createdAt: resolvedOld, resolved: true }), + note({ author: author(2, 'bob'), createdAt: unresolvedRecent, resolved: false }), + ], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + assert.equal(threads.length, 1); + assert.equal(threads[0].author.id, 2); + assert.equal(threads[0].timestamp, new Date(unresolvedRecent).getTime()); +}); + +test('extractUnresolvedThreads: null timestamp when no created_at present', () => { + const discussions = [ + { + notes: [note({ author: author(2, 'bob'), createdAt: undefined, resolved: false })], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + assert.equal(threads.length, 1); + assert.equal(threads[0].timestamp, null); +}); + +test('extractUnresolvedThreads: empty / null input is safe', () => { + assert.deepEqual(extractUnresolvedThreads([]), []); + assert.deepEqual(extractUnresolvedThreads(null), []); + assert.deepEqual(extractUnresolvedThreads(undefined), []); +}); + +test('extractUnresolvedThreads + classifyMR integration: old unresolved => stuck', () => { + const discussions = [ + { + notes: [ + note({ + author: author(2, 'bob'), + createdAt: daysAgo(STUCK_DAYS + 1), + resolved: false, + }), + ], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + if (new Date().getDay() !== 1) { + assert.equal(classifyMR([], threads), 'stuck'); + } else { + assert.equal(classifyMR([], threads), 'changes_requested'); + } +}); + +// ---- shouldNotifyMR (open-MR filter) ---- + +test('shouldNotifyMR: plain open MR is included', () => { + assert.equal(shouldNotifyMR({ source_branch: 'feature/x', labels: [] }), true); +}); + +test('shouldNotifyMR: drafts and WIP are excluded', () => { + assert.equal(shouldNotifyMR({ draft: true, source_branch: 'x' }), false); + assert.equal(shouldNotifyMR({ work_in_progress: true, source_branch: 'x' }), false); +}); + +test('shouldNotifyMR: release-* source branches are excluded (case-insensitive)', () => { + assert.equal(shouldNotifyMR({ source_branch: 'release-1.2' }), false); + assert.equal(shouldNotifyMR({ source_branch: 'Release-1.2' }), false); +}); + +test('shouldNotifyMR: autorelease and changelog bot MRs are excluded', () => { + assert.equal(shouldNotifyMR({ source_branch: 'x', labels: ['autorelease'] }), false); + assert.equal(shouldNotifyMR({ source_branch: 'x', labels: ['Autorelease/v1'] }), false); + assert.equal(shouldNotifyMR({ source_branch: 'x', labels: ['changelog'] }), false); +}); + +test('shouldNotifyMR: missing source_branch/labels is safe', () => { + assert.equal(shouldNotifyMR({}), true); +}); + +// ---- formatUser (Loop mention rendering) ---- + +test('formatUser: null user => unknown', () => { + assert.equal(formatUser(null, null), 'unknown'); +}); + +test('formatUser: profile name rendered as @first.last lowercase', () => { + assert.equal( + formatUser({ username: 'jdoe' }, { name: 'John Doe' }), + '@john.doe', + ); +}); + +test('formatUser: no profile name => username with nudge', () => { + assert.equal( + formatUser({ username: 'jdoe' }, null), + 'jdoe (Set name in profile!)', + ); +}); + +test('formatUser: falls back to login when username absent', () => { + assert.equal( + formatUser({ login: 'ghuser' }, {}), + 'ghuser (Set name in profile!)', + ); +}); diff --git a/.gitlab/scripts/js/package-lock.json b/.gitlab/scripts/js/package-lock.json new file mode 100644 index 0000000000..f3e1fd430d --- /dev/null +++ b/.gitlab/scripts/js/package-lock.json @@ -0,0 +1,357 @@ +{ + "name": "mrs-notifier-gitlab", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mrs-notifier-gitlab", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "axios": "^1.7.7", + "moment": "^2.30.1" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz", + "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.5", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + } + } +} diff --git a/.gitlab/scripts/js/package.json b/.gitlab/scripts/js/package.json new file mode 100644 index 0000000000..ba51d40bbc --- /dev/null +++ b/.gitlab/scripts/js/package.json @@ -0,0 +1,18 @@ +{ + "name": "mrs-notifier-gitlab", + "version": "1.0.0", + "private": true, + "description": "GitLab counterpart of .github/scripts/prs_notifier.mjs", + "type": "module", + "main": "mrs_notifier.mjs", + "scripts": { + "start": "node mrs_notifier.mjs", + "lint": "eslint mrs_notifier.mjs", + "test": "node --test mrs_notifier.test.mjs" + }, + "dependencies": { + "axios": "^1.7.7", + "moment": "^2.30.1" + }, + "license": "Apache-2.0" +} diff --git a/Taskfile.yaml b/Taskfile.yaml index cc83d74680..501eca036f 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -188,16 +188,30 @@ tasks: desc: "Run shellcheck for CI shell scripts." cmds: - | + set -e + # The .gitlab/ci/scripts tree is split into bash/ and python/ folders, + # so lint:shellcheck points directly at the bash folder (no find scan). + EXPLICIT=".github/scripts/bash/e2e/*.sh + .gitlab/ci/scripts/bash/*.sh + .gitlab/ci/scripts/bash/lib/*.sh + api/scripts/update-codegen.sh + images/virtualization-artifact/hack/args.sh + images/virtualization-artifact/hack/dlv.sh + images/virtualization-artifact/hack/pyroscope.sh" + EXPANDED="$(for f in $EXPLICIT; do [ -f "$f" ] && echo "$f"; done | sort -u)" + if [ -z "$EXPANDED" ]; then + echo "No shell scripts found to lint."; exit 0 + fi + echo "==> shellcheck will scan $(printf '%s\n' "$EXPANDED" | wc -l | tr -d ' ') files:" + printf ' %s\n' $EXPANDED + echo "==> running shellcheck (koalaman/shellcheck-alpine:v0.11.0)" docker run --rm \ -v "$PWD:/mnt" \ -w /mnt \ - koalaman/shellcheck-alpine:v0.10.0 \ + koalaman/shellcheck-alpine:v0.11.0 \ shellcheck \ - .github/scripts/bash/e2e/*.sh \ - api/scripts/update-codegen.sh \ - images/virtualization-artifact/hack/args.sh \ - images/virtualization-artifact/hack/dlv.sh \ - images/virtualization-artifact/hack/pyroscope.sh + $EXPANDED + echo "==> shellcheck: no issues found" lint:actionlint: desc: "Run actionlint for E2E GitHub workflows." diff --git a/images/virtualization-artifact/Taskfile.init.yaml b/images/virtualization-artifact/Taskfile.init.yaml index 73cbf043b4..c702352320 100644 --- a/images/virtualization-artifact/Taskfile.init.yaml +++ b/images/virtualization-artifact/Taskfile.init.yaml @@ -9,7 +9,10 @@ tasks: golangci-lint: cmds: - - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 + # Install golangci-lint v2. Keep in sync with GOLANGCI_LINT_VERSION in + # .gitlab/ci/variables.yml and GOLANGCI_LINT_MIN_VERSION in the + # _ensure:golangci-lint tasks. v2 lives under the /v2 module path. + - go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.11.1 ginkgo: cmds: diff --git a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go index 8f5464236b..11ec2949a3 100644 --- a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go +++ b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go @@ -54,9 +54,10 @@ func loadEbpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { type ebpfSpecs struct { ebpfProgramSpecs ebpfMapSpecs + ebpfVariableSpecs } -// ebpfSpecs contains programs before they are loaded into the kernel. +// ebpfProgramSpecs contains programs before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type ebpfProgramSpecs struct { @@ -71,12 +72,20 @@ type ebpfMapSpecs struct { RouteEventsMap *ebpf.MapSpec `ebpf:"route_events_map"` } +// ebpfVariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type ebpfVariableSpecs struct { + Unused *ebpf.VariableSpec `ebpf:"unused"` +} + // ebpfObjects contains all objects after they have been loaded into the kernel. // // It can be passed to loadEbpfObjects or ebpf.CollectionSpec.LoadAndAssign. type ebpfObjects struct { ebpfPrograms ebpfMaps + ebpfVariables } func (o *ebpfObjects) Close() error { @@ -99,6 +108,13 @@ func (m *ebpfMaps) Close() error { ) } +// ebpfVariables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to loadEbpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type ebpfVariables struct { + Unused *ebpf.Variable `ebpf:"unused"` +} + // ebpfPrograms contains all programs after they have been loaded into the kernel. // // It can be passed to loadEbpfObjects or ebpf.CollectionSpec.LoadAndAssign. diff --git a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.o b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.o index 1adcd61bd2..a9d7a95041 100644 Binary files a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.o and b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.o differ