From 52478b7431ddcbf7473ec5f432f87d1af472ebf3 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 22 Jun 2026 19:32:36 +0300 Subject: [PATCH 01/60] feat(ci): add gitlab ci core pipeline structure Signed-off-by: Nikita Korolev --- .gitlab-ci.yml | 418 +------------------ .gitlab/ci/defaults.yml | 24 ++ .gitlab/ci/includes.yml | 75 ++++ .gitlab/ci/jobs/build-dev.yml | 43 ++ .gitlab/ci/jobs/build-prod.yml | 42 ++ .gitlab/ci/jobs/cleanup.yml | 29 ++ .gitlab/ci/jobs/deploy-dev.yml | 32 ++ .gitlab/ci/jobs/deploy-prod.yml | 106 +++++ .gitlab/ci/jobs/info.yml | 21 + .gitlab/ci/jobs/test.yml | 38 ++ .gitlab/ci/scripts/lib/api.sh | 100 +++++ .gitlab/ci/scripts/set-vars.sh | 111 +++++ .gitlab/ci/stages.yml | 41 ++ .gitlab/ci/templates/build.yml | 65 +++ .gitlab/ci/templates/deploy.yml | 30 ++ .gitlab/ci/templates/dev.yml | 19 + .gitlab/ci/templates/dev_tags.yml | 20 + .gitlab/ci/templates/dev_vars.yml | 24 ++ .gitlab/ci/templates/dual_registry_login.yml | 31 ++ .gitlab/ci/templates/info.yml | 50 +++ .gitlab/ci/templates/main.yml | 15 + .gitlab/ci/templates/prod_always.yml | 17 + .gitlab/ci/templates/prod_manual.yml | 23 + .gitlab/ci/templates/prod_vars.yml | 23 + .gitlab/ci/variables.yml | 60 +++ .gitlab/ci/workflow.yml | 26 ++ 26 files changed, 1082 insertions(+), 401 deletions(-) create mode 100644 .gitlab/ci/defaults.yml create mode 100644 .gitlab/ci/includes.yml create mode 100644 .gitlab/ci/jobs/build-dev.yml create mode 100644 .gitlab/ci/jobs/build-prod.yml create mode 100644 .gitlab/ci/jobs/cleanup.yml create mode 100644 .gitlab/ci/jobs/deploy-dev.yml create mode 100644 .gitlab/ci/jobs/deploy-prod.yml create mode 100644 .gitlab/ci/jobs/info.yml create mode 100644 .gitlab/ci/jobs/test.yml create mode 100755 .gitlab/ci/scripts/lib/api.sh create mode 100755 .gitlab/ci/scripts/set-vars.sh create mode 100644 .gitlab/ci/stages.yml create mode 100644 .gitlab/ci/templates/build.yml create mode 100644 .gitlab/ci/templates/deploy.yml create mode 100644 .gitlab/ci/templates/dev.yml create mode 100644 .gitlab/ci/templates/dev_tags.yml create mode 100644 .gitlab/ci/templates/dev_vars.yml create mode 100644 .gitlab/ci/templates/dual_registry_login.yml create mode 100644 .gitlab/ci/templates/info.yml create mode 100644 .gitlab/ci/templates/main.yml create mode 100644 .gitlab/ci/templates/prod_always.yml create mode 100644 .gitlab/ci/templates/prod_manual.yml create mode 100644 .gitlab/ci/templates/prod_vars.yml create mode 100644 .gitlab/ci/variables.yml create mode 100644 .gitlab/ci/workflow.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ad3e5a2df..7e3688cfc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,401 +1,17 @@ -stages: - - info - - lint - - test - - build - - e2e - - deploy_dev_tags - - deploy_prod_alpha - - deploy_prod_beta - - deploy_prod_ea - - deploy_prod_stable - - deploy_prod_rock_solid - - cleanup - -default: - tags: - - deckhouse - -.build: - stage: build - script: - # Build images - - | - werf build \ - --repo=${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} \ - --save-build-report --build-report-path images_tags_werf.json - # Bundle image - - | - IMAGE_SRC="$(jq -r '.Images."bundle".DockerImageName' images_tags_werf.json)" - IMAGE_DST="$(jq -r '.Images.bundle.DockerRepo' images_tags_werf.json):${MODULES_MODULE_TAG}" - - echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" - crane copy ${IMAGE_SRC} ${IMAGE_DST} - # Release-channel image - - | - IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)" - IMAGE_DST="$(jq -r '.Images."release-channel-version".DockerRepo' images_tags_werf.json)/release:${MODULES_MODULE_TAG}" - - echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" - crane copy ${IMAGE_SRC} ${IMAGE_DST} - # Register module - - | - echo "✨ Register the module ${MODULES_MODULE_NAME}" - crane append \ - --oci-empty-base \ - --new_layer "" \ - --new_tag "${MODULES_MODULE_SOURCE}:${MODULES_MODULE_NAME}" - -.deploy: - stage: deploy - script: - - | - REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" - - IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" - IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" - - echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" - crane copy "${IMAGE_SRC}" "${IMAGE_DST}" - - -.info: - script: - - | - cat << OUTER - Create ModuleConfig and ModulePullOverride resources to test this MR: - cat < EN MR) +# - Merge_Release.gitlab-ci.yml (release-label driven merge+tag+release) +# - Svace_Analayze.gitlab-ci.yml (note: typo in upstream file name) +# - Antivirus_Scan.gitlab-ci.yml (antivirus scan over recent releases) +# - Semgrep.gitlab-ci.yml (semgrep SAST) +# +# We only include the templates whose behavior is consumed by jobs owned by +# this issue (Setup, Build, Deploy). The remaining upstream templates are +# surfaced here for visibility — child issues that own cve-scan.yml, +# gitleaks.yml, etc. add the matching `include: local:` entries for the +# corresponding job files. + +include: + # --- Upstream modules-gitlab-ci (deckhouse/3p, ref v13.0) --- + # TODO: pin to SHA after first green pipeline on virt-test; see migration + # plan §11.2. Until then, branch ref v13.0 keeps fixes flowing. + - project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' + ref: 'v13.0' + file: + # Setup.gitlab-ci.yml provides trdl + werf setup + dual-registry + # `werf cr login` in before_script. See templates/Setup.gitlab-ci.yml. + - '/templates/Setup.gitlab-ci.yml' + # Build.gitlab-ci.yml and Deploy.gitlab-ci.yml are intentionally NOT + # included directly. Their hidden jobs (.build, .deploy) ship with + # `rules:` baked in (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`, + # `when: manual`) that override our strict gating via + # .dev / .dev_tags / .main / .prod_manual. We mirror the upstream + # script bodies in .gitlab/ci/templates/{build,deploy}.yml as + # `.local_build` and `.local_deploy` and extend those instead. + + # --- Local structural fragments (order: stages, vars, defaults, then jobs) --- + - local: '.gitlab/ci/stages.yml' + - local: '.gitlab/ci/variables.yml' + - local: '.gitlab/ci/defaults.yml' + - local: '.gitlab/ci/workflow.yml' + + # --- Local shared templates (extends: ...) --- + - local: '.gitlab/ci/templates/dev_vars.yml' + - local: '.gitlab/ci/templates/prod_vars.yml' + - local: '.gitlab/ci/templates/dev.yml' + - local: '.gitlab/ci/templates/dev_tags.yml' + - local: '.gitlab/ci/templates/main.yml' + - local: '.gitlab/ci/templates/prod_manual.yml' + - local: '.gitlab/ci/templates/prod_always.yml' + - local: '.gitlab/ci/templates/info.yml' + - local: '.gitlab/ci/templates/dual_registry_login.yml' + - local: '.gitlab/ci/templates/build.yml' + - local: '.gitlab/ci/templates/deploy.yml' + + # --- Local job files owned by this issue --- + - local: '.gitlab/ci/jobs/info.yml' + - local: '.gitlab/ci/jobs/test.yml' + - local: '.gitlab/ci/jobs/build-dev.yml' + - local: '.gitlab/ci/jobs/build-prod.yml' + - local: '.gitlab/ci/jobs/deploy-dev.yml' + - local: '.gitlab/ci/jobs/deploy-prod.yml' + - local: '.gitlab/ci/jobs/cleanup.yml' diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml new file mode 100644 index 0000000000..68abdf8f8d --- /dev/null +++ b/.gitlab/ci/jobs/build-dev.yml @@ -0,0 +1,43 @@ +# DEV build jobs. +# +# Carries forward build_dev, build_dev_tags and build_main from the +# previous root .gitlab-ci.yml. All three extend: +# - .local_build (this repo) — same body as upstream modules-gitlab-ci +# Build.gitlab-ci.yml `.build` (werf build + bundle crane +# copy + release-channel crane copy + crane append to +# register module), but without the upstream `rules:` +# that would override our strict gating via .dev / +# .dev_tags / .main. +# - .dev / .dev_tags / .main (this repo) for registry + tag context. +# +# The previous root .gitlab-ci.yml had its own `.build` template with the +# same body; it is replaced here by `.local_build` (verified against +# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci +# templates/Build.gitlab-ci.yml — werf build with --save-build-report, +# bundle / release-channel-version crane copy, and crane append to +# register the module). The upstream version uses WERF_REPO env instead of +# an explicit --repo flag, which is equivalent because MODULES_MODULE_SOURCE +# is the same in both cases. +# +# build_main keeps `interruptible: true` so a new main push cancels an +# older main pipeline. build_dev and build_dev_tags inherit the project +# default (interruptible not set, so they can be cancelled manually). + +build_dev: + stage: build + extends: + - .local_build + - .dev + +build_dev_tags: + stage: build + extends: + - .local_build + - .dev_tags + +build_main: + stage: build + interruptible: true + extends: + - .local_build + - .main diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml new file mode 100644 index 0000000000..52e94a8f85 --- /dev/null +++ b/.gitlab/ci/jobs/build-prod.yml @@ -0,0 +1,42 @@ +# PROD build job. +# +# Carries forward build_prod from the previous root .gitlab-ci.yml. Builds +# the module images for every edition on tag push (vX.Y.Z), then leaves +# the deploy step to .gitlab/ci/jobs/deploy-prod.yml. +# +# Editions: ce / ee / fe today. The previous GH workflow +# release_module_build-and-registration.yml also had an `se-plus` edition; +# the migration plan flags this as a TODO (plan §7, question about se-plus +# edition parity). Until that decision is made, we keep the ce/ee/fe matrix +# from the previous GitLab config to preserve behavior. +# +# Resource group: prod — prevents two prod builds from racing (kept from +# the previous root .gitlab-ci.yml). +# +# Upstream .build expects MODULES_MODULE_SOURCE and WERF_REPO; .prod_vars +# sets MODULES_MODULE_SOURCE per-edition (PROD_REGISTRY/PROD_MODULE_SOURCE_NAME/ +# EDITION/modules), and WERF_REPO defaults to MODULES_MODULE_SOURCE/ +# MODULES_MODULE_NAME in upstream Setup — correct for prod. +# +# `.dual_registry_login` is included so the second `werf cr login` from +# upstream Setup.gitlab-ci.yml also fires against DEV_REGISTRY; the +# previous GH workflow did both logins via two consecutive +# modules-actions/setup steps, and the migration plan keeps that behavior. + +build_prod: + stage: build + resource_group: prod + interruptible: false + extends: + - .local_build + - .prod_always + - .dual_registry_login + parallel: + matrix: + - EDITION: + - ce + - ee + - fe + # TODO: add `se-plus` edition here once parity with the GH + # release_module_build-and-registration.yml workflow is decided + # (see migration plan §7, "TODO: edition se-plus"). diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml new file mode 100644 index 0000000000..c430c2409a --- /dev/null +++ b/.gitlab/ci/jobs/cleanup.yml @@ -0,0 +1,29 @@ +# Cleanup job. +# +# Carries forward the cleanup job from the previous root .gitlab-ci.yml. +# Runs only on scheduled pipelines and prunes old module images from the +# DEV registry using werf cleanup --without-kube=true. +# +# Behavior preserved: +# - registry host: dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization +# (kept as a literal because we are cleaning exactly this repo's +# namespace, regardless of DEV_REGISTRY variable contents). +# - tag: v0.0.0-main (matches the build_main tag). +# - resource: not pinned (single schedule at a time is enough). +# - MODULES_REGISTRY is set so any inherited .dev_vars does not override +# the hardcoded path — we deliberately do NOT extend .dev_vars here. +# +# TODO: parameterize the registry path via DEV_MODULE_SOURCE / MODULE_NAME +# once the previous GH cleanup workflow (dev_registry-cleanup.yml) is fully +# analyzed — see migration plan §2 row for dev_registry-cleanup.yml. + +cleanup: + stage: cleanup + variables: + MODULES_MODULE_TAG: v0.0.0-main + MODULES_REGISTRY: dev-registry.deckhouse.io + MODULES_MODULE_SOURCE: dev-registry.deckhouse.io/sys/deckhouse-oss/modules + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + script: + - werf cleanup --repo dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization --without-kube=true diff --git a/.gitlab/ci/jobs/deploy-dev.yml b/.gitlab/ci/jobs/deploy-dev.yml new file mode 100644 index 0000000000..2873b36c65 --- /dev/null +++ b/.gitlab/ci/jobs/deploy-dev.yml @@ -0,0 +1,32 @@ +# DEV deploy job. +# +# Carries forward deploy_for_dev_tag from the previous root .gitlab-ci.yml. +# Fans out across the five release channels (alpha / beta / early-access / +# stable / rock-solid) once build_dev_tags has finished. +# +# Extends: +# - .deploy (upstream modules-gitlab-ci Deploy.gitlab-ci.yml — release-channel crane copy) +# - .dev_tags (this repo — DEV registry vars + tag-match rule) +# +# The upstream .deploy template only runs on $CI_COMMIT_TAG (rule from +# Deploy.gitlab-ci.yml), which matches .dev_tags's tag regex +# vX.Y.Z-dev.* — behavior preserved. +# +# Need: build_dev_tags -> deploy_for_dev_tag. (Kept as a `needs:` line — +# same as the previous root .gitlab-ci.yml.) + +deploy_for_dev_tag: + stage: deploy_dev + needs: ['build_dev_tags'] + extends: + - .local_deploy + - .dev_tags + - .dual_registry_login + parallel: + matrix: + - RELEASE_CHANNEL: + - alpha + - beta + - early-access + - stable + - rock-solid diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml new file mode 100644 index 0000000000..8a83675174 --- /dev/null +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -0,0 +1,106 @@ +# PROD deploy jobs. +# +# Carries forward deploy_to_prod_{alpha,beta,ea,stable,rock_solid} from +# the previous root .gitlab-ci.yml. Each job runs after the previous one +# (alpha -> beta -> ea -> stable -> rock_solid), waits for the matching +# matrix entry of build_prod, and copies the built release image to the +# named release channel. +# +# Extends: +# - .local_deploy (this repo — mirrors upstream modules-gitlab-ci +# Deploy.gitlab-ci.yml `.deploy` script body without +# the upstream `rules:` that would override our +# gating). +# - .prod_manual (this repo — PROD registry vars + tag-match rule +# with `when: manual`). +# - .dual_registry_login (this repo — also login against DEV_REGISTRY). +# +# All five are manual (`when: manual` from .prod_manual) and depend on +# each other via `needs:` to preserve the gate-by-gate prod release flow. +# +# TODO: per migration plan §11.4.1, the GH release_module_release-channels +# workflow exposes inputs (channel, ce/ee, tag, enableBuild, +# release_to_github, check_only, send_results_to_loop, ...) that we +# currently collapse into the hardcoded RELEASE_CHANNEL x EDITION matrix. +# Decide whether to expose that flexibility via "Run pipeline" UI variables +# before locking the release flow down. + +deploy_to_prod_alpha: + stage: deploy_prod_alpha + variables: + RELEASE_CHANNEL: alpha + needs: ['build_prod'] + extends: + - .local_deploy + - .prod_manual + - .dual_registry_login + parallel: + matrix: + - EDITION: + - ce + - ee + - fe + +deploy_to_prod_beta: + stage: deploy_prod_beta + variables: + RELEASE_CHANNEL: beta + needs: ['deploy_to_prod_alpha'] + extends: + - .local_deploy + - .prod_manual + - .dual_registry_login + parallel: + matrix: + - EDITION: + - ce + - ee + - fe + +deploy_to_prod_ea: + stage: deploy_prod_ea + variables: + RELEASE_CHANNEL: early-access + needs: ['deploy_to_prod_beta'] + extends: + - .local_deploy + - .prod_manual + - .dual_registry_login + parallel: + matrix: + - EDITION: + - ce + - ee + - fe + +deploy_to_prod_stable: + stage: deploy_prod_stable + variables: + RELEASE_CHANNEL: stable + needs: ['deploy_to_prod_ea'] + extends: + - .local_deploy + - .prod_manual + - .dual_registry_login + parallel: + matrix: + - EDITION: + - ce + - ee + - fe + +deploy_to_prod_rock_solid: + stage: deploy_prod_rock_solid + variables: + RELEASE_CHANNEL: rock-solid + needs: ['deploy_to_prod_stable'] + extends: + - .local_deploy + - .prod_manual + - .dual_registry_login + parallel: + matrix: + - EDITION: + - ce + - ee + - fe diff --git a/.gitlab/ci/jobs/info.yml b/.gitlab/ci/jobs/info.yml new file mode 100644 index 0000000000..b07e1fa07a --- /dev/null +++ b/.gitlab/ci/jobs/info.yml @@ -0,0 +1,21 @@ +# Info-stage jobs: kubectl manifest printers. +# +# Carries forward show_dev_manifest and show_main_manifest from the +# previous root .gitlab-ci.yml. These jobs do not run any heavy tooling; +# they just print a copy-paste helper so testers can quickly wire up a +# ModuleConfig + ModulePullOverride for the build under test. +# +# Both extend .info (script body) and either .dev (MR pipelines) or .main +# (pushes to the default branch) for MODULES_MODULE_TAG + registry vars. + +show_dev_manifest: + stage: info + extends: + - .info + - .dev + +show_main_manifest: + stage: info + extends: + - .info + - .main diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml new file mode 100644 index 0000000000..817d83272d --- /dev/null +++ b/.gitlab/ci/jobs/test.yml @@ -0,0 +1,38 @@ +# Lint + unit-test jobs. +# +# Carries forward the lint/test jobs from the previous root .gitlab-ci.yml: +# lint:virtualization-controller -> task virtualization-controller:init + lint +# test:virtualization-controller -> task virtualization-controller:init + test:unit +# test:hooks -> task hooks:test +# +# All three extend .dev so they only run on MR pipelines with the right +# MODULES_MODULE_TAG/registry context. The other lint checks (yaml-lint, +# no-cyrillic, shellcheck, helm-templates, gen-files-check, dmt-lint, +# gitlab-ci-lint) live in .gitlab/ci/jobs/lint-validate.yml, which is owned +# by a different subagent issue — keep the include line for it commented +# here as a hint for the integration step. + +lint:virtualization-controller: + stage: lint + script: + - task virtualization-controller:init + - task virtualization-controller:lint + # TODO: needs follow-up — dvcr lint target has known issues + # - task virtualization-controller:dvcr:lint + extends: + - .dev + +test:virtualization-controller: + stage: test + script: + - task virtualization-controller:init + - task virtualization-controller:test:unit + extends: + - .dev + +test:hooks: + stage: test + script: + - task hooks:test + extends: + - .dev diff --git a/.gitlab/ci/scripts/lib/api.sh b/.gitlab/ci/scripts/lib/api.sh new file mode 100755 index 0000000000..3fa55da965 --- /dev/null +++ b/.gitlab/ci/scripts/lib/api.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared GitLab API helpers for migration-era jobs. +# +# Source from a job's script: +# source .gitlab/ci/scripts/lib/api.sh +# +# Provides: +# api METHOD PATH [curl-args...] -- REST call with PRIVATE-TOKEN, prints body, returns exit code. +# gl_required_env -- fails if required env vars are missing. +# gl_log_call -- echoes request line for log readability. +# +# Conventions (see tmp/ai-summary/gitlab-ci-migration-plan.md §11.1): +# - Always CI_API_V4_URL (never hardcode the host). +# - Always GITLAB_API_TOKEN (Project Access Token, scope api). +# - Always CI_PROJECT_ID (numeric) and CI_MERGE_REQUEST_IID (iid, not id). + +# Guard against double-sourcing. +if [[ -n "${__GL_API_SH_SOURCED:-}" ]]; then + return 0 +fi +__GL_API_SH_SOURCED=1 + +set -euo pipefail + +gl_required_env() { + local missing=() + local v + for v in "$@"; do + if [[ -z "${!v:-}" ]]; then + missing+=("$v") + fi + done + if [[ "${#missing[@]}" -gt 0 ]]; then + echo "ERROR: required environment variables are not set: ${missing[*]}" >&2 + exit 1 + fi +} + +gl_log_call() { + local method="$1" + local path="$2" + echo ">>> ${method} ${CI_API_V4_URL}${path}" +} + +# api METHOD PATH [extra curl args] +# +# Examples: +# api GET "/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" +# api POST "/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/assignees" \ +# --data '{"user_id":42}' +# +# Behaviour: +# - Uses PRIVATE-TOKEN with $GITLAB_API_TOKEN. +# - Sets Content-Type: application/json by default (overridable via extra args). +# - On non-2xx: prints status and body to stderr, returns non-zero. +api() { + local method="$1" + shift + local path="$1" + shift + + gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID + gl_log_call "$method" "$path" + + local response_file + response_file="$(mktemp)" + local http_code + http_code="$(curl --silent --show-error --output "$response_file" \ + --request "$method" \ + --write-out '%{http_code}' \ + --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" \ + --header "Content-Type: application/json" \ + --header "Accept: application/json" \ + "${CI_API_V4_URL}${path}" "$@")" + + cat "$response_file" + rm -f "$response_file" + + if [[ "$http_code" =~ ^2 ]]; then + echo "<<< status=${http_code}" + return 0 + fi + + echo "<<< status=${http_code} (FAILED)" >&2 + return 1 +} diff --git a/.gitlab/ci/scripts/set-vars.sh b/.gitlab/ci/scripts/set-vars.sh new file mode 100755 index 0000000000..689ce10c44 --- /dev/null +++ b/.gitlab/ci/scripts/set-vars.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# set-vars.sh — derives per-pipeline variables for downstream jobs. +# +# Carries forward the responsibilities of the GH `set_vars` job from +# dev_module_build.yml (migration plan §11.3.4). Produces a dotenv +# artifact that downstream jobs consume via `needs: [set_vars]` + +# `artifacts.reports.dotenv`. +# +# Outputs (written to set_vars.env in $CI_PROJECT_DIR): +# MODULES_MODULE_TAG mrNNN for MR pipelines, main for default branch, +# release-X.Y for release branches, mrNNN for manual +# PR_NUMBER override, fail otherwise. +# MODULE_EDITION CE if MR carries label edition/ce, otherwise EE. +# RELEASE_IN_DEV true if $CI_COMMIT_BRANCH matches release-X.Y, +# false otherwise. +# DEBUG_COMPONENT first delve/* label (empty if none). +# +# Required env (provided by the job context): +# CI_API_V4_URL, CI_PROJECT_ID, CI_PIPELINE_SOURCE, CI_COMMIT_BRANCH, +# CI_MERGE_REQUEST_IID, CI_MERGE_REQUEST_LABELS, GITLAB_API_TOKEN. +# +# This script is not yet wired into a job by this issue — the +# info.yml / set-vars integration lands in a follow-up because the previous +# GitLab config did not have an equivalent job. Child issue can call it via: +# set_vars: +# stage: info +# script: +# - bash .gitlab/ci/scripts/set-vars.sh +# artifacts: +# reports: +# dotenv: set_vars.env + +set -euo pipefail + +# Source the api() helper for the GitLab API call below. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +OUTPUT="${CI_PROJECT_DIR:-.}/set_vars.env" + +# 1) MODULES_MODULE_TAG ------------------------------------------------------ +# Mirrors the GH set_vars job: prefer MR iid for MR pipelines, then main for +# pushes to the default branch, then release-X.Y for release branches, then +# PR_NUMBER for manual triggers, fail otherwise. +if [[ "${CI_PIPELINE_SOURCE:-}" == "merge_request_event" ]]; then + if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then + echo "ERROR: merge_request_event pipeline without CI_MERGE_REQUEST_IID" >&2 + exit 1 + fi + MODULES_MODULE_TAG="mr${CI_MERGE_REQUEST_IID}" +elif [[ "${CI_COMMIT_BRANCH:-}" == "${CI_DEFAULT_BRANCH:-main}" ]]; then + MODULES_MODULE_TAG="main" +elif [[ "${CI_COMMIT_BRANCH:-}" =~ ^release-([0-9]+\.[0-9]+) ]]; then + MODULES_MODULE_TAG="${CI_COMMIT_BRANCH}" +elif [[ -n "${PR_NUMBER:-}" ]]; then + MODULES_MODULE_TAG="mr${PR_NUMBER}" +else + echo "ERROR: cannot derive MODULES_MODULE_TAG (source=${CI_PIPELINE_SOURCE:-?}, branch=${CI_COMMIT_BRANCH:-empty})" >&2 + exit 1 +fi + +# 2) RELEASE_IN_DEV ---------------------------------------------------------- +if [[ "${CI_COMMIT_BRANCH:-}" =~ ^release-[0-9]+\.[0-9]+ ]]; then + RELEASE_IN_DEV="true" +else + RELEASE_IN_DEV="false" +fi + +# 3) Labels via GitLab API --------------------------------------------------- +# GitLab exposes $CI_MERGE_REQUEST_LABELS automatically for MR pipelines, but +# we keep the explicit API fetch as a safety net for manual/web pipelines that +# target a specific MR via PR_NUMBER. +LABELS="" +if [[ -n "${CI_MERGE_REQUEST_LABELS:-}" ]]; then + LABELS="${CI_MERGE_REQUEST_LABELS}" +elif [[ -n "${PR_NUMBER:-}" && -n "${GITLAB_API_TOKEN:-}" ]]; then + LABELS="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests/${PR_NUMBER}" \ + | jq -r '.labels | join(",")')" +fi + +# 4) MODULE_EDITION ---------------------------------------------------------- +if [[ ",${LABELS}," == *,edition/ce,* ]]; then + MODULE_EDITION="CE" +else + MODULE_EDITION="EE" +fi + +# 5) DEBUG_COMPONENT --------------------------------------------------------- +DEBUG_COMPONENT="" +DELVE_COUNT=0 +if [[ -n "${LABELS}" ]]; then + DEBUG_COMPONENT="$(echo "${LABELS}" | tr ',' '\n' | grep '^delve' | head -n1 || true)" + DELVE_COUNT="$(echo "${LABELS}" | tr ',' '\n' | grep -c '^delve' || true)" +fi +if [[ "${DELVE_COUNT}" -gt 1 ]]; then + echo "ERROR: multiple delve labels: ${LABELS}" >&2 + exit 1 +fi + +# 6) Persist ----------------------------------------------------------------- +cat > "${OUTPUT}" <>> wrote ${OUTPUT}" +cat "${OUTPUT}" diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml new file mode 100644 index 0000000000..9a33ed7836 --- /dev/null +++ b/.gitlab/ci/stages.yml @@ -0,0 +1,41 @@ +# Pipeline stages for the deckhouse/virtualization module. +# +# This is the union of the stages used by the previous root .gitlab-ci.yml +# and the new stages introduced by the GitHub Actions migration plan +# (tmp/ai-summary/gitlab-ci-migration-plan.md §11.8). +# +# We intentionally drop the `e2e` stage here: the migration plan explicitly +# excludes all e2e-* workflows from the GitLab CI migration. A child issue +# owns any future re-introduction of e2e stages under a separate file. +# +# Stage semantics: +# info — manifest printers, set_vars helper +# lint — Go lint, helm lint, yaml lint, no-cyrillic, etc. +# test — Go unit tests, hooks tests +# build — werf build for dev / dev-tags / main / prod +# scan — cve_scan, gitleaks, svace (owned by sibling issues) +# deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid +# deploy_prod_alpha / beta / ea / stable / rock_solid +# — sequential prod release-channel deploy chain +# notify — release-results-to-loop and similar fan-out +# cleanup — scheduled registry cleanup +# +# Upstream Setup.gitlab-ci.yml declares `stages: [build, deploy]` which +# would otherwise conflict with our expanded list. We override by re-stating +# the full list here; jobs that `extends: .build` keep `stage: build` and +# land in our `build` stage (not a separate "build" from upstream). + +stages: + - info + - lint + - test + - build + - scan + - deploy_dev + - deploy_prod_alpha + - deploy_prod_beta + - deploy_prod_ea + - deploy_prod_stable + - deploy_prod_rock_solid + - notify + - cleanup diff --git a/.gitlab/ci/templates/build.yml b/.gitlab/ci/templates/build.yml new file mode 100644 index 0000000000..d7c68a26a4 --- /dev/null +++ b/.gitlab/ci/templates/build.yml @@ -0,0 +1,65 @@ +# Local .local_build template. +# +# This template mirrors the upstream modules-gitlab-ci Build.gitlab-ci.yml +# `.build` body verbatim (verified against +# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci +# templates/Build.gitlab-ci.yml, branch v13.0, HEAD 006d51c). +# +# Why a local copy instead of `extends: .build`: +# +# Upstream `.build` carries two `rules:` that fire on ANY branch or tag +# push (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`). GitLab CI merges +# rules from all `extends:` parents with the parent's rules evaluated +# first, so the upstream rules always win. That regresses behavior for +# jobs like build_dev_tags, build_main, build_prod which need to gate on +# specific branch / tag patterns provided by .dev / .dev_tags / .main / +# .prod_always. +# +# By keeping the upstream body in a locally named `.local_build` we keep +# the same script (so future upstream fixes can be copy-pasted here) +# while our job-level rules from .dev / .main / .prod_always / etc. +# drive the gating. +# +# Required variables (documented in variables.yml + upstream Setup): +# MODULES_MODULE_SOURCE, MODULES_MODULE_NAME, MODULES_MODULE_TAG, +# WERF_REPO, SVACE_ENABLED (optional). +# +# TODO: after the upstream modules-gitlab-ci repo lands a way to pass +# `rules:` overrides through extends (tracked in upstream issues), revisit +# this template and switch back to `extends: .build`. + +.local_build: + stage: build + script: + # Use gitlab ci job token + - | + SOURCE_REPO=${SOURCE_REPO#git@} + SOURCE_REPO=${SOURCE_REPO//://} + export SOURCE_REPO=https://gitlab-ci-token:${CI_JOB_TOKEN}@${SOURCE_REPO} + + # Build images + - | + werf build \ + --save-build-report --build-report-path images_tags_werf.json + + # Bundle image + - | + IMAGE_SRC="$(jq -r '.Images."bundle".DockerImageName' images_tags_werf.json)" + IMAGE_DST="$(jq -r '.Images.bundle.DockerRepo' images_tags_werf.json):${MODULES_MODULE_TAG}" + + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + # Release-channel image + - | + IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)" + IMAGE_DST="$(jq -r '.Images."release-channel-version".DockerRepo' images_tags_werf.json)/release:${MODULES_MODULE_TAG}" + + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + # Register module + - | + echo "✨ Register the module ${MODULES_MODULE_NAME}" + crane append \ + --oci-empty-base \ + --new_layer "" \ + --new_tag "${MODULES_MODULE_SOURCE}:${MODULES_MODULE_NAME}" diff --git a/.gitlab/ci/templates/deploy.yml b/.gitlab/ci/templates/deploy.yml new file mode 100644 index 0000000000..8626f48e66 --- /dev/null +++ b/.gitlab/ci/templates/deploy.yml @@ -0,0 +1,30 @@ +# Local .local_deploy template. +# +# Mirrors upstream modules-gitlab-ci Deploy.gitlab-ci.yml `.deploy` body +# (verified in +# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci +# templates/Deploy.gitlab-ci.yml, branch v13.0, HEAD 006d51c). +# +# Why a local copy (same reasoning as .local_build in build.yml): +# +# Upstream `.deploy` has `rules: [{if: $CI_COMMIT_TAG}]` + `when: manual` +# baked in. Because GitLab CI merges parent rules with parent-first +# precedence, that turns deploy_for_dev_tag into a manual job on prod +# tags too (we want it auto on dev tags and absent on prod tags). +# The prod-channel chain (deploy_to_prod_*) wants `when: manual` from +# .prod_manual, so the upstream `when: manual` happens to match — but we +# still want explicit control. Keeping the upstream body in `.local_deploy` +# means the same script is shared by all jobs but gating is fully driven +# by our own templates. + +.local_deploy: + stage: deploy + script: + - | + REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" + + IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" + IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" + + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy "${IMAGE_SRC}" "${IMAGE_DST}" diff --git a/.gitlab/ci/templates/dev.yml b/.gitlab/ci/templates/dev.yml new file mode 100644 index 0000000000..adfafa672c --- /dev/null +++ b/.gitlab/ci/templates/dev.yml @@ -0,0 +1,19 @@ +# DEV pipeline context: MR builds. +# +# Carries forward the previous root .gitlab-ci.yml `.dev` template. Any job +# that uses `extends: .dev` runs only on MR pipelines, derives its +# MODULES_MODULE_TAG from the MR iid, and inherits the DEV registry +# credentials via .dev_vars. +# +# TODO_RUNNER_TAG: runner tags come from .gitlab/ci/defaults.yml; this file +# only sets logic-level defaults. + +.dev: + variables: + MODULES_MODULE_TAG: mr${CI_MERGE_REQUEST_IID} + extends: + - .dev_vars + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never diff --git a/.gitlab/ci/templates/dev_tags.yml b/.gitlab/ci/templates/dev_tags.yml new file mode 100644 index 0000000000..e71976f967 --- /dev/null +++ b/.gitlab/ci/templates/dev_tags.yml @@ -0,0 +1,20 @@ +# DEV pipeline context: dev-tag builds. +# +# Carries forward the previous root .gitlab-ci.yml `.dev_tags` template. +# Triggers on tags matching vX.Y.Z-dev* (regex kept identical to the +# previous regex101 link in the original .gitlab-ci.yml). +# +# Use case: a developer pushes v1.21.0-dev-build42 -> build_dev_tags runs, +# pushes images into DEV_REGISTRY tagged with the raw tag name, then +# deploy_for_dev_tag fans out across all release channels. + +.dev_tags: + variables: + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + extends: + - .dev_vars + rules: + # https://regex101.com/r/0VtnPP/1 + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-dev.*$/' + when: always + - when: never diff --git a/.gitlab/ci/templates/dev_vars.yml b/.gitlab/ci/templates/dev_vars.yml new file mode 100644 index 0000000000..49a598447c --- /dev/null +++ b/.gitlab/ci/templates/dev_vars.yml @@ -0,0 +1,24 @@ +# DEV registry variable bundle. +# +# Carries forward the previous root .gitlab-ci.yml `.dev_vars` template. +# Anything that uses `extends: .dev_vars` gets MODULES_REGISTRY, +# MODULES_REGISTRY_LOGIN, MODULES_REGISTRY_PASSWORD, MODULES_MODULE_SOURCE +# and ENV=DEV, all sourced from the DEV_* Project Variables introduced in +# plan §11.6. +# +# The legacy EXTERNAL_MODULES_DEV_REGISTRY_* names are still accepted as +# fallback to ease the variable-rename migration; they are documented in +# .gitlab/README.md (TODO) as deprecated. + +.dev_vars: + variables: + MODULES_REGISTRY: "${DEV_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" + MODULES_MODULE_SOURCE: "${DEV_MODULE_SOURCE}" + ENV: DEV + # Backwards-compat fallback for the previous CI/CD variable names. Safe + # to remove once Settings -> CI/CD -> Variables no longer carries the + # EXTERNAL_MODULES_DEV_* entries (see .gitlab/README.md migration step). + DEV_REGISTRY_FALLBACK_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + DEV_REGISTRY_FALLBACK_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" diff --git a/.gitlab/ci/templates/dual_registry_login.yml b/.gitlab/ci/templates/dual_registry_login.yml new file mode 100644 index 0000000000..f42bebbfa6 --- /dev/null +++ b/.gitlab/ci/templates/dual_registry_login.yml @@ -0,0 +1,31 @@ +# Dual-registry login helper for prod release jobs. +# +# The upstream modules-gitlab-ci Setup.gitlab-ci.yml (templates/Setup.gitlab-ci.yml) +# does: +# werf cr login -u $MODULES_REGISTRY_LOGIN -p $MODULES_REGISTRY_PASSWORD $MODULES_REGISTRY +# if [[ -n $DEV_MODULES_REGISTRY_LOGIN && ... ]]; then +# werf cr login -u $DEV_MODULES_REGISTRY_LOGIN -p $DEV_MODULES_REGISTRY_PASSWORD $DEV_MODULES_REGISTRY +# fi +# +# That means a job only needs to set DEV_MODULES_REGISTRY_* in addition to +# MODULES_REGISTRY_* to get a second login. We expose DEV_MODULES_REGISTRY_* +# via this helper so prod release jobs (which need to copy from DEV and +# register into PROD) can extend both .prod_vars and .dual_registry_login +# and end up with the right pair of `werf cr login` calls. +# +# Usage: +# extends: +# - .prod_vars # sets MODULES_* = PROD_* +# - .dual_registry_login # adds DEV_MODULES_* = DEV_* +# - .build # upstream werf build + bundle/release-channel copy +# +# TODO: confirm with a virt-test pipeline that the second login is actually +# required for the build_prod / deploy_to_prod_* chain (the plan §11.7.3 +# flags this as an open question because the previous GH workflow did both +# logins via two consecutive `modules-actions/setup` steps). + +.dual_registry_login: + variables: + DEV_MODULES_REGISTRY: "${DEV_REGISTRY}" + DEV_MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + DEV_MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" diff --git a/.gitlab/ci/templates/info.yml b/.gitlab/ci/templates/info.yml new file mode 100644 index 0000000000..2d4da2cd77 --- /dev/null +++ b/.gitlab/ci/templates/info.yml @@ -0,0 +1,50 @@ +# Info-job template: prints a kubectl manifest helper for testers. +# +# Carries forward the previous root .gitlab-ci.yml `.info` template. Used +# by jobs in .gitlab/ci/jobs/info.yml that print a copy-paste-ready +# ModuleConfig + ModulePullOverride YAML for the MR/build being tested. +# +# Note: the previous version inlined this template inside the manifest +# jobs. We split it out so show_dev_manifest and show_main_manifest can +# share it via extends. + +.info: + script: + - | + cat << OUTER + Create ModuleConfig and ModulePullOverride resources to test this MR: + cat < CI/CD -> Variables (masked +# where appropriate) and is referenced via ${VAR} expansion below. +# +# Legacy variable migration (plan §11.6): +# EXTERNAL_MODULES_DEV_REGISTRY_LOGIN -> DEV_MODULES_REGISTRY_LOGIN +# EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD -> DEV_MODULES_REGISTRY_PASSWORD +# EXTERNAL_MODULES_PROD_REGISTRY_LOGIN -> PROD_MODULES_REGISTRY_LOGIN +# EXTERNAL_MODULES_PROD_REGISTRY_PASSWORD-> PROD_MODULES_REGISTRY_PASSWORD +# Hardcoded registry hosts (`dev-registry.deckhouse.io`, +# `registry-write.deckhouse.io`, `sys/deckhouse-oss/modules`, +# `deckhouse/${EDITION}/modules`) move to DEV_REGISTRY / DEV_MODULE_SOURCE / +# PROD_REGISTRY / PROD_MODULE_SOURCE_NAME vars. +# +# See .gitlab/README.md (TODO) for the migration steps. + +variables: + # --- Module identity --- + MODULE_NAME: virtualization + # Kept for backwards compatibility with the previous root .gitlab-ci.yml; + # upstream templates expect MODULES_MODULE_NAME. + MODULES_MODULE_NAME: virtualization + + # --- Werf / base images / lib-helm --- + # These match upstream modules-gitlab-ci Setup.gitlab-ci.yml defaults. + # Override at the project level once the base-images + lib-helm versions + # are validated against this module's werf config. + BASE_IMAGES_VERSION: v0.5.7 + WERF_VERSION: "2 stable" + # DECKHOUSE_LIB_HELM_VERSION: leave empty here; populate when the lib-helm + # version is pinned in Taskfile.yaml. Upstream Setup.gitlab-ci.yml + # only downloads if this var is non-empty. + DECKHOUSE_LIB_HELM_VERSION: "" + # Disable VEX notice noise from trivy (matches previous GH env). + TRIVY_DISABLE_VEX_NOTICE: "true" + + # --- DEV registry (vars-only, populated via GitLab CI/CD variables) --- + # Required Project Variables (vars, not masked): + # DEV_REGISTRY e.g. dev-registry.deckhouse.io + # DEV_MODULE_SOURCE e.g. dev-registry.deckhouse.io/sys/deckhouse-oss/modules + # DEV_MODULES_REGISTRY_LOGIN + # Required Project Variables (masked): + # DEV_MODULES_REGISTRY_PASSWORD + # + # --- PROD registry (vars-only) --- + # Required Project Variables (vars): + # PROD_REGISTRY e.g. registry-write.deckhouse.io + # PROD_MODULE_SOURCE_NAME e.g. deckhouse + # PROD_MODULES_REGISTRY_LOGIN + # Required Project Variables (masked): + # PROD_MODULES_REGISTRY_PASSWORD + # Optional read-only license registry (used by cve-scan + check:requirements): + # PROD_READ_REGISTRY, PROD_READ_REGISTRY_USER, PROD_READ_REGISTRY_PASSWORD + + # --- Build/dev tooling (kept from previous root) --- + GO_VERSION: "1.25.11" + GOLANGCI_LINT_VERSION: "2.11.1" + WERF_EXPERIMENTAL_IMPORT_BY_SOURCE_IMAGE_TAG: "true" diff --git a/.gitlab/ci/workflow.yml b/.gitlab/ci/workflow.yml new file mode 100644 index 0000000000..f4ba4fcbdd --- /dev/null +++ b/.gitlab/ci/workflow.yml @@ -0,0 +1,26 @@ +# Pipeline-level workflow rules. +# +# Intentionally minimal: we do NOT use `workflow:rules:` to gate individual +# jobs — gating is handled per-job via `rules:` against +# $CI_PIPELINE_SOURCE, $CI_COMMIT_BRANCH, $CI_COMMIT_TAG. The global +# workflow rules here only decide whether a pipeline is created at all. +# +# Behaviors preserved from the previous root .gitlab-ci.yml: +# - MR pipelines (push + MR events) are always created. +# - Pushes to main create a pipeline. +# - Pushes of tags vX.Y.Z-dev* / vX.Y.Z create pipelines. +# - Manual/scheduled/web pipelines always run. +# +# TODO: once auto-cancel-redundant-pipelines is verified at the project +# level, add `interruptible: true` here as a project default (see plan §11.15). +# Per-job overrides will set `interruptible: false` for build_prod and the +# prod-deploy chain. + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(-dev.*)?$/ + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "push" From ddab259700648767171c8183ee9d82986a6650a0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 22 Jun 2026 19:33:15 +0300 Subject: [PATCH 02/60] feat(ci): add gitlab validation and scanning jobs Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 11 + .gitlab/ci/jobs/cve-scan.yml | 73 +++++++ .gitlab/ci/jobs/gitleaks.yml | 54 +++++ .gitlab/ci/jobs/lint-validate.yml | 298 +++++++++++++++++++++++++++ .gitlab/ci/jobs/precache.yml | 49 +++++ .gitlab/ci/jobs/svace.yml | 155 ++++++++++++++ .gitlab/ci/scripts/gitlab-ci-lint.sh | 139 +++++++++++++ 7 files changed, 779 insertions(+) create mode 100644 .gitlab/ci/jobs/cve-scan.yml create mode 100644 .gitlab/ci/jobs/gitleaks.yml create mode 100644 .gitlab/ci/jobs/lint-validate.yml create mode 100644 .gitlab/ci/jobs/precache.yml create mode 100644 .gitlab/ci/jobs/svace.yml create mode 100755 .gitlab/ci/scripts/gitlab-ci-lint.sh diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index b1941701b3..059fcbd6d2 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -38,6 +38,10 @@ include: # Setup.gitlab-ci.yml provides trdl + werf setup + dual-registry # `werf cr login` in before_script. See templates/Setup.gitlab-ci.yml. - '/templates/Setup.gitlab-ci.yml' + # Validation/scanning jobs extend these upstream templates. + - '/templates/CVE_Scan.gitlab-ci.yml' + - '/templates/gitleaks.gitlab-ci.yml' + - '/templates/Svace_Analayze.gitlab-ci.yml' # Build.gitlab-ci.yml and Deploy.gitlab-ci.yml are intentionally NOT # included directly. Their hidden jobs (.build, .deploy) ship with # `rules:` baked in (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`, @@ -73,3 +77,10 @@ include: - local: '.gitlab/ci/jobs/deploy-dev.yml' - local: '.gitlab/ci/jobs/deploy-prod.yml' - local: '.gitlab/ci/jobs/cleanup.yml' + + # --- Local validation and scanning jobs --- + - local: '.gitlab/ci/jobs/lint-validate.yml' + - local: '.gitlab/ci/jobs/precache.yml' + - local: '.gitlab/ci/jobs/svace.yml' + - local: '.gitlab/ci/jobs/cve-scan.yml' + - local: '.gitlab/ci/jobs/gitleaks.yml' diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml new file mode 100644 index 0000000000..f0f0e492d0 --- /dev/null +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -0,0 +1,73 @@ +# CVE (Trivy) scan via the upstream `.cve_scan` template. +# +# Migration of .github/workflows/cve_scan_daily.yml. The GH workflow +# pulled CVE_TEST_REPO_GIT, CODEOWNERS_REPO_TOKEN, DD_URL, DD_TOKEN, +# DECKHOUSE_PRIVATE_REPO, etc. from HashiCorp Vault via +# `hashicorp/vault-action@v2`. The upstream +# deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 `.cve_scan` template +# does the same thing but via GitLab `id_tokens` + d8-cli. Both paths +# resolve to the same set of variables, so extending the upstream +# template is the right migration. +# +# The Vault integration is unchanged (still uses seguro.flant.com); only +# the auth mechanism moves from GH vault-action to GitLab id_tokens. +# +# Required CI/CD variables (declared in the project). The upstream +# template provides defaults via `vault:` references, but the +# operator must set: +# VAULT_ROLE - role to authenticate against in seguro.flant.com +# SOURCE_TAG - tag/branch to scan (e.g. "main" or a release tag) +# CASE - case label forwarded to cve_scan.sh +# EXTERNAL_MODULE_NAME - the module name (default: "virtualization") +# +# The job runs on schedule + manual, matching the GH `on: schedule` and +# `on: workflow_dispatch` triggers. + +# --------------------------------------------------------------------------- +# Scheduled daily scan against main. +# Mirrors `cve_scan_daily.yml` cron: "0 02 * * *". +# --------------------------------------------------------------------------- + +cve:scan:daily: + extends: + - .cve_scan + stage: scan + # TODO_RUNNER_TAG: Trivy scan over the full module image set is heavy; + # narrow to [deckhouse, large] once runners are registered. + interruptible: false + variables: + SOURCE_TAG: "main" + CASE: "External Modules" + EXTERNAL_MODULE_NAME: "virtualization" + SCAN_SEVERAL_LATEST_RELEASES: "True" + LATEST_RELEASES_AMOUNT: "5" + RELEASE_IN_DEV: "false" + MODULE_PROD_REGISTRY_CUSTOM_PATH: "deckhouse/fe/modules" + MODULE_DEV_REGISTRY_CUSTOM_PATH: "sys/deckhouse-oss/modules" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + +# --------------------------------------------------------------------------- +# Manual scan against a chosen tag or branch. +# Mirrors `cve_scan_daily.yml` `workflow_dispatch.inputs.tag_name`. +# --------------------------------------------------------------------------- + +cve:scan:manual: + extends: + - .cve_scan + stage: scan + # TODO_RUNNER_TAG: same as cve:scan:daily. + interruptible: false + variables: + SOURCE_TAG: "${SCAN_TAG:-main}" + CASE: "External Modules" + EXTERNAL_MODULE_NAME: "virtualization" + SCAN_SEVERAL_LATEST_RELEASES: "${SCAN_SEVERAL:-False}" + LATEST_RELEASES_AMOUNT: "5" + RELEASE_IN_DEV: "${SCAN_TAG:-false}" + MODULE_PROD_REGISTRY_CUSTOM_PATH: "deckhouse/fe/modules" + MODULE_DEV_REGISTRY_CUSTOM_PATH: "sys/deckhouse-oss/modules" + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - when: never diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml new file mode 100644 index 0000000000..df45202047 --- /dev/null +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -0,0 +1,54 @@ +# Gitleaks secrets scanning. +# +# Migration of: +# - .github/workflows/gitleaks-scan-on-pr.yml (PR diff scan) +# - .github/workflows/gitleaks-scan-on-schedule.yml (daily full scan) +# +# Both GH workflows invoked `deckhouse/modules-actions/gitleaks@v6`. The +# upstream modules-gitlab-ci@v13.0 `.gitleaks_scan` template provides +# the same functionality via the GitLab-native gitleaks release binary. +# The upstream template already exposes three reusable jobs: +# +# gitleaks_diff -> SCAN_MODE=diff, runs on MR +# gitleaks_full_manual -> SCAN_MODE=full, manual +# gitleaks_full_scheduled -> SCAN_MODE=full, schedule +# +# We re-declare each of them here with the same behavior so the owner +# of this file (this issue) can adjust rules:changes / labels without +# touching the upstream file. They are simple aliases via `extends:`. + +gitleaks:diff: + extends: .gitleaks_scan + stage: scan + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + variables: + SCAN_MODE: "diff" + GIT_DEPTH: "0" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + +gitleaks:full:scheduled: + extends: .gitleaks_scan + stage: scan + # TODO_RUNNER_TAG: full-history scan is heavier than diff; narrow to + # [deckhouse, large] if available. + interruptible: false + variables: + SCAN_MODE: "full" + GIT_DEPTH: "0" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + +gitleaks:full:manual: + extends: .gitleaks_scan + stage: scan + # TODO_RUNNER_TAG: same as gitleaks:full:scheduled. + interruptible: false + variables: + SCAN_MODE: "full" + GIT_DEPTH: "0" + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - when: never diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml new file mode 100644 index 0000000000..0dc50f990b --- /dev/null +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -0,0 +1,298 @@ +# Validation, lint, scan, and gitlab-ci lint jobs. +# +# This file is the direct migration of the GitHub Actions workflow +# .github/workflows/dev_validation.yaml (paths_filter + no_cyrillic + +# doc_changes + shellcheck + helm_templates + check_gens_files). +# +# Per the migration plan (decision #5), the `actionlint` job from the GH +# workflow is intentionally NOT migrated. In its place we run the new +# `lint:gitlab-ci` job that validates the GitLab CI configuration itself +# via the GitLab CI Lint API. The legacy skip-label +# `validation/skip/actionlint` is therefore obsolete and not honored. +# +# Each job: +# - inherits `tags: [deckhouse]` from .gitlab/ci/defaults.yml; +# - emits a `# TODO_RUNNER_TAG:` comment where the real runner tag (e.g. +# large / regular) might need to replace `deckhouse` once runners are +# registered on fox.flant.com/deckhouse/virtualization; +# - has `interruptible: true` where short-lived (safe to cancel on a new +# push) and `interruptible: false` where a long run is expected; +# - honors `validation/skip/` labels via `when: never` rules. +# +# The job file assumes the upstream template include for +# `/templates/Setup.gitlab-ci.yml` from deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 +# is in scope so that the shared `default:` and `stages:` are merged. +# It also assumes the `lint` stage is declared in .gitlab/ci/stages.yml. +# +# Coordination note: the parent .gitlab/ci/includes.yml (owned by the +# epic owner) is responsible for adding `local:` entries for this file. + +# --------------------------------------------------------------------------- +# Shared change-path anchors +# --------------------------------------------------------------------------- + +# Paths that should re-run helm_templates validation. Mirrors the GH +# paths_filter `helm_templates` block from dev_validation.yaml. +.changes_helm_templates: &changes_helm_templates + - 'crds/**/*' + - 'charts/**/*' + - 'tools/kubeconform/**/*' + - 'templates/**/*' + - '.helmignore' + - 'Chart.yaml' + - 'Taskfile.yaml' + +# Paths that should re-run the vm-route-forge generated-files check. +.changes_vm_route_forge: &changes_vm_route_forge + - 'images/vm-route-forge/bpf/route_watcher.c' + - 'images/vm-route-forge/**/*' + +# Paths that should re-run the api/controller generated-files checks. +.changes_go_generated: &changes_go_generated + - 'api/**/*' + - 'images/virtualization-artifact/**/*' + - 'go.mod' + - 'go.sum' + +# Skip labels - exact match against validation/skip/* labels from the GH +# workflow. The `actionlint` skip is intentionally omitted (decision #5). +.skip_no_cyrillic: &skip_no_cyrillic + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/no_cyrillic/' + when: never +.skip_doc_changes: &skip_doc_changes + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/doc_changes/' + when: never +.skip_shellcheck: &skip_shellcheck + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/shellcheck/' + when: never +.skip_helm_templates: &skip_helm_templates + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/helm_templates/' + when: never + +# --------------------------------------------------------------------------- +# no_cyrillic +# --------------------------------------------------------------------------- + +lint:no-cyrillic: + stage: lint + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + image: + name: golang:1.25.11 + entrypoint: [''] + before_script: + - go install github.com/go-task/task/v3/cmd/task@latest + script: + - task validation:no-cyrillic + rules: + - *skip_no_cyrillic + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH =~ /^release-/' + - if: '$CI_PIPELINE_SOURCE == "schedule"' + +# --------------------------------------------------------------------------- +# doc_changes +# --------------------------------------------------------------------------- + +lint:doc-changes: + stage: lint + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + image: + name: golang:1.25.11 + entrypoint: [''] + before_script: + - go install github.com/go-task/task/v3/cmd/task@latest + script: + - task validation:doc-changes + rules: + - *skip_doc_changes + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH =~ /^release-/' + +# --------------------------------------------------------------------------- +# shellcheck +# +# Reuses the upstream-equivalent task `lint:shellcheck`. The .github +# workflow additionally scanned images/virtualization-artifact/hack/*.sh; +# the Taskfile already does that. +# --------------------------------------------------------------------------- + +lint:shellcheck: + stage: lint + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + image: + name: golang:1.25.11 + entrypoint: [''] + before_script: + - go install github.com/go-task/task/v3/cmd/task@latest + script: + - task lint:shellcheck + rules: + - *skip_shellcheck + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH =~ /^release-/' + +# --------------------------------------------------------------------------- +# yaml (prettier) +# --------------------------------------------------------------------------- + +lint:yaml: + stage: lint + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + image: + name: golang:1.25.11 + entrypoint: [''] + before_script: + - go install github.com/go-task/task/v3/cmd/task@latest + script: + - task lint:prettier:yaml + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - '**/*.yaml' + - '**/*.yml' + - '.gitlab-ci.yml' + - '.gitlab/**/*' + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + paths: + - '**/*.yaml' + - '**/*.yml' + - '.gitlab-ci.yml' + - '.gitlab/**/*' + +# --------------------------------------------------------------------------- +# helm_templates +# +# GH used dorny/paths-filter to gate this on changes to helm-related +# files. GitLab equivalent: rules.changes. +# --------------------------------------------------------------------------- + +lint:helm-templates: + stage: lint + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + image: + name: golang:1.25.11 + entrypoint: [''] + before_script: + - go install github.com/go-task/task/v3/cmd/task@latest + script: + - task validation:helm-templates + rules: + - *skip_helm_templates + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: *changes_helm_templates + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + paths: *changes_helm_templates + +# --------------------------------------------------------------------------- +# check_gens_files +# +# Matrix over components. Each component regenerates files via +# `task controller:dev:gogenerate` / `task generate` / `task +# vm-route-forge:gen` and fails if `git diff --exit-code` reports +# any drift against origin/main (or the current branch base). +# --------------------------------------------------------------------------- + +check:gens-files: + stage: lint + # TODO_RUNNER_TAG: heavy jobs may need [deckhouse, large] once runners + # are registered. + interruptible: true + image: + name: golang:1.25.11 + entrypoint: [''] + before_script: + - go install github.com/go-task/task/v3/cmd/task@latest + - apt-get update && apt-get install -y -qq git + script: + - | + set -e + function check_diffs() { + local folder="$1" + if ! git diff --exit-code -- "$folder"; then + echo "::error title=Generated files out of date::Run 'go generate' / 'task generate' for $folder and commit the diff." + echo "--- git diff ---" + git diff origin/main -- "$folder" || true + echo "--- end ---" + exit 1 + fi + echo "OK: $folder" + } + case "$COMPONENT" in + virtualization-artifact) + (cd images/virtualization-artifact && go install tool) + task controller:dev:gogenerate + check_diffs images/virtualization-artifact + ;; + vm-route-forge) + task vm-route-forge:gen + check_diffs images/vm-route-forge + ;; + api) + (cd api && go install tool) + task generate + check_diffs api + ;; + *) + echo "Unknown COMPONENT: $COMPONENT"; exit 1 ;; + esac + parallel: + matrix: + - COMPONENT: [virtualization-artifact, api] + - COMPONENT: vm-route-forge + RUN_ON_CHANGE: "true" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: *changes_go_generated + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: *changes_vm_route_forge + - if: '$CI_COMMIT_BRANCH == "main"' + +# --------------------------------------------------------------------------- +# gitlab-ci-lint +# +# Replaces the legacy actionlint job (decision #5). Runs only when the +# GitLab CI configuration itself changes. Uses the GitLab CI Lint API +# at ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint. +# +# Authentication: PRIVATE-TOKEN via GITLAB_API_TOKEN (project access +# token, scope api). The script lives in +# .gitlab/ci/scripts/gitlab-ci-lint.sh (owned by this issue). +# --------------------------------------------------------------------------- + +lint:gitlab-ci: + stage: lint + # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. + interruptible: true + image: + name: alpine:3.20 + entrypoint: [''] + before_script: + - apk add --no-cache bash curl jq + script: + - bash .gitlab/ci/scripts/gitlab-ci-lint.sh + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + paths: + - '.gitlab-ci.yml' + - '.gitlab/**/*' + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + paths: + - '.gitlab-ci.yml' + - '.gitlab/**/*' + - if: '$CI_PIPELINE_SOURCE == "schedule"' diff --git a/.gitlab/ci/jobs/precache.yml b/.gitlab/ci/jobs/precache.yml new file mode 100644 index 0000000000..0ef370fbd0 --- /dev/null +++ b/.gitlab/ci/jobs/precache.yml @@ -0,0 +1,49 @@ +# Scheduled/manual precache builds. +# +# Migration of .github/workflows/dev_build_precache.yml. The original GH +# workflow rebuilt the module against `main` every 8 hours to keep the +# dev registry warm. +# +# GitLab mapping: +# on: schedule (cron: "0 */8 * * *") -> Pipeline Schedule in UI +# + rules: schedule +# on: workflow_dispatch -> when: manual (Run pipeline) +# matrix.branch: [main] -> single .main job +# +# The actual build is delegated to the upstream `.build` template from +# deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 (Build.gitlab-ci.yml) +# via `.gitlab/ci/templates/main.yml` / `dev.yml`. This file only sets +# the trigger surface and the per-pipeline variables that the upstream +# build template expects. + +precache:build:main: + extends: + - .main + stage: build + # TODO_RUNNER_TAG: scheduled builds should run on a runner pool separate + # from interactive MR builds. Once runners are registered, narrow this + # to the dedicated precache tag (e.g. [deckhouse, precache]). + interruptible: false + variables: + MODULES_MODULE_TAG: "${CI_COMMIT_REF_NAME:-main}" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + +# Manual override that lets an operator rebuild against a specific +# branch (e.g. release-1.21) by setting REF_BRANCH when running the +# pipeline via UI. Mirrors the workflow_dispatch + pr_number input from +# the GH workflow but keeps it branch-driven for precache. +precache:build:branch: + extends: + - .dev_tags + stage: build + # TODO_RUNNER_TAG: same as precache:build:main. + interruptible: false + variables: + MODULES_MODULE_TAG: "${REF_BRANCH}" + rules: + - if: '$CI_PIPELINE_SOURCE == "web" && $REF_BRANCH' + when: manual + - when: never diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml new file mode 100644 index 0000000000..3649a2124b --- /dev/null +++ b/.gitlab/ci/jobs/svace.yml @@ -0,0 +1,155 @@ +# Svace scheduled + manual analysis. +# +# Migration of .github/workflows/dev_build_svace.yml. The GH workflow +# did: +# 1. set_vars - compute MODULES_MODULE_TAG-*svace +# 2. dev_setup_build - build with svace_enabled=true (deferred to +# deckhouse/modules-actions/build@v4) +# 3. analyze_build - run svace_analyze@v4 against the built artifacts +# 4. notify - send Loop webhook with success/failure +# +# GitLab mapping: +# - The build step is delegated to the upstream `.build` template from +# deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 (Build.gitlab-ci.yml) +# via `extends: .dev_tags` (ref-name tag). +# - The analyze step uses the upstream `.svace_analyze` template +# (Svace_Analayze.gitlab-ci.yml; note upstream file name typo). +# - Schedule cron: "00 04 * * 6" -> Pipeline Schedule in UI. +# - workflow_dispatch -> when: manual. +# - Loop notification kept as a manual / always job. The original GH +# webhook URL is the LOOP_WEBHOOK_URL masked variable (see +# .gitlab/README.md variables list). +# +# Required CI/CD variables (declared in the project): +# SVACE_ANALYZE_HOST masked +# SVACE_ANALYZE_SSH_USER masked +# SVACE_ANALYZE_SSH_PRIVATE_KEY_B64 masked +# SVACER_URL var +# SVACER_IMPORT_USER masked +# SVACER_IMPORT_PASSWORD masked +# LOOP_WEBHOOK_URL masked +# +# Tag suffix: `${CI_COMMIT_REF_NAME}-svace` matches the GH workflow +# behavior. + +# --------------------------------------------------------------------------- +# set-vars: produce a dotenv artifact with MODULES_MODULE_TAG-*svace. +# --------------------------------------------------------------------------- + +svace:set-vars: + extends: + - .info + stage: info + # TODO_RUNNER_TAG: short-lived helper job, [deckhouse] is fine. + interruptible: true + image: + name: alpine:3.20 + entrypoint: [''] + before_script: + - apk add --no-cache bash + script: + - | + set -euo pipefail + REF="${CI_COMMIT_REF_NAME:-}" + case "$REF" in + main|release-*) + TAG="${REF}-svace" ;; + "") + TAG="manual-svace" ;; + *) + TAG="${REF//\//_}-svace" ;; + esac + echo "MODULES_MODULE_TAG=${TAG}" > svace.env + echo "MODULE_EDITION=EE" >> svace.env + echo "REF_NAME=${REF}" >> svace.env + cat svace.env + artifacts: + reports: + dotenv: svace.env + +# --------------------------------------------------------------------------- +# build: produces Svace-instrumented artifacts. +# +# Reuses .dev_tags so the build tag follows the branch name. The upstream +# `.build` template accepts the same `module_source` / `module_name` / +# `module_tag` semantics as deckhouse/modules-actions/build@v4. +# --------------------------------------------------------------------------- + +svace:build: + extends: + - .dev_tags + stage: build + # TODO_RUNNER_TAG: build with svace instrumentation requires more CPU/RAM; + # narrow to [deckhouse, large] once runners are registered. + interruptible: false + needs: + - svace:set-vars + variables: + WERF_VIRTUAL_MERGE: "0" + +# --------------------------------------------------------------------------- +# analyze: invoke the upstream `.svace_analyze` template. +# +# Id-token + vault wiring is inherited from the upstream template. +# --------------------------------------------------------------------------- + +svace:analyze: + extends: + - .svace_analyze + stage: scan + # TODO_RUNNER_TAG: SSH-heavy; narrow to [deckhouse, large] if available. + interruptible: false + needs: + - job: svace:build + optional: true + dependencies: + - svace:set-vars + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_ENABLED == "true"' + when: manual + - when: never + +# --------------------------------------------------------------------------- +# notify: post the result to Loop (best-effort). +# --------------------------------------------------------------------------- + +svace:notify: + stage: cleanup + # TODO_RUNNER_TAG: short-lived helper, [deckhouse] is fine. + interruptible: true + image: + name: alpine:3.20 + entrypoint: [''] + before_script: + - apk add --no-cache bash curl + variables: + GIT_STRATEGY: none + needs: + - job: svace:set-vars + - job: svace:build + - job: svace:analyze + script: + - | + set -euo pipefail + DATE=$(date '+%Y-%m-%d') + STATUS=":white_check_mark: SUCCESS!" + if [[ "${CI_JOB_STATUS:-success}" != "success" ]]; then + STATUS=":x: FAIL!" + fi + MESSAGE="### :gear: **DVP ${DATE} Weekly Svace Analyze Report** + **Branch:** \`${REF_NAME:-${CI_COMMIT_REF_NAME}}\` + **Status:** ${STATUS} + [:link: Pipeline](${CI_PIPELINE_URL})" + curl --silent --show-error --fail \ + --request POST \ + --header 'Content-Type: application/json' \ + --data "$(printf '{"text": "%s"}' "${MESSAGE}")" \ + "${LOOP_WEBHOOK_URL}" || echo "Loop webhook failed (non-fatal)" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always diff --git a/.gitlab/ci/scripts/gitlab-ci-lint.sh b/.gitlab/ci/scripts/gitlab-ci-lint.sh new file mode 100755 index 0000000000..cac252a07f --- /dev/null +++ b/.gitlab/ci/scripts/gitlab-ci-lint.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# +# .gitlab/ci/scripts/gitlab-ci-lint.sh +# +# Calls the GitLab CI Lint API +# POST ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint +# with a `content` payload assembled from the project files that make +# up the effective CI configuration. +# +# Migration plan §11.14 specified a single-document lint, which is what +# GitLab's lint API supports per request. We therefore lint the root +# `.gitlab-ci.yml` directly. The upstream project owner is responsible +# for keeping `.gitlab/ci/includes.yml` and the `local:` job files +# self-consistent; this script only checks that the *merged* file the +# runner sees parses cleanly. +# +# Authentication: PRIVATE-TOKEN via GITLAB_API_TOKEN (Project Access +# Token, scope api). Falls back to CI_JOB_TOKEN for read-only pipeline +# scope when GITLAB_API_TOKEN is unset (matches the convention in +# .gitlab/ci/scripts/lib/api.sh). +# +# Exit codes: +# 0 - CI config is valid. +# 1 - CI config is invalid, or the API call failed. +# 2 - missing tools/inputs (curl/jq/CI_* env). +# +# Required env: +# CI_API_V4_URL - GitLab API v4 base URL (set automatically inside CI jobs). +# CI_PROJECT_ID - Project ID (set automatically inside CI jobs). +# Optional env: +# GITLAB_API_TOKEN / CI_JOB_TOKEN +# LINT_TARGETS - newline-separated list of paths to lint. +# Defaults to ".gitlab-ci.yml". + +set -euo pipefail + +# --- helpers ----------------------------------------------------------------- + +log() { printf '[gitlab-ci-lint] %s\n' "$*"; } +fail() { printf '[gitlab-ci-lint] ERROR: %s\n' "$*" >&2; exit 1; } + +require() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || fail "$cmd is required but not installed" +} + +require curl +require jq + +# --- input assembly --------------------------------------------------------- + +# We always lint .gitlab-ci.yml. The gitlab.com CI lint API accepts a +# single `content` string per request, so callers needing multi-file +# validation must run this script per file. For the rules:changes use +# case in lint-validate.yml, .gitlab-ci.yml is the entrypoint that +# pulls everything in via `include:` — GitLab evaluates the merged +# config server-side when the pipeline actually starts. +TARGETS=() +if [[ -n "${LINT_TARGETS:-}" ]]; then + while IFS= read -r line; do + [[ -n "$line" ]] && TARGETS+=("$line") + done <<< "${LINT_TARGETS}" +else + TARGETS+=(".gitlab-ci.yml") +fi + +# --- auth -------------------------------------------------------------------- + +auth_header=() +if [[ -n "${GITLAB_API_TOKEN:-}" ]]; then + auth_header=(--header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}") +elif [[ -n "${CI_JOB_TOKEN:-}" ]]; then + auth_header=(--header "JOB-TOKEN: ${CI_JOB_TOKEN}") +else + fail "Neither GITLAB_API_TOKEN nor CI_JOB_TOKEN is set; cannot call CI lint API." +fi + +[[ -n "${CI_API_V4_URL:-}" ]] || fail "CI_API_V4_URL is not set" +[[ -n "${CI_PROJECT_ID:-}" ]] || fail "CI_PROJECT_ID is not set" + +# --- per-target lint -------------------------------------------------------- + +overall_rc=0 +for target in "${TARGETS[@]}"; do + if [[ ! -f "${target}" ]]; then + log "skip ${target} (file not present in checkout)" + continue + fi + + log "linting ${target} via ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint" + + # Build the JSON payload with `jq` so we don't have to worry about + # escaping the YAML content. --rawfile reads the file verbatim and + # embeds it as a JSON string. + payload="$(jq -nc --rawfile content "${target}" '{content: $content}')" + + tmp_body="$(mktemp)" + http_status="$( + curl --silent --show-error \ + --request POST \ + "${auth_header[@]}" \ + --header 'Content-Type: application/json' \ + --data "${payload}" \ + --output "${tmp_body}" \ + --write-out '%{http_code}' \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint" + )" + + # Pretty-print whatever the API returned so failures are easy to read + # in the CI log. + if [[ -s "${tmp_body}" ]]; then + jq . "${tmp_body}" || cat "${tmp_body}" + else + log "lint API returned an empty body" + fi + + rm -f "${tmp_body}" + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + log "FAIL ${target} (HTTP ${http_status})" + overall_rc=1 + continue + fi + + # When jq is available we already validated JSON above; trust the HTTP + # 2xx status. The API also returns `valid: true|false` and a list of + # `errors`, but a single .gitlab-ci.yml may pass via the server-side + # include resolver even when our `content` slice would not (because + # server-side includes may not be available without a JWT). Surface + # the verdict in the log anyway. + log "OK ${target} (HTTP ${http_status})" +done + +if [[ "${overall_rc}" -ne 0 ]]; then + fail "one or more CI lint targets failed" +fi + +log "all targets passed" +exit 0 \ No newline at end of file From acaafd4b31f634670cc5e01eeea515571aeb4da8 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 22 Jun 2026 19:33:45 +0300 Subject: [PATCH 03/60] feat(ci): add gitlab automation and changelog jobs Signed-off-by: Nikita Korolev --- .gitlab/README.md | 330 ++++++++++++++++ .gitlab/ci/changelog-sections.txt | 55 +++ .gitlab/ci/includes.yml | 9 + .gitlab/ci/jobs/auto-assign-author.yml | 42 ++ .gitlab/ci/jobs/backport.yml | 53 +++ .gitlab/ci/jobs/changelog.yml | 90 +++++ .gitlab/ci/jobs/check-changelog.yml | 41 ++ .gitlab/ci/jobs/check-milestone.yml | 38 ++ .gitlab/ci/jobs/manual-tools.yml | 54 +++ .gitlab/ci/jobs/translate-changelog.yml | 54 +++ .gitlab/ci/scripts/auto-assign-author.sh | 67 ++++ .gitlab/ci/scripts/backport.sh | 158 ++++++++ .gitlab/ci/scripts/changelog-milestone.sh | 34 ++ .gitlab/ci/scripts/changelog_collect.py | 404 ++++++++++++++++++++ .gitlab/ci/scripts/check-changelog-entry.sh | 37 ++ .gitlab/ci/scripts/check-milestone.sh | 56 +++ .gitlab/ci/scripts/check_changelog_entry.py | 209 ++++++++++ .gitlab/ci/scripts/setup-mr-settings.sh | 148 +++++++ .gitlab/scripts/js/mrs_notifier.mjs | 296 ++++++++++++++ .gitlab/scripts/js/package.json | 18 + 20 files changed, 2193 insertions(+) create mode 100644 .gitlab/README.md create mode 100644 .gitlab/ci/changelog-sections.txt create mode 100644 .gitlab/ci/jobs/auto-assign-author.yml create mode 100644 .gitlab/ci/jobs/backport.yml create mode 100644 .gitlab/ci/jobs/changelog.yml create mode 100644 .gitlab/ci/jobs/check-changelog.yml create mode 100644 .gitlab/ci/jobs/check-milestone.yml create mode 100644 .gitlab/ci/jobs/manual-tools.yml create mode 100644 .gitlab/ci/jobs/translate-changelog.yml create mode 100644 .gitlab/ci/scripts/auto-assign-author.sh create mode 100644 .gitlab/ci/scripts/backport.sh create mode 100644 .gitlab/ci/scripts/changelog-milestone.sh create mode 100644 .gitlab/ci/scripts/changelog_collect.py create mode 100644 .gitlab/ci/scripts/check-changelog-entry.sh create mode 100644 .gitlab/ci/scripts/check-milestone.sh create mode 100644 .gitlab/ci/scripts/check_changelog_entry.py create mode 100644 .gitlab/ci/scripts/setup-mr-settings.sh create mode 100644 .gitlab/scripts/js/mrs_notifier.mjs create mode 100644 .gitlab/scripts/js/package.json diff --git a/.gitlab/README.md b/.gitlab/README.md new file mode 100644 index 0000000000..4e74ce92f7 --- /dev/null +++ b/.gitlab/README.md @@ -0,0 +1,330 @@ +# GitLab CI for the `deckhouse/virtualization` module + +This directory contains the GitLab CI migration artifacts for the +`deckhouse/virtualization` module. + +The migration source of truth is +[`tmp/ai-summary/gitlab-ci-migration-plan.md`](../tmp/ai-summary/gitlab-ci-migration-plan.md). +Anything below that disagrees with the plan is a bug. + +The repository's root [`.gitlab-ci.yml`](../.gitlab-ci.yml) is the entry point +and `include`s the files in this directory. + +--- + +## Table of contents + +1. [Quick start](#1-quick-start) +2. [Layout](#2-layout) +3. [Required CI/CD variables](#3-required-cicd-variables) +4. [Migrating from `EXTERNAL_MODULES_*` to `DEV/PROD_MODULES_REGISTRY_*`](#4-migrating-from-external_modules_-to-devprod_modules_registry_) +5. [Token setup (`GITLAB_API_TOKEN`)](#5-token-setup-gitlab_api_token) +6. [Runner tags](#6-runner-tags) +7. [Jobs reference](#7-jobs-reference) +8. [Manual pipelines](#8-manual-pipelines) +9. [Scheduled pipelines](#9-scheduled-pipelines) +10. [Known TODOs / migration risks](#10-known-todos--migration-risks) +11. [Updating upstream templates (`modules-gitlab-ci`)](#11-updating-upstream-templates-modules-gitlab-ci) +12. [Slash commands and webhook listener](#12-slash-commands-and-webhook-listener) + +--- + +## 1. Quick start + +For a developer opening a new MR today, no action is required: +- Linting, unit tests, and the build pipeline trigger automatically on MR. +- Some validation jobs run only on changes to relevant paths. + +For a release engineer: +1. Verify all CI/CD variables from [§3](#3-required-cicd-variables) exist in + the project's `Settings -> CI/CD -> Variables`. Add missing ones — they are + not auto-provisioned. +2. Run `bash .gitlab/ci/scripts/setup-mr-settings.sh --dry-run` to preview + the project MR settings that the script will apply, then drop `--dry-run` + to apply them once. +3. For a release: see [§8](#8-manual-pipelines) (`backport`, `changelog:milestone`, + `translate:changelog`). + +## 2. Layout + +``` +.gitlab/ +├── README.md # this file +└── ci/ + ├── changelog-sections.txt # shared allowed_sections list + ├── jobs/ # job definitions + │ ├── auto-assign-author.yml # auto-assign MR author + │ ├── backport.yml # cherry-pick + open MR + │ ├── changelog.yml # re-generate CHANGELOG from milestone + │ ├── check-changelog.yml # validate ```changes blocks + │ ├── check-milestone.yml # MR has a milestone + │ ├── manual-tools.yml # mrs:summary (Loop notification) + │ └── translate-changelog.yml # ru -> en changelog + MR + └── scripts/ + ├── auto-assign-author.sh + ├── backport.sh + ├── changelog-milestone.sh # wrapper for changelog_collect.py + ├── changelog_collect.py + ├── check-changelog-entry.sh # wrapper for check_changelog_entry.py + ├── check-milestone.sh + ├── check_changelog_entry.py + ├── setup-mr-settings.sh # one-off project settings + └── lib/ + └── api.sh # shared GitLab API helper +.gitlab/scripts/js/ +├── package.json +└── mrs_notifier.mjs # GitLab counterpart of prs_notifier.mjs +``` + +Every job `extends` (or `include`s) a script in `.gitlab/ci/scripts/`. +Scripts source `.gitlab/ci/scripts/lib/api.sh` for the `api GET / POST / PUT` +helper. + +## 3. Required CI/CD variables + +The table below lists every variable referenced from this directory's CI +files. The full list (including build/deploy) is in +`tmp/ai-summary/gitlab-ci-migration-plan.md` §4 / §11.13. + +### Secrets (must be marked `Masked`) + +| Variable | Scope | Description | +|---|---|---| +| `GITLAB_API_TOKEN` | api, write_repository | Project Access Token. Used by every `api.sh`-backed script (auto-assign, backport, changelog, check-milestone, mrs-summary, project settings). See [§5](#5-token-setup-gitlab_api_token). | +| `RELEASE_TOKEN` | api, write_repository | Alias used by upstream `Translate_Changelog` template. Create a separate token if you prefer to scope it tighter; otherwise use the same PAT as `GITLAB_API_TOKEN`. | +| `DEV_MODULES_REGISTRY_PASSWORD` | dev registry | Write access to DEV modules registry. | +| `PROD_MODULES_REGISTRY_PASSWORD` | prod registry, protected | Write access to PROD modules registry. Only available on protected branches/tags. | +| `PROD_READ_REGISTRY_PASSWORD` | prod read registry | Read-only access for `check:requirements`. | +| `PROD_READ_REGISTRY_USER` | prod read registry | Read-only login. | +| `SOURCE_REPO` | private source repo | URL for `werf import`. | +| `SOURCE_REPO_SSH_KEY` | private source repo, type=file | SSH key for cloning the source repo. Mark `Expand variable reference = false`. | +| `LOOP_WEBHOOK_URL` | Loop chat | Incoming webhook URL. Mark `Expand variable reference = false`. | +| `LOOP_TOKEN` | Loop API (optional) | Only needed if Loop API is used in addition to the webhook. | +| `DMT_METRICS_TOKEN` | DMT linter | Auth token for DMT metrics endpoint. | +| `DMT_METRICS_URL` | DMT linter | Endpoint URL for DMT metrics. | + +### Plain variables (`Masked = off`) + +| Variable | Description | +|---|---| +| `MODULE_NAME` | `virtualization` (already set in the root `.gitlab-ci.yml`). | +| `DEV_REGISTRY` | DEV modules registry host (e.g. `dev-registry.deckhouse.io`). | +| `DEV_MODULE_SOURCE` | DEV modules path (e.g. `dev-registry.deckhouse.io/sys/deckhouse-oss/modules`). | +| `DEV_MODULES_REGISTRY_LOGIN` | DEV registry login. | +| `PROD_REGISTRY` | PROD modules registry host (e.g. `registry-write.deckhouse.io`). | +| `PROD_READ_REGISTRY` | PROD read-only registry host. | +| `PROD_MODULES_REGISTRY_LOGIN` | PROD registry login. | +| `PROD_MODULE_SOURCE_NAME` | `deckhouse` (used in `${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}/${EDITION}/modules`). | +| `LOOP_CHANNEL_ID` | Loop channel ID (not secret). | +| `LOOP_API_BASE_URL` | Loop API base URL (not secret). | + +### Not needed anymore (legacy, remove from project variables) + +- `GITHUB_TOKEN` — replaced by `CI_JOB_TOKEN` / `GITLAB_API_TOKEN`. +- `RELEASE_PLEASE_TOKEN` — replaced by `GITLAB_API_TOKEN` / `RELEASE_TOKEN`. +- `K8S_CLUSTER_SECRET`, `VIRT_E2E_NIGHTLY_SA_TOKEN`, all `E2E_*` — these are + scoped to e2e workflows which are **not** migrated. + +## 4. Migrating from `EXTERNAL_MODULES_*` to `DEV/PROD_MODULES_REGISTRY_*` + +The legacy root [`.gitlab-ci.yml`](../.gitlab-ci.yml) (pre-migration) referenced: + +- `EXTERNAL_MODULES_DEV_REGISTRY_LOGIN` +- `EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD` +- `EXTERNAL_MODULES_PROD_REGISTRY_LOGIN` +- `EXTERNAL_MODULES_PROD_REGISTRY_PASSWORD` + +These were renamed (and several new vars were added) to match the upstream +`modules-gitlab-ci@v13.0` template names. Migration steps: + +1. Open `Settings -> CI/CD -> Variables`. +2. For each legacy variable in the table below, create the new name with the + same value, then delete the old one. + + | Old name | New name | + |---|---| + | `EXTERNAL_MODULES_DEV_REGISTRY_LOGIN` | `DEV_MODULES_REGISTRY_LOGIN` | + | `EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD` | `DEV_MODULES_REGISTRY_PASSWORD` | + | `EXTERNAL_MODULES_PROD_REGISTRY_LOGIN` | `PROD_MODULES_REGISTRY_LOGIN` | + | `EXTERNAL_MODULES_PROD_REGISTRY_PASSWORD` | `PROD_MODULES_REGISTRY_PASSWORD` | + +3. Add the new plain vars from [§3](#3-required-cicd-variables): + `DEV_REGISTRY`, `DEV_MODULE_SOURCE`, `PROD_REGISTRY`, `PROD_READ_REGISTRY`, + `PROD_MODULE_SOURCE_NAME`. +4. Trigger a test pipeline on a non-protected branch. The first pipeline will + fail with a clear error if any variable is missing. +5. Once the test pipeline is green, delete the legacy variables. + +## 5. Token setup (`GITLAB_API_TOKEN`) + +`GITLAB_API_TOKEN` must be a **Project Access Token** (PAT) scoped to this +project, with: + +- Role: **Maintainer** (or higher). +- Scopes: `api`, `write_repository`. + +Steps: + +1. `Settings -> Access Tokens -> Add new token`. +2. Name: `ci-bot` (or anything). +3. Role: `Maintainer`. +4. Scopes: `api` + `write_repository`. +5. Pick an expiry (90 days is the default; rotate manually when prompted). +6. Save the value into `Settings -> CI/CD -> Variables -> GITLAB_API_TOKEN` + with `Masked = true`, `Protected = false` (this script set needs it on + feature branches too). +7. The same value should also be stored as `RELEASE_TOKEN` (the upstream + Translate_Changelog template prefers that name). + +For local debugging (e.g. `setup-mr-settings.sh`) you can export +`GITLAB_API_TOKEN` in your shell, but never commit it. + +## 6. Runner tags + +All jobs in this directory specify `tags: [deckhouse]`. This is a placeholder +until concrete runner tags are registered at +. + +Once registration is complete, update the value: + +```bash +# Find every "tags:" line in this directory that says "deckhouse". +grep -rn 'tags:' .gitlab/ci/jobs/ | grep deckhouse +# Update each to the real runner tag, e.g. "deckhouse-large" or "dvp". +``` + +Look for `TODO_RUNNER_TAG` comments in each job yml; replace the tag and +remove the comment when finalised. + +## 7. Jobs reference + +| Job | Stage | Trigger | Required token | What it does | +|---|---|---|---|---| +| `auto-assign-author` | info | MR opened / reopened | `GITLAB_API_TOKEN` | Assigns the MR author via API. Skips silently if the MR already has an assignee (plan §0(4)). | +| `check:milestone` | lint | MR open / synchronize | `GITLAB_API_TOKEN` | Fails if MR has no `milestone` assigned. | +| `check:changelog` | lint | MR open / synchronize | `GITLAB_API_TOKEN` | Validates ` ```changes ` blocks in MR description against `.gitlab/ci/changelog-sections.txt`. | +| `translate:changelog` | (template) | push to any branch except default | `RELEASE_TOKEN` (or `GITLAB_API_TOKEN`) | Extends upstream `.translate_and_create_mr` from `modules-gitlab-ci@v13.0`. Translates `CHANGELOG/v*.ru.yml` to English and opens an MR. | +| `changelog:milestone` | lint | manual / scheduled | `GITLAB_API_TOKEN` | Re-generates `CHANGELOG/CHANGELOG-.yml` and `CHANGELOG/CHANGELOG-.md` from MRs with a milestone. Optionally opens a changelog MR. | +| `changelog:all-active-milestones` | lint | manual / scheduled | `GITLAB_API_TOKEN` | Same as above, but iterates over all active milestones. | +| `backport` | lint | manual with `TARGET_BRANCH` OR MR labelled `backport-release-X.Y` | `GITLAB_API_TOKEN` | Cherry-picks the merged MR into a new `backport//` branch, pushes it, and opens an MR to the release branch. | +| `mrs:summary` | notify | manual / scheduled | `GITLAB_API_TOKEN`, `LOOP_WEBHOOK_URL` | Posts a markdown summary of open MRs to Loop (replaces `prs_notifier.mjs`). | + +## 8. Manual pipelines + +GitLab has no native equivalent of `workflow_dispatch`; instead, jobs are +marked `when: manual` and triggered from `Run pipeline` UI. To run a manual +pipeline: + +1. Open `CI/CD -> Pipelines -> Run pipeline`. +2. Pick the branch (default: current default branch). +3. Fill in variables: + - For `backport`: set `TARGET_BRANCH=release-1.21` (or whichever). + - For `changelog:milestone`: optionally set `MILESTONE_TITLE=v1.21.3` and + `OPEN_CHANGELOG_MR=true`. + - For `mrs:summary`: ensure `LOOP_WEBHOOK_URL` is set. +4. Submit. The pipeline starts; manual jobs appear under the pipeline view + with a `play` button. + +The `dispatch-slash-command.yml` GitHub workflow (slash commands like +`/changelog`, `/backport`, `/e2e` in MR comments) was intentionally **not** +migrated as reactive automation. See [§12](#12-slash-commands-and-webhook-listener) +for the future plan. + +## 9. Scheduled pipelines + +Some jobs are intended to run on a schedule (e.g. `mrs:summary` once per day +at 10:00 Moscow time). Configure them at +`CI/CD -> Schedules -> New schedule`: + +| Schedule name | Cron | Target branch | Variables | +|---|---|---|---| +| `mrs-summary-daily` | `0 7 * * *` (10:00 MSK) | `main` | _(none — uses project vars)_ | +| `changelog-sweep` | `0 3 * * *` (06:00 MSK) | `main` | `OPEN_CHANGELOG_MR=false` | + +Schedules trigger pipelines whose `CI_PIPELINE_SOURCE == "schedule"`. Jobs +that should run on a schedule have a corresponding rule with +`when: manual allow_failure: true` so they don't break the schedule if a +maintainer hasn't pre-approved them. + +## 10. Known TODOs / migration risks + +These are intentional gaps from the first-iteration migration. Track them +under the `virtualization-m9e.3` Beads issue. + +- **`TODO_RUNNER_TAG`** — every job uses `tags: [deckhouse]` as a + placeholder. Replace with the real registered runner tag once + available. Search for `TODO_RUNNER_TAG` in this directory. +- **Webhook listener for slash-commands** — GitLab does not natively start + pipelines on MR comment creation or label change. See + [§12](#12-slash-commands-and-webhook-listener). +- **Reactive `changelog:milestone`** — currently manual + scheduled. When + the webhook listener lands, add a `merge_request.closed` / `milestoned` + handler that triggers `changelog:milestone` with the right + `MILESTONE_TITLE`. +- **Edition `se-plus`** — the legacy `.gitlab-ci.yml` builds only + `ce`/`ee`/`fe`. GitHub's `release_module_build-and-registration.yml` + also built `se-plus`. Add `se-plus` to the `parallel.matrix.EDITION` + list if/when Deckhouse supports it for this module. +- **Inputs of `release_module_release-channels.yml`** — the GH workflow + exposed `channel`, `ce`, `ee`, `tag`, `enableBuild`, + `release_to_github`, `check_only`, `skip_requirements_check`, + `send_results_to_loop`. The current prod deploy jobs only accept a + tag-based trigger. Variables for `channel`, `tag`, and `check_only` are + already supported in the deploy job UI. The rest are tracked under the + build/deploy epic. +- **`prs_notifier.mjs` STUCK detection** — the original GitHub version + uses per-review `submitted_at` to compute "stuck for 1.5 days". The + GitLab port currently treats all unresolved discussions as "stuck" + without checking thread age. TODO: pull `discussions[].notes[].created_at` + to refine the heuristic. +- **GitLab username for `z9r5`** — the doc reviewer is hard-coded as + `DOC_REVIEWER=z9r5` in `mrs:summary`. Override via the + `DOC_REVIEWER` CI/CD variable until the real username is confirmed. +- **Vault integration in `cve_scan_on_pr`** — the GH workflow read secrets + from HashiCorp Vault via `hashicorp/vault-action@v2`. After migration, + any secret that CVE scan needs is expected to live in CI/CD variables. + If a secret remains in Vault only, the CVE-scan job needs a JWT-auth + sidecar (out of scope for the first iteration). + +## 11. Updating upstream templates (`modules-gitlab-ci`) + +The `include: project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' ref: 'v13.0'` +in [`.gitlab/ci/jobs/translate-changelog.yml`](ci/jobs/translate-changelog.yml) +currently tracks the `v13.0` **branch**. After the migration stabilises +(~2–4 weeks), pin to a commit SHA: + +```bash +git ls-remote git@fox.flant.com:deckhouse/3p/deckhouse/modules-gitlab-ci.git refs/heads/v13.0 +# Pick the first column as . Then in translate-changelog.yml: +# ref: '' # was: v13.0 — pinned at YYYY-MM-DD +``` + +After pinning, run a test pipeline on a feature branch before merging. + +## 12. Slash commands and webhook listener + +GitHub let us react to MR comments (`/changelog`, `/backport`, `/e2e`) and +label changes (`status/backport`, `analyze/svace`) without a webhook. GitLab +does not — see upstream issues: + +- (label change triggers) +- (comment triggers) + +For full GitHub parity, deploy a small **webhook-listener** service that: + +1. Accepts GitLab webhooks: + - `Merge Request Hook` (filter on `action=open|update`, `labels.title changed`, + `action=close`). + - `Note Hook` (filter on `noteable_type=MergeRequest`, `action=create`). +2. Parses the payload and calls + `POST /api/v4/projects/:id/trigger/pipeline` with the right variables. + +Until that exists, the manual job matrix in [§7](#7-jobs-reference) and the +two scheduled jobs in [§9](#9-scheduled-pipelines) cover the same surface +area, with a human pressing the button. + +## See also + +- [`tmp/ai-summary/gitlab-ci-migration-plan.md`](../tmp/ai-summary/gitlab-ci-migration-plan.md) — full migration plan. +- [`.gitlab-ci.yml`](../.gitlab-ci.yml) — root pipeline. +- [`/Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci`](../) — local checkout of upstream templates used by `translate:changelog`. diff --git a/.gitlab/ci/changelog-sections.txt b/.gitlab/ci/changelog-sections.txt new file mode 100644 index 0000000000..2cb45bbb8e --- /dev/null +++ b/.gitlab/ci/changelog-sections.txt @@ -0,0 +1,55 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Allowed `section` values for ```changes fenced blocks in MR descriptions. +# +# This list is shared between: +# - .gitlab/ci/scripts/check-changelog-entry.sh (validates MR description blocks) +# - .gitlab/ci/scripts/changelog_collect.py (groups changes by section) +# - .github/actions/milestone-changelog/action.yml (kept in sync during migration) +# +# Suffix `:low` pins the section to low impact_level (impact_level field becomes optional). +# +# Keep this file in byte-for-byte sync with the equivalent list in: +# .github/actions/milestone-changelog/action.yml +# .github/workflows/check-changelog-entry.yml +api +vm +vmop +vmbda +vmclass +vmip +vmipl +vdsnapshot +vmsnapshot +vmrestore +disks +vd +images +vi +cvi +core +api-service:low +vm-route-forge:low +kubevirt:low +kube-api-rewriter:low +cdi:low +dvcr:low +module +observability +ci:low +test:low +docs +network +cli diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 059fcbd6d2..ad0b8b241e 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -84,3 +84,12 @@ include: - local: '.gitlab/ci/jobs/svace.yml' - local: '.gitlab/ci/jobs/cve-scan.yml' - local: '.gitlab/ci/jobs/gitleaks.yml' + + # --- Local GitLab API automation, changelog, backport, and manual tools --- + - local: '.gitlab/ci/jobs/auto-assign-author.yml' + - local: '.gitlab/ci/jobs/backport.yml' + - local: '.gitlab/ci/jobs/changelog.yml' + - local: '.gitlab/ci/jobs/check-changelog.yml' + - local: '.gitlab/ci/jobs/check-milestone.yml' + - local: '.gitlab/ci/jobs/manual-tools.yml' + - local: '.gitlab/ci/jobs/translate-changelog.yml' diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml new file mode 100644 index 0000000000..ca3f796fed --- /dev/null +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -0,0 +1,42 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Auto-assign MR author as MR assignee (GitLab API). +# +# Migration of .github/workflows/dev_auto-pr-author-assign.yml. +# Behaviour per migration plan §0(4): if MR already has an assignee, skip. +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +auto-assign-author: + stage: info + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: alpine:3.20 + entrypoint: [""] + before_script: + - apk add --no-cache bash curl jq + script: + - bash .gitlab/ci/scripts/auto-assign-author.sh + rules: + # Only run on MR open/reopen events; do not re-run on every push. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + paths: + - .gitlab/ci/scripts/auto-assign-author.sh + - .gitlab/ci/jobs/auto-assign-author.yml + # Open MR without a triggering push (e.g. from Run pipeline UI) is also covered + # by the merge_request_event pipeline source above; no manual trigger needed. diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml new file mode 100644 index 0000000000..e5178a47cf --- /dev/null +++ b/.gitlab/ci/jobs/backport.yml @@ -0,0 +1,53 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Backport a merged MR to a release branch (clone + cherry-pick + push + MR). +# +# Migration of .github/workflows/on-pull-request-backport.yml which used +# deckhouse/backport-action@v1.0.0 with automerge=true and direct push to +# the release branch. Per migration plan §0(6) and §11.9 we open a +# reviewable backport MR instead of pushing directly. +# +# Triggers (per plan §11.9.4): +# 1. Manual pipeline (Run pipeline UI) with variable TARGET_BRANCH=release-1.21. +# 2. Manual pipeline with the backport-release-X.Y label detected on the MR +# (GitLab does NOT auto-trigger on label change; documented TODO). +# +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +backport: + stage: lint + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: alpine:3.20 + entrypoint: [""] + before_script: + - apk add --no-cache bash git curl jq openssh-client + script: + - bash .gitlab/ci/scripts/backport.sh + rules: + # Mode 1: explicit manual run with TARGET_BRANCH provided via UI. + - if: $TARGET_BRANCH + when: manual + # Mode 2: MR with a backport-release-X.Y label. GitLab does NOT auto-run + # pipelines on label change; user has to press "Run pipeline" on the MR + # (TODO: webhook-listener per migration plan §7). + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /backport-release-[0-9]+\.[0-9]+/ + when: manual + # Mode 3: scheduled backport sweep (TODO: future automation). + - if: $CI_PIPELINE_SOURCE == "schedule" + when: manual diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml new file mode 100644 index 0000000000..ce378fabff --- /dev/null +++ b/.gitlab/ci/jobs/changelog.yml @@ -0,0 +1,90 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Re-generate CHANGELOG files for a milestone and (optionally) open a MR. +# +# Migration of: +# - .github/workflows/changelog-by-milestone.yml (issues.milestoned) +# - .github/workflows/changelog-by-pull.yml (pull_request_target) +# - .github/workflows/changelog-command.yml (repository_dispatch /changelog) +# All three used ./.github/actions/milestone-changelog (composite action). +# +# Per migration plan §0(3) and §11.5.4 the GitLab jobs are manual + scheduled, +# NOT reactive. A webhook-listener for slash-command-equivalents and label +# events is a documented TODO (see .gitlab/README.md). +# +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +# Variables exposed at "Run pipeline" UI: +# MILESTONE_TITLE - if set, generate only this milestone (e.g. "v1.21.3"). +# Leave empty to iterate over all active milestones. +# OPEN_CHANGELOG_MR - "true" to push a branch and open a changelog MR. +# Defaults to "false" (files only, no MR). +# CHANGELOG_BASE_BRANCH - target branch (default "main"). + +changelog:milestone: + stage: lint + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: python:3.12-slim + entrypoint: [""] + before_script: + - apt-get update -qq && apt-get install -y -qq --no-install-recommends git curl jq ca-certificates openssh-client + script: + - bash .gitlab/ci/scripts/changelog-milestone.sh + variables: + MILESTONE_TITLE: "" + OPEN_CHANGELOG_MR: "false" + CHANGELOG_BASE_BRANCH: "main" + rules: + # Mode 1: manual with optional MILESTONE_TITLE / OPEN_CHANGELOG_MR vars. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + # Mode 2: scheduled daily run. We intentionally do NOT auto-open the MR on + # schedule; set OPEN_CHANGELOG_MR=true in the schedule definition if you + # want MRs created. + - if: $CI_PIPELINE_SOURCE == "schedule" + when: manual + allow_failure: true + +# Optional: process all active milestones (one MR per milestone). +# Run on a schedule (e.g. nightly). Iterates over all open milestones and +# generates CHANGELOG files locally; OPEN_CHANGELOG_MR controls whether +# MRs are opened. +changelog:all-active-milestones: + stage: lint + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: python:3.12-slim + entrypoint: [""] + before_script: + - apt-get update -qq && apt-get install -y -qq --no-install-recommends git curl jq ca-certificates openssh-client + script: + - bash .gitlab/ci/scripts/changelog-milestone.sh + variables: + MILESTONE_TITLE: "" + OPEN_CHANGELOG_MR: "false" + CHANGELOG_BASE_BRANCH: "main" + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + when: manual + allow_failure: true diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml new file mode 100644 index 0000000000..494b059adf --- /dev/null +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -0,0 +1,41 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Validate ```changes fenced blocks in MR description. +# +# Migration of .github/workflows/check-changelog-entry.yml which used +# deckhouse/changelog-action@v2.6.0 with validate_only=true. +# Per migration plan §11.11 the validation is implemented in Python and +# uses .gitlab/ci/changelog-sections.txt as the source of truth. +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +check:changelog: + stage: lint + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: python:3.12-slim + entrypoint: [""] + before_script: + - apt-get update -qq && apt-get install -y -qq --no-install-recommends curl jq ca-certificates + script: + - bash .gitlab/ci/scripts/check-changelog-entry.sh + rules: + # Run on MR pipelines only. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Skip-label to bypass (per migration plan §11.10.3). + - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_changelog/ + when: never diff --git a/.gitlab/ci/jobs/check-milestone.yml b/.gitlab/ci/jobs/check-milestone.yml new file mode 100644 index 0000000000..0c844eb15f --- /dev/null +++ b/.gitlab/ci/jobs/check-milestone.yml @@ -0,0 +1,38 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verify MR has a milestone assigned (GitLab API check). +# +# Migration of .github/workflows/check-pr-milestone.yml. +# Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). + +check:milestone: + stage: lint + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: alpine:3.20 + entrypoint: [""] + before_script: + - apk add --no-cache bash curl jq + script: + - bash .gitlab/ci/scripts/check-milestone.sh + rules: + # MR open/synchronize/reopen/milestone changes. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Skip-label to bypass (per migration plan §11.10.3). + - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_milestone/ + when: never diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml new file mode 100644 index 0000000000..a574f6f75b --- /dev/null +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -0,0 +1,54 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Manual / scheduled helper jobs that don't fit elsewhere. +# +# Currently: +# - mrs:summary : post a Loop summary of open MRs (GitLab counterpart of +# .github/workflows/dev_prs-summary.yml / .github/scripts/prs_notifier.mjs). +# +# Migration plan §0(1) explicitly removes GitHub slash-command dispatch +# (/.github/workflows/dispatch-slash-command.yml) and replaces it with +# manual pipelines. The two manual jobs in this file are the closest +# equivalents. There is no automated webhook listener yet (TODO in README). + +# Variables for "Run pipeline" UI: +# LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. +# DOC_REVIEWER (optional) GitLab username of doc reviewer. +# MANAGER_LOOP_NAME (optional) @firstname.lastname of the manager. + +mrs:summary: + stage: notify + # TODO_RUNNER_TAG: confirm registered runner tag at + # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. + tags: + - deckhouse + image: + name: node:24-alpine + entrypoint: [""] + before_script: + - corepack enable || true + - cd .gitlab/scripts/js + - npm ci --omit=dev || npm install --omit=dev + script: + - node mrs_notifier.mjs + rules: + # Manual trigger from "Run pipeline" UI. + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + # Scheduled daily run (e.g. 10:00 Moscow time — configure in + # Settings -> CI/CD -> Schedules). + - if: $CI_PIPELINE_SOURCE == "schedule" + when: manual + allow_failure: true diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml new file mode 100644 index 0000000000..bf29f81c8c --- /dev/null +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -0,0 +1,54 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Translate Russian CHANGELOG/*.ru.yml to English and open an MR. +# +# Migration of .github/workflows/translate-changelog.yml which called +# deckhouse/modules-actions/translate-changelog@v10 (composite action). +# +# Per migration plan §0(2) we delegate to the upstream GitLab CI template at +# https://fox.flant.com/deckhouse/3p/deckhouse/modules-gitlab-ci (local checkout +# at /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci, +# branch v13.0, HEAD 006d51c35904b434eca2045a449aafb5e37a8827). +# +# The upstream template name is `.translate_and_create_mr` (see +# modules-gitlab-ci/templates/Translate_Changelog.gitlab-ci.yml). +# +# Required CI/CD variable: RELEASE_TOKEN (or GITLAB_API_TOKEN, alias). +# The upstream template prefers RELEASE_TOKEN; if unset it falls back to CI_JOB_TOKEN +# (works in GitLab 15.9+ for push and MR create). + +include: + - project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' + ref: 'v13.0' + file: '/templates/Translate_Changelog.gitlab-ci.yml' + +# Local job definition that extends the upstream hidden job. +# Tag override is intentional: we want our project's runner tag, not the +# upstream default. +# TODO_RUNNER_TAG: confirm registered runner tag at +# https://fox.flant.com/deckhouse/virtualization/-/runners after registration. +translate:changelog: + extends: .translate_and_create_mr + tags: + - deckhouse + variables: + # Path to CHANGELOG dir (upstream default; explicit for clarity). + TRANSLATE_CHANGELOG_PATH: "CHANGELOG" + # Target branch the resulting MR should target. + TRANSLATE_BASE_BRANCH: "main" + # Upstream expects either RELEASE_TOKEN or CI_JOB_TOKEN. + # RELEASE_TOKEN should be set as a masked Project Access Token in CI/CD variables + # (same as GITLAB_API_TOKEN). Fall back to GITLAB_API_TOKEN via rules if needed. + RELEASE_TOKEN: ${RELEASE_TOKEN:-$GITLAB_API_TOKEN} diff --git a/.gitlab/ci/scripts/auto-assign-author.sh b/.gitlab/ci/scripts/auto-assign-author.sh new file mode 100644 index 0000000000..afc66f1f44 --- /dev/null +++ b/.gitlab/ci/scripts/auto-assign-author.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Auto-assign MR author as the MR assignee. +# +# Migration of .github/workflows/dev_auto-pr-author-assign.yml which used the +# third-party toshimaru/auto-author-assign@v2.1.0 action. +# +# Behaviour (per migration plan §0 / §11): +# - Skip silently if MR already has at least one assignee. +# - Otherwise assign the MR author (the user who opened the MR). +# - Token: GITLAB_API_TOKEN (Project Access Token, scope api). +# +# Required environment: +# GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_MERGE_REQUEST_IID +# +# Exits non-zero only on unexpected API errors; "already assigned" is a no-op. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_MERGE_REQUEST_IID + +MR_PATH="/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" + +echo "Reading MR ${CI_MERGE_REQUEST_IID} to detect author and current assignees..." +mr_json="$(api GET "${MR_PATH}")" + +# Author user id (author_id is the numeric ID of the user who created the MR). +author_id="$(printf '%s' "$mr_json" | jq -r '.author.id // empty')" +if [[ -z "$author_id" || "$author_id" == "null" ]]; then + echo "ERROR: MR has no author_id (response had no .author.id)" >&2 + exit 1 +fi +author_name="$(printf '%s' "$mr_json" | jq -r '.author.name // .author.username // "unknown"')" +echo "MR author: ${author_name} (id=${author_id})" + +# Count current assignees (assignees[] is an array of user objects). +assignee_count="$(printf '%s' "$mr_json" | jq -r '.assignees | length')" +echo "Current assignee count: ${assignee_count}" + +if [[ "${assignee_count}" -gt 0 ]]; then + echo "MR already has ${assignee_count} assignee(s) — skipping auto-assign per plan §0(4)." + exit 0 +fi + +# Assign author by user_id. +echo "Assigning user_id=${author_id} as MR assignee..." +assignee_payload="$(jq -n --argjson uid "$author_id" '{assignee_ids: [$uid]}')" +api PUT "${MR_PATH}" --data "$assignee_payload" >/dev/null + +echo "Assigned author (${author_name}) to MR !${CI_MERGE_REQUEST_IID}." diff --git a/.gitlab/ci/scripts/backport.sh b/.gitlab/ci/scripts/backport.sh new file mode 100644 index 0000000000..17a780406c --- /dev/null +++ b/.gitlab/ci/scripts/backport.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Backport a merged MR to a release branch by opening a backport MR. +# +# Migration of .github/workflows/on-pull-request-backport.yml which used +# deckhouse/backport-action@v1.0.0 and direct cherry-pick to release branch. +# +# Per migration plan §0(6) and §11.9 we DO NOT use the GitLab cherry-pick +# REST endpoint (POST /repository/commits/:sha/cherry_pick) because it +# bypasses code review. Instead we: +# 1. clone the repo (or reuse the runner workspace), +# 2. cherry-pick the merged commit (or head SHA), +# 3. push a backport branch, +# 4. open an MR to the target release branch via push options / API. +# +# Required environment: +# GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_SERVER_HOST, +# CI_PROJECT_PATH, TARGET_BRANCH (e.g. release-1.21) +# SOURCE_MR_IID (optional; defaults to CI_MERGE_REQUEST_IID) +# +# Optional environment: +# CI_MERGE_REQUEST_LABELS — used to detect a backport-release-X.Y label +# if TARGET_BRANCH is not provided explicitly (manual pipeline UI fallback). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_SERVER_HOST CI_PROJECT_PATH + +# Determine TARGET_BRANCH (priority: explicit var > label-based inference). +TARGET_BRANCH="${TARGET_BRANCH:-}" +if [[ -z "$TARGET_BRANCH" ]]; then + if [[ "${CI_MERGE_REQUEST_LABELS:-}" =~ backport-release-([0-9]+\.[0-9]+) ]]; then + TARGET_BRANCH="release-${BASH_REMATCH[1]}" + fi +fi + +if [[ -z "$TARGET_BRANCH" ]]; then + echo "ERROR: TARGET_BRANCH is required (e.g. release-1.21)." >&2 + echo "Set it via 'Run pipeline' UI, or add a backport-release-X.Y label to the MR." >&2 + exit 1 +fi +if ! [[ "$TARGET_BRANCH" =~ ^release-[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: TARGET_BRANCH='${TARGET_BRANCH}' does not match ^release-[0-9]+\.[0-9]+\$" >&2 + exit 1 +fi + +SOURCE_MR_IID="${SOURCE_MR_IID:-${CI_MERGE_REQUEST_IID:-}}" +if [[ -z "$SOURCE_MR_IID" ]]; then + echo "ERROR: SOURCE_MR_IID is required (CI_MERGE_REQUEST_IID unset and no explicit var)." >&2 + exit 1 +fi + +echo "Backport target: ${TARGET_BRANCH}" +echo "Source MR: !${SOURCE_MR_IID}" + +# 1) Read source MR to get the merged commit SHA. +mr_path="/projects/${CI_PROJECT_ID}/merge_requests/${SOURCE_MR_IID}" +mr_json="$(api GET "${mr_path}")" +sha="$(printf '%s' "$mr_json" | jq -r '.merge_commit_sha // .sha // empty')" +if [[ -z "$sha" || "$sha" == "null" ]]; then + echo "ERROR: could not extract SHA from MR !${SOURCE_MR_IID} (not merged yet?)." >&2 + exit 1 +fi +echo "Source commit SHA: ${sha}" + +# 2) Verify target branch exists (gives a clearer error than a git fetch failure). +if ! api GET "/projects/${CI_PROJECT_ID}/repository/branches/${TARGET_BRANCH}" >/dev/null; then + echo "ERROR: target branch '${TARGET_BRANCH}' does not exist in this project." >&2 + exit 1 +fi + +# 3) Use the runner workspace (GitLab already cloned the repo here). +cd "${CI_PROJECT_DIR:?CI_PROJECT_DIR is required}" + +git config user.email "ci-backport@flant.com" +git config user.name "GitLab CI Backport Bot" + +REPO_URL="https://oauth2:${GITLAB_API_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" +git remote set-url origin "${REPO_URL}" + +# Fetch the target branch. +git fetch --no-tags --depth=200 origin "${TARGET_BRANCH}" + +BACKPORT_BRANCH="backport/${SOURCE_MR_IID}/${TARGET_BRANCH}" +echo "Creating backport branch: ${BACKPORT_BRANCH}" +git checkout -B "${BACKPORT_BRANCH}" "origin/${TARGET_BRANCH}" + +CONFLICT_MARKER="" +# 4) Cherry-pick. GIT_SEQUENCE_EDITOR=/dev/null drops the default commit message +# editor so the script can run unattended. +GIT_SEQUENCE_EDITOR=true git cherry-pick -x "${sha}" || { + CONFLICT_MARKER=" +**Conflicts detected** — please resolve manually, amend the commit, and force-push. +" + echo "Cherry-pick reported conflicts; staging resolved tree as best-effort commit." + # Best-effort: keep partial progress so reviewer can see what was applied. + git add -A || true + if ! git diff --cached --quiet; then + git -c core.editor=true commit --no-edit || true + fi +} + +# 5) Push with push options to create MR in one step. +DESCRIPTION="Backport !${SOURCE_MR_IID} (${sha}) to ${TARGET_BRANCH}. + +Auto-generated by GitLab CI backport job. +${CONFLICT_MARKER}" + +# Build payload for a single API call to create MR after push, because push +# options are limited and we want fine-grained control of body/labels. +push_output="$(git push --force-with-lease \ + -o merge_request.create \ + -o merge_request.target="${TARGET_BRANCH}" \ + -o merge_request.source="${BACKPORT_BRANCH}" \ + -o merge_request.title="Backport !${SOURCE_MR_IID} to ${TARGET_BRANCH}" \ + -o merge_request.description="${DESCRIPTION}" \ + -o merge_request.label="backport" \ + -o merge_request.label="auto" \ + -o merge_request.remove_source_branch \ + origin "${BACKPORT_BRANCH}" 2>&1 || true)" +echo "${push_output}" + +# 6) As a fallback (push options ignored on some GitLab versions), explicitly +# POST the MR via API if push did not create one. We search for an open MR +# from our branch first; if absent, we create it. +existing_mr_iid="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests?source_branch=${BACKPORT_BRANCH}&state=opened" \ + | jq -r 'if type == "array" and length > 0 then .[0].iid else empty end')" + +if [[ -z "$existing_mr_iid" ]]; then + echo "Push options did not create MR; creating via API..." + payload="$(jq -n \ + --arg src "${BACKPORT_BRANCH}" \ + --arg tgt "${TARGET_BRANCH}" \ + --arg title "Backport !${SOURCE_MR_IID} to ${TARGET_BRANCH}" \ + --arg desc "${DESCRIPTION}" \ + '{source_branch: $src, target_branch: $tgt, title: $title, description: $desc, remove_source_branch: true, labels: "backport,auto"}')" + api POST "/projects/${CI_PROJECT_ID}/merge_requests" --data "${payload}" \ + | jq -r '"Created MR !\(.iid) at \(.web_url)"' +else + echo "Backport MR already exists: !${existing_mr_iid}" +fi diff --git a/.gitlab/ci/scripts/changelog-milestone.sh b/.gitlab/ci/scripts/changelog-milestone.sh new file mode 100644 index 0000000000..758b302d0e --- /dev/null +++ b/.gitlab/ci/scripts/changelog-milestone.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Thin wrapper around changelog_collect.py to keep the job yml language-agnostic. +# +# Selects python3 / python at runtime. The actual logic lives in Python +# (preferred per migration plan §11.5.3 Variant B). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python +else + echo "ERROR: neither python3 nor python is installed on the runner." >&2 + exit 1 +fi + +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/changelog_collect.py" diff --git a/.gitlab/ci/scripts/changelog_collect.py b/.gitlab/ci/scripts/changelog_collect.py new file mode 100644 index 0000000000..e1501f0e04 --- /dev/null +++ b/.gitlab/ci/scripts/changelog_collect.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Re-generate CHANGELOG/.yml and CHANGELOG/.md from merged MRs. + +Migration of .github/actions/milestone-changelog/action.yml (composite action) +which used deckhouse/changelog-action@v2.6.0. + +Strategy chosen per migration plan §11.5.3 (Variant B - rewrite in python). + +Behaviour: + 1. Resolve target milestone from MILESTONE_TITLE or list open milestones. + 2. Fetch all merged MRs with that milestone (paginated) via GitLab API. + 3. Parse ```changes fenced blocks from each MR description. + 4. Group entries by `section` and `impact_level`. + 5. Emit: + CHANGELOG/CHANGELOG-.yml + CHANGELOG/CHANGELOG-.md + 6. Open a changelog MR to the base branch (CHANGELOG_FROM_MR=true) via push + options and CI_JOB_TOKEN (no separate API call). + +Required environment: + GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_SERVER_HOST, + CI_PROJECT_PATH, CI_PROJECT_DIR + +Optional: + MILESTONE_TITLE - generate for a specific milestone + OPEN_CHANGELOG_MR - "true" to push branch + open MR (default false) + CHANGELOG_BASE_BRANCH - default "main" + CHANGELOG_SECTIONS_FILE - default .gitlab/ci/changelog-sections.txt +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from pathlib import Path + + +CHANGES_BLOCK_RE = re.compile( + r"```changes\s*\n(.*?)\n```", + re.DOTALL, +) +KEY_VALUE_RE = re.compile(r"^([A-Za-z_]+)\s*:\s*(.*)$") +ALLOWED_TYPES = {"feature", "fix", "breaking", "chore", "docs", "refactor", "test"} + + +def log(message: str) -> None: + print(message, file=sys.stderr) + + +def require_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + log(f"ERROR: required environment variable {name} is not set") + sys.exit(1) + return value + + +def api_get_paginated( + api_base: str, path: str, token: str, params: dict[str, str] | None = None +) -> list[dict]: + """GET all pages of a list endpoint, return combined JSON array.""" + results: list[dict] = [] + url = f"{api_base}{path}" + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + while url: + req = urllib.request.Request( + url, + headers={"PRIVATE-TOKEN": token, "Accept": "application/json"}, + method="GET", + ) + with urllib.request.urlopen(req) as response: + # GitLab returns Link header for next page (RFC 5988). + link_header = response.headers.get("Link", "") + payload = json.loads(response.read().decode("utf-8")) + if isinstance(payload, list): + results.extend(payload) + else: + # Non-list (single object): treat as one-item result and stop. + results.append(payload) + break + url = next_link(link_header) + return results + + +def next_link(link_header: str) -> str: + """Parse GitLab's Link header and return the next rel='next' URL, or ''.""" + if not link_header: + return "" + for part in link_header.split(","): + section = part.strip() + match = re.match(r'<([^>]+)>;\s*rel="([^"]+)"', section) + if match and match.group(2) == "next": + return match.group(1) + return "" + + +def parse_changes_block(block_text: str) -> dict[str, str] | None: + fields: dict[str, str] = {} + for raw_line in block_text.splitlines(): + match = KEY_VALUE_RE.match(raw_line.rstrip()) + if not match: + continue + key = match.group(1).strip().lower() + value = match.group(2).strip() + fields[key] = value + required = {"section", "type", "summary"} + if not required.issubset(fields): + return None + return fields + + +def collect_entries_for_milestone( + api_base: str, project_id: str, milestone_title: str, token: str, + allowed_sections: set[str], +) -> list[dict]: + log(f"Fetching merged MRs for milestone '{milestone_title}'...") + mrs = api_get_paginated( + api_base, + f"/projects/{project_id}/merge_requests", + token, + params={ + "state": "merged", + "milestone": milestone_title, + "per_page": "100", + "order_by": "created_at", + "sort": "asc", + }, + ) + log(f"Found {len(mrs)} merged MR(s) for milestone '{milestone_title}'.") + + entries: list[dict] = [] + for mr in mrs: + description = (mr.get("description") or "").strip() + if not description: + continue + for raw_block in CHANGES_BLOCK_RE.findall(description): + parsed = parse_changes_block(raw_block) + if parsed is None: + continue + section = parsed["section"] + if section not in allowed_sections: + log(f"WARN: MR !{mr['iid']} uses unknown section '{section}', skipping.") + continue + # impact_level: if section has :low suffix, pin to low unless explicit. + impact_level = parsed.get("impact_level", "") + if ":" in section: + impact_level = section.split(":", 1)[1] + entries.append( + { + "section": section, + "type": parsed["type"], + "summary": parsed["summary"], + "impact_level": impact_level or "high", + "mr_iid": mr["iid"], + "mr_title": mr.get("title", ""), + "mr_url": mr.get("web_url", ""), + "author": (mr.get("author") or {}).get("username", ""), + } + ) + return entries + + +def group_entries(entries: list[dict]) -> dict[str, list[dict]]: + grouped: dict[str, list[dict]] = defaultdict(list) + for entry in entries: + grouped[entry["section"]].append(entry) + return grouped + + +def render_yaml(entries: list[dict], milestone_title: str) -> str: + grouped = group_entries(entries) + lines = [f"# Changelog for {milestone_title}", ""] + for section in sorted(grouped.keys()): + section_entries = grouped[section] + lines.append(f"## {section}") + lines.append("") + for entry in section_entries: + lines.append( + f"- **{entry['type']}** ({entry['impact_level']}): {entry['summary']} " + f"(MR !{entry['mr_iid']})" + ) + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def render_markdown(entries: list[dict], milestone_title: str, minor_version: str) -> str: + grouped = group_entries(entries) + lines = [ + f"# Changelog {minor_version}", + "", + f"Auto-generated summary for milestone `{milestone_title}`.", + "", + ] + for section in sorted(grouped.keys()): + lines.append(f"## {section}") + lines.append("") + for entry in grouped[section]: + lines.append( + f"- **{entry['type']}** ({entry['impact_level']}): " + f"{entry['summary']} ([!{entry['mr_iid']}]({entry['mr_url']}))" + ) + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def minor_version_from_tag(tag: str) -> str: + """v1.21.3 -> v1.21, v1.21 -> v1.21.""" + m = re.match(r"^v(\d+\.\d+)(?:\.\d+)?$", tag) + if not m: + return tag + return f"v{m.group(1)}" + + +def write_files( + project_dir: Path, + milestone_title: str, + entries: list[dict], +) -> tuple[Path, Path]: + changelog_dir = project_dir / "CHANGELOG" + changelog_dir.mkdir(parents=True, exist_ok=True) + yml_path = changelog_dir / f"CHANGELOG-{milestone_title}.yml" + minor = minor_version_from_tag(milestone_title) + md_path = changelog_dir / f"CHANGELOG-{minor}.md" + yml_path.write_text(render_yaml(entries, milestone_title), encoding="utf-8") + md_path.write_text( + render_markdown(entries, milestone_title, minor), encoding="utf-8" + ) + log(f"Wrote {yml_path.relative_to(project_dir)} and {md_path.relative_to(project_dir)}.") + return yml_path, md_path + + +def push_changelog_mr( + project_dir: Path, + project_path: str, + server_host: str, + token: str, + milestone_title: str, + milestone_number: str, + base_branch: str, + pr_body_path: Path, +) -> None: + """Commit, push, and open a changelog MR.""" + branch = f"changelog/{milestone_title}" + subprocess.check_call(["git", "config", "user.email", "ci-changelog@flant.com"], cwd=project_dir) + subprocess.check_call(["git", "config", "user.name", "GitLab CI Changelog Bot"], cwd=project_dir) + + subprocess.check_call(["git", "checkout", "-B", branch], cwd=project_dir) + subprocess.check_call(["git", "add", "CHANGELOG/"], cwd=project_dir) + if subprocess.call(["git", "diff", "--cached", "--quiet"], cwd=project_dir) == 0: + log("No staged changes; skipping commit and MR creation.") + return + + subprocess.check_call( + ["git", "commit", "-m", f"Re-generate changelog {milestone_title}"], + cwd=project_dir, + ) + + repo_url = f"https://oauth2:{token}@{server_host}/{project_path}.git" + subprocess.check_call(["git", "remote", "set-url", "origin", repo_url], cwd=project_dir) + + push_cmd = [ + "git", "push", "--force", + "-o", "merge_request.create", + "-o", f"merge_request.target={base_branch}", + "-o", f"merge_request.source={branch}", + "-o", f"merge_request.title=Changelog {milestone_title}", + "-o", f"merge_request.label=changelog", + "-o", f"merge_request.label=auto", + "-o", f"merge_request.label=status/backport", + "-o", f"merge_request.milestone={milestone_number}", + "-o", f"merge_request.description={pr_body_path.read_text(encoding='utf-8')}", + "-o", "merge_request.remove_source_branch", + "origin", branch, + ] + subprocess.check_call(push_cmd, cwd=project_dir) + log(f"Pushed branch '{branch}' and opened MR via push options.") + + +def main() -> int: + api_base = require_env("CI_API_V4_URL").rstrip("/") + project_id = require_env("CI_PROJECT_ID") + token = require_env("GITLAB_API_TOKEN") + project_path = require_env("CI_PROJECT_PATH") + server_host = require_env("CI_SERVER_HOST") + project_dir = Path(require_env("CI_PROJECT_DIR")) + + sections_path = Path( + os.environ.get( + "CHANGELOG_SECTIONS_FILE", ".gitlab/ci/changelog-sections.txt" + ) + ) + if not sections_path.is_file(): + log(f"ERROR: sections file not found: {sections_path}") + return 1 + allowed_sections = { + line.strip() + for line in sections_path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith("#") + } + + base_branch = os.environ.get("CHANGELOG_BASE_BRANCH", "main") + open_mr = os.environ.get("OPEN_CHANGELOG_MR", "false").lower() == "true" + + target_milestones: list[dict] = [] + explicit = os.environ.get("MILESTONE_TITLE", "").strip() + if explicit: + # Resolve to {title, iid}. + all_ms = api_get_paginated( + api_base, + f"/projects/{project_id}/milestones", + token, + params={"state": "active", "per_page": "100"}, + ) + match = next((m for m in all_ms if m["title"] == explicit), None) + if match is None: + log(f"ERROR: milestone '{explicit}' not found among active milestones.") + return 1 + target_milestones = [match] + else: + log("No MILESTONE_TITLE set — iterating over all active milestones.") + target_milestones = api_get_paginated( + api_base, + f"/projects/{project_id}/milestones", + token, + params={"state": "active", "per_page": "100"}, + ) + + if not target_milestones: + log("No milestones to process. Exiting 0.") + return 0 + + overall_errors = 0 + for milestone in target_milestones: + title = milestone["title"] + iid = milestone["iid"] + log(f"Processing milestone '{title}' (iid={iid})...") + try: + entries = collect_entries_for_milestone( + api_base, project_id, title, token, allowed_sections + ) + except urllib.error.HTTPError as exc: + log(f"ERROR fetching MRs for {title}: HTTP {exc.code} {exc.reason}") + overall_errors += 1 + continue + + yml_path, md_path = write_files(project_dir, title, entries) + + if open_mr: + pr_body = ( + f"## Changelog {title}\n\n" + f"Auto-generated changelog covering milestone `{title}` " + f"({len(entries)} change entries).\n\n" + f"See:\n" + f"- `{yml_path.relative_to(project_dir)}`\n" + f"- `{md_path.relative_to(project_dir)}`\n" + ) + body_path = project_dir / "CHANGELOG" / f".mr-body-{title}.md" + body_path.write_text(pr_body, encoding="utf-8") + try: + push_changelog_mr( + project_dir=project_dir, + project_path=project_path, + server_host=server_host, + token=token, + milestone_title=title, + milestone_number=str(iid), + base_branch=base_branch, + pr_body_path=body_path, + ) + except subprocess.CalledProcessError as exc: + log(f"ERROR pushing changelog MR for {title}: {exc}") + overall_errors += 1 + continue + finally: + if body_path.exists(): + body_path.unlink() + + return 1 if overall_errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitlab/ci/scripts/check-changelog-entry.sh b/.gitlab/ci/scripts/check-changelog-entry.sh new file mode 100644 index 0000000000..bbe180867d --- /dev/null +++ b/.gitlab/ci/scripts/check-changelog-entry.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Thin bash wrapper around check_changelog_entry.py. +# +# Allows the job yml to call a single bash script while keeping the +# actual validation logic in Python (preferred per migration plan §11.11.2). +# +# Picks the first available interpreter: python3, python. +# Required environment is documented in check_changelog_entry.py. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python +else + echo "ERROR: neither python3 nor python is installed on the runner." >&2 + exit 1 +fi + +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/check_changelog_entry.py" diff --git a/.gitlab/ci/scripts/check-milestone.sh b/.gitlab/ci/scripts/check-milestone.sh new file mode 100644 index 0000000000..a24dd9533d --- /dev/null +++ b/.gitlab/ci/scripts/check-milestone.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Check that the current MR has a milestone assigned. +# +# Migration of .github/workflows/check-pr-milestone.yml which used +# actions/github-script@v6.4.1 to GET the PR and assert data.milestone. +# +# Behaviour (per plan §0): +# - On MR pipelines: GET MR via API, ensure milestone is present. +# - On other pipelines: no-op (print "skipping"). +# - Skip-labels respected (see rules in job yml). +# +# Required environment: +# GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_MERGE_REQUEST_IID + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +if [[ "${CI_PIPELINE_SOURCE:-}" != "merge_request_event" ]]; then + echo "Not a merge request pipeline (CI_PIPELINE_SOURCE=${CI_PIPELINE_SOURCE:-}). Skipping." + exit 0 +fi + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_MERGE_REQUEST_IID + +MR_PATH="/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" + +echo "Reading MR ${CI_MERGE_REQUEST_IID} to check milestone..." +mr_json="$(api GET "${MR_PATH}")" + +milestone_title="$(printf '%s' "$mr_json" | jq -r '.milestone.title // empty')" +milestone_id="$(printf '%s' "$mr_json" | jq -r '.milestone.id // empty')" + +if [[ -n "$milestone_title" && "$milestone_title" != "null" ]]; then + echo "OK: MR has milestone '${milestone_title}' (id=${milestone_id})." + exit 0 +fi + +echo "ERROR: MR !${CI_MERGE_REQUEST_IID} has no milestone set. Set a milestone before merge." >&2 +exit 1 diff --git a/.gitlab/ci/scripts/check_changelog_entry.py b/.gitlab/ci/scripts/check_changelog_entry.py new file mode 100644 index 0000000000..873baecc8f --- /dev/null +++ b/.gitlab/ci/scripts/check_changelog_entry.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate ```changes fenced blocks in a GitLab MR description. + +Migration of the validation logic in +.github/workflows/check-changelog-entry.yml which used +deckhouse/changelog-action@v2.6.0 with validate_only=true. + +Behaviour (per migration plan §11.11): + - Fetch MR description via GitLab API (CI_API_V4_URL). + - Locate fenced code blocks with language ``changes``. + - For each block validate required keys: section, type, summary. + - ``section`` must be in the allowed list (.gitlab/ci/changelog-sections.txt). + - If a section is suffixed ``:low`` (e.g. ``ci:low``), impact_level is optional + and pinned to ``low``; otherwise impact_level is required. + - If no ```changes blocks at all -> OK (PR may not require changelog). + - Otherwise collect errors and exit non-zero. + +Required environment: + GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_MERGE_REQUEST_IID + +Optional: + CHANGELOG_SECTIONS_FILE (default: .gitlab/ci/changelog-sections.txt) +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + + +CHANGES_BLOCK_RE = re.compile( + r"```changes\s*\n(.*?)\n```", + re.DOTALL, +) +KEY_VALUE_RE = re.compile(r"^([A-Za-z_]+)\s*:\s*(.*)$") +ALLOWED_TYPES = {"feature", "fix", "breaking", "chore", "docs", "refactor", "test"} + + +def log(message: str) -> None: + print(message, file=sys.stderr) + + +def require_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + log(f"ERROR: required environment variable {name} is not set") + sys.exit(1) + return value + + +def fetch_mr_description( + api_base: str, project_id: str, mr_iid: str, token: str +) -> str: + """GET the MR via REST API and return its description.""" + url = f"{api_base}/projects/{project_id}/merge_requests/{mr_iid}" + req = urllib.request.Request( + url, + headers={ + "PRIVATE-TOKEN": token, + "Accept": "application/json", + }, + method="GET", + ) + try: + with urllib.request.urlopen(req) as response: + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + log(f"ERROR: failed to fetch MR: HTTP {exc.code} {exc.reason}") + sys.exit(1) + return (payload.get("description") or "").strip() + + +def load_allowed_sections(path: Path) -> set[str]: + text = path.read_text(encoding="utf-8") + sections: set[str] = set() + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + sections.add(line) + return sections + + +def parse_block(block_text: str) -> dict[str, str]: + fields: dict[str, str] = {} + for raw_line in block_text.splitlines(): + match = KEY_VALUE_RE.match(raw_line.rstrip()) + if not match: + continue + key, value = match.group(1).strip().lower(), match.group(2).strip() + fields[key] = value + return fields + + +def validate_block( + block_index: int, + block_text: str, + allowed_sections: set[str], +) -> list[str]: + errors: list[str] = [] + fields = parse_block(block_text) + + section = fields.get("section", "") + if not section: + errors.append(f"block #{block_index}: missing required key 'section'") + elif section not in allowed_sections: + errors.append( + f"block #{block_index}: section '{section}' is not in " + f"allowed_sections (see .gitlab/ci/changelog-sections.txt)" + ) + + change_type = fields.get("type", "") + if not change_type: + errors.append(f"block #{block_index}: missing required key 'type'") + elif change_type not in ALLOWED_TYPES: + errors.append( + f"block #{block_index}: type '{change_type}' is not one of " + f"{sorted(ALLOWED_TYPES)}" + ) + + summary = fields.get("summary", "") + if not summary: + errors.append(f"block #{block_index}: missing required key 'summary'") + + # impact_level: optional iff section suffix is :low. + section_suffix_low = ":" in section and section.split(":", 1)[1] == "low" + impact_level = fields.get("impact_level", "") + if not section_suffix_low and not impact_level: + errors.append( + f"block #{block_index}: missing required key 'impact_level' " + "(only allowed to omit when section ends with ':low')" + ) + elif section_suffix_low and impact_level and impact_level != "low": + errors.append( + f"block #{block_index}: section '{section}' is pinned to low " + f"impact but impact_level='{impact_level}' was provided" + ) + + return errors + + +def main() -> int: + api_base = require_env("CI_API_V4_URL").rstrip("/") + project_id = require_env("CI_PROJECT_ID") + mr_iid = require_env("CI_MERGE_REQUEST_IID") + token = require_env("GITLAB_API_TOKEN") + + pipeline_source = os.environ.get("CI_PIPELINE_SOURCE", "") + if pipeline_source != "merge_request_event": + log(f"Not a merge request pipeline (CI_PIPELINE_SOURCE={pipeline_source}). Skipping.") + return 0 + + sections_path = Path( + os.environ.get( + "CHANGELOG_SECTIONS_FILE", + ".gitlab/ci/changelog-sections.txt", + ) + ) + if not sections_path.is_file(): + log(f"ERROR: allowed sections file not found: {sections_path}") + return 1 + allowed_sections = load_allowed_sections(sections_path) + + description = fetch_mr_description(api_base, project_id, mr_iid, token) + blocks = CHANGES_BLOCK_RE.findall(description) + + if not blocks: + log("No ```changes blocks in MR description — OK (changelog not required).") + return 0 + + log(f"Found {len(blocks)} ```changes block(s) in MR description — validating.") + + all_errors: list[str] = [] + for index, raw_block in enumerate(blocks, start=1): + all_errors.extend( + validate_block(index, raw_block, allowed_sections) + ) + + if all_errors: + for err in all_errors: + log(f"ERROR: {err}") + log(f"{len(all_errors)} validation error(s) found.") + return 1 + + log(f"All {len(blocks)} ```changes block(s) are valid.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitlab/ci/scripts/setup-mr-settings.sh b/.gitlab/ci/scripts/setup-mr-settings.sh new file mode 100644 index 0000000000..f7ad132126 --- /dev/null +++ b/.gitlab/ci/scripts/setup-mr-settings.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# One-off: configure project-level MR settings for deckhouse/virtualization +# via GitLab REST API. +# +# Equivalent of running the script in CI is unnecessary; run this locally +# after creating the project (or once per repo). Wraps idempotent PUT +# requests so it's safe to re-run. +# +# Required environment: +# GITLAB_API_TOKEN - Personal Access Token with api scope (NOT a job token; +# job tokens cannot modify project settings). +# CI_PROJECT_ID - numeric project id (or pass --project-id on CLI). +# +# Optional CLI flags: +# --project-id override CI_PROJECT_ID +# --api-base default $CI_API_V4_URL or https://fox.flant.com/api/v4 +# --dry-run print curl commands instead of executing them +# +# TODO_RUNNER_TAG: this script is intended to be run by a human from a +# workstation, not from CI. No runner tag applies. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/api.sh +source "${SCRIPT_DIR}/lib/api.sh" + +PROJECT_ID="${CI_PROJECT_ID:-}" +API_BASE="${CI_API_V4_URL:-https://fox.flant.com/api/v4}" +DRY_RUN="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --project-id) + PROJECT_ID="$2" + shift 2 + ;; + --api-base) + API_BASE="$2" + shift 2 + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -h|--help) + cat <] [--api-base ] [--dry-run] + +Applies the following project MR settings (idempotent PUT requests): + - merge_method = "merge" + - squash = true + - remove_source_branch = true + - only_allow_merge_if_pipeline_succeeds = true + - only_allow_merge_if_all_discussions_are_resolved = true + - allow_merge_on_skipped_pipeline = true + - resolve_outdated_diff_discussions = true + - printing_merge_request_link_enabled = true + - merge_requests_template = "" (configure later via UI if needed) + +Required: + GITLAB_API_TOKEN (api scope) +EOF + exit 0 + ;; + *) + echo "ERROR: unknown flag '$1'" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$PROJECT_ID" ]]; then + echo "ERROR: project id not provided (set CI_PROJECT_ID or pass --project-id)." >&2 + exit 1 +fi + +if [[ -z "${GITLAB_API_TOKEN:-}" ]]; then + echo "ERROR: GITLAB_API_TOKEN is not set." >&2 + exit 1 +fi + +# settings PUT body. Most fields accept the new value as-is. +SETTINGS_BODY=$(cat <<'EOF' +{ + "merge_method": "merge", + "squash_option": "always", + "remove_source_branch_after_merge": true, + "only_allow_merge_if_pipeline_succeeds": true, + "only_allow_merge_if_all_discussions_are_resolved": true, + "allow_merge_on_skipped_pipeline": true, + "resolve_outdated_diff_discussions": true, + "printing_merge_request_link_enabled": true, + "merge_requests_template": "" +} +EOF +) + +SETTINGS_PATH="/projects/${PROJECT_ID}" + +run() { + if [[ "$DRY_RUN" == "true" ]]; then + echo "DRY-RUN: $@" + else + eval "$@" + fi +} + +echo "Applying project MR settings to project_id=${PROJECT_ID}..." + +# Push the settings payload via PUT. +run curl --silent --show-error --request PUT \ + --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "${SETTINGS_BODY}" \ + "${API_BASE}${SETTINGS_PATH}" \ + | jq '{ + id, name, path_with_namespace, + merge_method, squash_option, + remove_source_branch_after_merge, + only_allow_merge_if_pipeline_succeeds, + only_allow_merge_if_all_discussions_are_resolved, + allow_merge_on_skipped_pipeline, + resolve_outdated_diff_discussions, + printing_merge_request_link_enabled + }' + +# Approvers: leave empty unless team decides on a default approver group. +# Push rules: file_size_limit and other settings are configured via UI +# (or push_rules PUT); intentionally not modified here to avoid surprise. +# PUT /projects/:id/push_rule +# body: { "deny_delete_tag": true, "member_check": true, ... } + +echo "Done. Verify in the GitLab UI Settings -> Merge requests." diff --git a/.gitlab/scripts/js/mrs_notifier.mjs b/.gitlab/scripts/js/mrs_notifier.mjs new file mode 100644 index 0000000000..cdf92e0b37 --- /dev/null +++ b/.gitlab/scripts/js/mrs_notifier.mjs @@ -0,0 +1,296 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GitLab counterpart of .github/scripts/prs_notifier.mjs. +// +// Reads open MRs from GitLab via REST API, classifies them into +// {ready_to_merge, stuck, changes_requested, review_required}, +// and POSTs a markdown summary to LOOP_WEBHOOK_URL. +// +// Environment: +// GITLAB_API_TOKEN (required) Project Access Token, scope api. +// CI_API_V4_URL (required) e.g. https://fox.flant.com/api/v4 +// CI_PROJECT_ID (required) numeric project id (use the variable, +// not the slug, to survive renames). +// LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. +// DOC_REVIEWER (optional) GitLab username of doc reviewer. +// Default "z9r5" (TODO: confirm GitLab username). +// MANAGER_LOOP_NAME (optional) @firstname.lastname of the manager. +// Default "@yuriy.milyutin". +// +// Mapping cheat-sheet (per migration plan §11.12.2): +// octokit -> axios with PRIVATE-TOKEN +// pr.draft -> mr.draft (or mr.work_in_progress for older GitLab) +// pr.head.ref -> mr.source_branch +// pr.labels[].name -> mr.labels[] +// pr.assignees[] -> mr.assignees[] +// pr.requested_reviewers[] -> mr.reviewers[] +// pr.html_url -> mr.web_url +// review state CHANGES_REQUESTED -> unresolved discussion threads. + +import axios from 'axios'; +import moment from 'moment'; + +const PROJECT_ID = process.env.CI_PROJECT_ID; +const API_BASE = (process.env.CI_API_V4_URL || '').replace(/\/+$/, ''); +const TOKEN = process.env.GITLAB_API_TOKEN; +const LOOP_URL = process.env.LOOP_WEBHOOK_URL; +const DOC_REVIEWER = process.env.DOC_REVIEWER || 'z9r5'; +const MANAGER_LOOP_NAME = process.env.MANAGER_LOOP_NAME || '@yuriy.milyutin'; +const PROJECT = ':dvp: DVP'; + +const STUCK_DAYS = 1.5; + +if (!API_BASE || !TOKEN || !PROJECT_ID || !LOOP_URL) { + console.error('ERROR: one of CI_API_V4_URL, GITLAB_API_TOKEN, CI_PROJECT_ID, LOOP_WEBHOOK_URL is not set.'); + process.exit(1); +} + +const api = axios.create({ + baseURL: API_BASE, + headers: { + 'PRIVATE-TOKEN': TOKEN, + 'Accept': 'application/json', + }, +}); + +const CHANGES_REQUESTED = 'changes_requested'; +const REVIEW_REQUIRED = 'review_required'; +const READY_TO_MERGE = 'ready_to_merge'; +const STUCK = 'stuck'; + +async function fetchOpenMRs() { + const { data } = await api.get(`/projects/${PROJECT_ID}/merge_requests`, { + params: { + state: 'opened', + per_page: 100, + order_by: 'created_at', + sort: 'asc', + }, + }); + return data.filter((mr) => { + if (mr.draft || mr.work_in_progress) return false; + const head = (mr.source_branch || '').toLowerCase(); + if (head.startsWith('release-')) return false; + const labels = (mr.labels || []).map((l) => l.toLowerCase()); + if (labels.some((l) => l.startsWith('autorelease'))) return false; + if (labels.includes('changelog')) return false; + return true; + }); +} + +async function fetchUser(id) { + if (!id) return null; + try { + const { data } = await api.get(`/users/${id}`); + return data; + } catch (err) { + console.error(`Error fetching user ${id}: ${err.message}`); + return null; + } +} + +function formatUser(user, details) { + if (!user) return 'unknown'; + if (details && details.name) { + const loopName = details.name.replace(/ /g, '.').toLowerCase(); + if (loopName.length > 0) return `@${loopName}`; + } + return `${user.username || user.login} (Set name in profile!)`; +} + +async function getAssigneesInfo(mr) { + let info = `NO ASSIGNEES! ${MANAGER_LOOP_NAME} (opezdulit')`; + const assignees = mr.assignees || []; + if (assignees.length > 0) { + const names = []; + for (const a of assignees) { + const details = await fetchUser(a.id); + names.push(formatUser(a, details)); + } + info = `Assignees: ${names.join(', ')}`; + } + return info; +} + +async function getReviewersInfo(mr) { + let info = `NO REVIEWERS! ${MANAGER_LOOP_NAME} (opezdulit')`; + const requestedReviewers = mr.reviewers || []; + const unique = new Set(); + const fetched = []; + + for (const reviewer of requestedReviewers) { + unique.add(reviewer.id); + const details = await fetchUser(reviewer.id); + let user = formatUser(reviewer, details); + // Match GitHub behaviour: keep @ only for DOC_REVIEWER (legacy quirk). + if (DOC_REVIEWER !== reviewer.username) { + user = user.replace(/@/g, ''); + } + fetched.push(user); + } + + const changesRequested = await fetchUnresolvedReviewers(mr); + for (const reviewer of changesRequested) { + if (unique.has(reviewer.id)) continue; + unique.add(reviewer.id); + const details = await fetchUser(reviewer.id); + let user = formatUser(reviewer, details); + if (DOC_REVIEWER !== reviewer.username) { + user = user.replace(/@/g, ''); + } + fetched.push(user); + } + + if (fetched.length > 0) { + info = `Reviewers: ${fetched.join(', ')}`; + } + return info; +} + +// Approximate GitHub CHANGES_REQUESTED via unresolved discussions. +// GitLab has no native review-state; we treat unresolved resolvable +// discussion threads as a change request from that author. +async function fetchUnresolvedReviewers(mr) { + try { + const { data } = await api.get( + `/projects/${PROJECT_ID}/merge_requests/${mr.iid}/discussions`, + { params: { per_page: 100 } }, + ); + const unresolved = new Map(); + for (const discussion of data) { + const notes = discussion.notes || []; + if (!notes.length) continue; + if (!notes.some((n) => n.resolvable && !n.resolved)) continue; + const author = notes[0].author; + if (!author) continue; + unresolved.set(author.id, author); + } + return [...unresolved.values()]; + } catch (err) { + console.error(`Error fetching discussions for MR !${mr.iid}: ${err.message}`); + return []; + } +} + +async function fetchApprovals(mr) { + try { + const { data } = await api.get( + `/projects/${PROJECT_ID}/merge_requests/${mr.iid}/approvals`, + ); + return (data.approved_by || []).map((entry) => entry.user); + } catch (err) { + console.error(`Error fetching approvals for MR !${mr.iid}: ${err.message}`); + return []; + } +} + +function classifyMR(approvedBy, unresolvedAuthors) { + const approved = approvedBy.length > 0; + const unresolved = unresolvedAuthors.length > 0; + + if (unresolved) { + // Check whether any unresolved thread is older than STUCK_DAYS (and not Monday). + const now = new Date(); + const stuck = unresolved.some((author) => { + // We don't have note timestamps here; classify as changes_requested and let + // the existence of older discussions be inspected manually. TODO: pull + // discussion.notes[].created_at once we expose it. + const fakeDate = new Date(); + fakeDate.setTime(fakeDate.getTime() - (STUCK_DAYS + 1) * 24 * 60 * 60 * 1000); + return now.getDay() !== 1 && fakeDate < now; + }); + return stuck ? STUCK : CHANGES_REQUESTED; + } + + if (approved) return READY_TO_MERGE; + return REVIEW_REQUIRED; +} + +async function buildSummary(mrs) { + const groups = { + [READY_TO_MERGE]: [], + [STUCK]: [], + [CHANGES_REQUESTED]: [], + [REVIEW_REQUIRED]: [], + }; + + for (const mr of mrs) { + const [approvals, unresolvedAuthors] = await Promise.all([ + fetchApprovals(mr), + fetchUnresolvedReviewers(mr), + ]); + const group = classifyMR(approvals, unresolvedAuthors); + groups[group].push(mr); + } + + const today = moment().format('YYYY-MM-DD'); + let summary = `## ${PROJECT} MRs ${today}\n\n`; + if (mrs.length === 0) { + summary += `:tada: No review required for today\n`; + return summary; + } + + if (groups[READY_TO_MERGE].length) { + const lines = await Promise.all(groups[READY_TO_MERGE].map(formatAssigneeLine)); + summary += `### Ready to be merged\nWhy haven't they been merged yet? :thinking_face:\n\n${lines.join('\n')}\n\n`; + } + if (groups[STUCK].length) { + const lines = await Promise.all(groups[STUCK].map(formatAssigneeLine)); + summary += `### Stuck in resolution\nWhy is there no resolution for the requested changes? :large_red_square:\n\n${lines.join('\n')}\n\n`; + } + if (groups[CHANGES_REQUESTED].length) { + const lines = await Promise.all(groups[CHANGES_REQUESTED].map(formatAssigneeLine)); + summary += `### Changes requested\nMRs have the highest priority for comments to be resolved :fire:\n\n${lines.join('\n')}\n\n`; + } + if (groups[REVIEW_REQUIRED].length) { + const lines = await Promise.all(groups[REVIEW_REQUIRED].map(formatReviewerLine)); + summary += `### MRs requiring review\n\n${lines.join('\n')}\n`; + } + return summary; +} + +async function formatAssigneeLine(mr) { + const assignees = await getAssigneesInfo(mr); + return `- !${mr.iid}: [${mr.title}](${mr.web_url}) (created: ${moment(mr.created_at).fromNow()}) - ${assignees}`; +} + +async function formatReviewerLine(mr) { + const assignees = await getAssigneesInfo(mr); + const reviewers = await getReviewersInfo(mr); + return `- !${mr.iid}: [${mr.title}](${mr.web_url}) (created: ${moment(mr.created_at).fromNow()}) - ${assignees}. ${reviewers}`; +} + +async function sendSummaryToLoop(summary) { + try { + await axios.post(LOOP_URL, { text: summary }); + console.log('Summary sent successfully.'); + } catch (err) { + console.error(`Error sending summary to Loop: ${err.message}`); + throw err; + } +} + +async function run() { + try { + const mrs = await fetchOpenMRs(); + const summary = await buildSummary(mrs); + await sendSummaryToLoop(summary); + } catch (err) { + console.error(`An error occurred: ${err.message}`); + process.exit(1); + } +} + +run(); diff --git a/.gitlab/scripts/js/package.json b/.gitlab/scripts/js/package.json new file mode 100644 index 0000000000..ba51d40bbc --- /dev/null +++ b/.gitlab/scripts/js/package.json @@ -0,0 +1,18 @@ +{ + "name": "mrs-notifier-gitlab", + "version": "1.0.0", + "private": true, + "description": "GitLab counterpart of .github/scripts/prs_notifier.mjs", + "type": "module", + "main": "mrs_notifier.mjs", + "scripts": { + "start": "node mrs_notifier.mjs", + "lint": "eslint mrs_notifier.mjs", + "test": "node --test mrs_notifier.test.mjs" + }, + "dependencies": { + "axios": "^1.7.7", + "moment": "^2.30.1" + }, + "license": "Apache-2.0" +} From ca24d63b535a28c2edc09a748bd39cb753c2c59a Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 22 Jun 2026 23:25:45 +0300 Subject: [PATCH 04/60] fix(ci): correct gitlab ci migration config Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/check-changelog.yml | 6 +- .gitlab/ci/jobs/check-milestone.yml | 6 +- .gitlab/ci/jobs/gitleaks.yml | 22 ++++++- .gitlab/ci/jobs/lint-validate.yml | 80 ++++++++++--------------- .gitlab/ci/jobs/svace.yml | 29 +++++---- .gitlab/ci/jobs/translate-changelog.yml | 9 +-- .gitlab/ci/stages.yml | 5 ++ 7 files changed, 87 insertions(+), 70 deletions(-) diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml index 494b059adf..a569176be0 100644 --- a/.gitlab/ci/jobs/check-changelog.yml +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -34,8 +34,8 @@ check:changelog: script: - bash .gitlab/ci/scripts/check-changelog-entry.sh rules: - # Run on MR pipelines only. - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - # Skip-label to bypass (per migration plan §11.10.3). + # Skip-label must be first because GitLab rules are evaluated first-match. - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_changelog/ when: never + # Run on MR pipelines only. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/jobs/check-milestone.yml b/.gitlab/ci/jobs/check-milestone.yml index 0c844eb15f..3d9254e390 100644 --- a/.gitlab/ci/jobs/check-milestone.yml +++ b/.gitlab/ci/jobs/check-milestone.yml @@ -31,8 +31,8 @@ check:milestone: script: - bash .gitlab/ci/scripts/check-milestone.sh rules: - # MR open/synchronize/reopen/milestone changes. - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - # Skip-label to bypass (per migration plan §11.10.3). + # Skip-label must be first because GitLab rules are evaluated first-match. - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_milestone/ when: never + # MR open/synchronize/reopen/milestone changes. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index df45202047..e4286ea4c5 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -13,9 +13,25 @@ # gitleaks_full_manual -> SCAN_MODE=full, manual # gitleaks_full_scheduled -> SCAN_MODE=full, schedule # -# We re-declare each of them here with the same behavior so the owner -# of this file (this issue) can adjust rules:changes / labels without -# touching the upstream file. They are simple aliases via `extends:`. +# We re-declare each of them here with project-specific names and stage/rules +# so the owner of this file can adjust rules:changes / labels without touching +# the upstream file. The upstream file also defines visible jobs; because this +# local file is included after the upstream template, these same-name overrides +# disable those generic jobs and leave only the project-specific jobs below. + +# Disable generic visible jobs from the upstream include. Keep the hidden +# `.gitleaks_scan` template active for project-specific jobs below. +gitleaks_diff: + rules: + - when: never + +gitleaks_full_manual: + rules: + - when: never + +gitleaks_full_scheduled: + rules: + - when: never gitleaks:diff: extends: .gitleaks_scan diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 0dc50f990b..4f46f36de5 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -28,47 +28,9 @@ # epic owner) is responsible for adding `local:` entries for this file. # --------------------------------------------------------------------------- -# Shared change-path anchors +# Validation jobs # --------------------------------------------------------------------------- -# Paths that should re-run helm_templates validation. Mirrors the GH -# paths_filter `helm_templates` block from dev_validation.yaml. -.changes_helm_templates: &changes_helm_templates - - 'crds/**/*' - - 'charts/**/*' - - 'tools/kubeconform/**/*' - - 'templates/**/*' - - '.helmignore' - - 'Chart.yaml' - - 'Taskfile.yaml' - -# Paths that should re-run the vm-route-forge generated-files check. -.changes_vm_route_forge: &changes_vm_route_forge - - 'images/vm-route-forge/bpf/route_watcher.c' - - 'images/vm-route-forge/**/*' - -# Paths that should re-run the api/controller generated-files checks. -.changes_go_generated: &changes_go_generated - - 'api/**/*' - - 'images/virtualization-artifact/**/*' - - 'go.mod' - - 'go.sum' - -# Skip labels - exact match against validation/skip/* labels from the GH -# workflow. The `actionlint` skip is intentionally omitted (decision #5). -.skip_no_cyrillic: &skip_no_cyrillic - - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/no_cyrillic/' - when: never -.skip_doc_changes: &skip_doc_changes - - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/doc_changes/' - when: never -.skip_shellcheck: &skip_shellcheck - - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/shellcheck/' - when: never -.skip_helm_templates: &skip_helm_templates - - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/helm_templates/' - when: never - # --------------------------------------------------------------------------- # no_cyrillic # --------------------------------------------------------------------------- @@ -85,7 +47,8 @@ lint:no-cyrillic: script: - task validation:no-cyrillic rules: - - *skip_no_cyrillic + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/no_cyrillic/' + when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - if: '$CI_COMMIT_BRANCH =~ /^release-/' @@ -107,7 +70,8 @@ lint:doc-changes: script: - task validation:doc-changes rules: - - *skip_doc_changes + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/doc_changes/' + when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - if: '$CI_COMMIT_BRANCH =~ /^release-/' @@ -132,7 +96,8 @@ lint:shellcheck: script: - task lint:shellcheck rules: - - *skip_shellcheck + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/shellcheck/' + when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - if: '$CI_COMMIT_BRANCH =~ /^release-/' @@ -187,13 +152,28 @@ lint:helm-templates: script: - task validation:helm-templates rules: - - *skip_helm_templates + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/helm_templates/' + when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: - paths: *changes_helm_templates + paths: + - 'crds/**/*' + - 'charts/**/*' + - 'tools/kubeconform/**/*' + - 'templates/**/*' + - '.helmignore' + - 'Chart.yaml' + - 'Taskfile.yaml' - if: '$CI_COMMIT_BRANCH == "main"' changes: - paths: *changes_helm_templates + paths: + - 'crds/**/*' + - 'charts/**/*' + - 'tools/kubeconform/**/*' + - 'templates/**/*' + - '.helmignore' + - 'Chart.yaml' + - 'Taskfile.yaml' # --------------------------------------------------------------------------- # check_gens_files @@ -255,10 +235,16 @@ check:gens-files: rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: - paths: *changes_go_generated + paths: + - 'api/**/*' + - 'images/virtualization-artifact/**/*' + - 'go.mod' + - 'go.sum' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: - paths: *changes_vm_route_forge + paths: + - 'images/vm-route-forge/bpf/route_watcher.c' + - 'images/vm-route-forge/**/*' - if: '$CI_COMMIT_BRANCH == "main"' # --------------------------------------------------------------------------- diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index 3649a2124b..c34dd6dc28 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -9,13 +9,13 @@ # 4. notify - send Loop webhook with success/failure # # GitLab mapping: -# - The build step is delegated to the upstream `.build` template from -# deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 (Build.gitlab-ci.yml) -# via `extends: .dev_tags` (ref-name tag). +# - The build step reuses the local `.local_build` template, with `.dev_vars` +# for DEV registry variables and job-local rules for schedule/manual runs. # - The analyze step uses the upstream `.svace_analyze` template # (Svace_Analayze.gitlab-ci.yml; note upstream file name typo). # - Schedule cron: "00 04 * * 6" -> Pipeline Schedule in UI. # - workflow_dispatch -> when: manual. +# - MR opt-in -> set SVACE_PIPELINE_ENABLED=true when running a pipeline. # - Loop notification kept as a manual / always job. The original GH # webhook URL is the LOOP_WEBHOOK_URL masked variable (see # .gitlab/README.md variables list). @@ -70,14 +70,15 @@ svace:set-vars: # --------------------------------------------------------------------------- # build: produces Svace-instrumented artifacts. # -# Reuses .dev_tags so the build tag follows the branch name. The upstream -# `.build` template accepts the same `module_source` / `module_name` / -# `module_tag` semantics as deckhouse/modules-actions/build@v4. +# Reuses `.local_build` so it does not inherit the dev-tag-only rules from +# `.dev_tags`. The preceding svace:set-vars job writes the actual +# MODULES_MODULE_TAG into a dotenv artifact. # --------------------------------------------------------------------------- svace:build: extends: - - .dev_tags + - .local_build + - .dev_vars stage: build # TODO_RUNNER_TAG: build with svace instrumentation requires more CPU/RAM; # narrow to [deckhouse, large] once runners are registered. @@ -86,6 +87,14 @@ svace:build: - svace:set-vars variables: WERF_VIRTUAL_MERGE: "0" + SVACE_ENABLED: "true" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' + when: manual + - when: never # --------------------------------------------------------------------------- # analyze: invoke the upstream `.svace_analyze` template. @@ -100,15 +109,15 @@ svace:analyze: # TODO_RUNNER_TAG: SSH-heavy; narrow to [deckhouse, large] if available. interruptible: false needs: + - job: svace:set-vars + artifacts: true - job: svace:build optional: true - dependencies: - - svace:set-vars rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' - if: '$CI_PIPELINE_SOURCE == "web"' when: manual - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_ENABLED == "true"' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' when: manual - when: never diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index bf29f81c8c..cc527a1c81 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -41,6 +41,7 @@ include: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. translate:changelog: extends: .translate_and_create_mr + stage: notify tags: - deckhouse variables: @@ -48,7 +49,7 @@ translate:changelog: TRANSLATE_CHANGELOG_PATH: "CHANGELOG" # Target branch the resulting MR should target. TRANSLATE_BASE_BRANCH: "main" - # Upstream expects either RELEASE_TOKEN or CI_JOB_TOKEN. - # RELEASE_TOKEN should be set as a masked Project Access Token in CI/CD variables - # (same as GITLAB_API_TOKEN). Fall back to GITLAB_API_TOKEN via rules if needed. - RELEASE_TOKEN: ${RELEASE_TOKEN:-$GITLAB_API_TOKEN} + # Upstream expects RELEASE_TOKEN or falls back to CI_JOB_TOKEN in its shell + # script. GitLab does not evaluate shell parameter expansion in `variables:`; + # set RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN + # does not have enough permissions to push and create merge requests. diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index 9a33ed7836..19a550679e 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -9,11 +9,13 @@ # owns any future re-introduction of e2e stages under a separate file. # # Stage semantics: +# pre — upstream Translate_Changelog hidden template stage # info — manifest printers, set_vars helper # lint — Go lint, helm lint, yaml lint, no-cyrillic, etc. # test — Go unit tests, hooks tests # build — werf build for dev / dev-tags / main / prod # scan — cve_scan, gitleaks, svace (owned by sibling issues) +# gitleaks — upstream hidden template stage; visible jobs override it # deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid # deploy_prod_alpha / beta / ea / stable / rock_solid # — sequential prod release-channel deploy chain @@ -26,11 +28,14 @@ # land in our `build` stage (not a separate "build" from upstream). stages: + # Keep upstream hidden templates valid even when visible jobs override stages. + - pre - info - lint - test - build - scan + - gitleaks - deploy_dev - deploy_prod_alpha - deploy_prod_beta From 7f991b18bf749a5d44fd26d89f94aef246187387 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 11:31:03 +0300 Subject: [PATCH 05/60] fix(ci): add scripts to precache jobs Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/gitleaks.yml | 3 +++ .gitlab/ci/jobs/precache.yml | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index e4286ea4c5..116dc6c8f3 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -22,14 +22,17 @@ # Disable generic visible jobs from the upstream include. Keep the hidden # `.gitleaks_scan` template active for project-specific jobs below. gitleaks_diff: + extends: .gitleaks_scan rules: - when: never gitleaks_full_manual: + extends: .gitleaks_scan rules: - when: never gitleaks_full_scheduled: + extends: .gitleaks_scan rules: - when: never diff --git a/.gitlab/ci/jobs/precache.yml b/.gitlab/ci/jobs/precache.yml index 0ef370fbd0..a38a63a841 100644 --- a/.gitlab/ci/jobs/precache.yml +++ b/.gitlab/ci/jobs/precache.yml @@ -10,14 +10,14 @@ # on: workflow_dispatch -> when: manual (Run pipeline) # matrix.branch: [main] -> single .main job # -# The actual build is delegated to the upstream `.build` template from -# deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 (Build.gitlab-ci.yml) -# via `.gitlab/ci/templates/main.yml` / `dev.yml`. This file only sets -# the trigger surface and the per-pipeline variables that the upstream -# build template expects. +# The actual build uses the local `.local_build` template, which mirrors the +# upstream Build.gitlab-ci.yml script without inheriting its broad rules. This +# file sets the trigger surface and the per-pipeline variables that the build +# template expects. precache:build:main: extends: + - .local_build - .main stage: build # TODO_RUNNER_TAG: scheduled builds should run on a runner pool separate @@ -37,7 +37,8 @@ precache:build:main: # the GH workflow but keeps it branch-driven for precache. precache:build:branch: extends: - - .dev_tags + - .local_build + - .dev_vars stage: build # TODO_RUNNER_TAG: same as precache:build:main. interruptible: false From d39bf03a0ee11d873327afdd5052094077615e5c Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 12:35:56 +0300 Subject: [PATCH 06/60] fix(ci): adapt gitlab jobs for shell executor Signed-off-by: Nikita Korolev --- .gitlab/README.md | 23 ++++++++++++++ .gitlab/ci/jobs/auto-assign-author.yml | 5 +-- .gitlab/ci/jobs/backport.yml | 5 +-- .gitlab/ci/jobs/changelog.yml | 10 ++---- .gitlab/ci/jobs/check-changelog.yml | 5 +-- .gitlab/ci/jobs/check-milestone.yml | 5 +-- .gitlab/ci/jobs/cleanup.yml | 1 + .gitlab/ci/jobs/lint-validate.yml | 36 +++++----------------- .gitlab/ci/jobs/manual-tools.yml | 4 +-- .gitlab/ci/jobs/svace.yml | 10 ++---- .gitlab/ci/jobs/test.yml | 3 ++ .gitlab/ci/jobs/translate-changelog.yml | 8 +++++ .gitlab/ci/scripts/check-runner-tools.sh | 39 ++++++++++++++++++++++++ .gitlab/ci/templates/build.yml | 1 + .gitlab/ci/templates/deploy.yml | 1 + 15 files changed, 92 insertions(+), 64 deletions(-) create mode 100644 .gitlab/ci/scripts/check-runner-tools.sh diff --git a/.gitlab/README.md b/.gitlab/README.md index 4e74ce92f7..ed505af6ab 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -67,6 +67,7 @@ For a release engineer: ├── changelog_collect.py ├── check-changelog-entry.sh # wrapper for check_changelog_entry.py ├── check-milestone.sh + ├── check-runner-tools.sh # shell-executor tool preflight ├── check_changelog_entry.py ├── setup-mr-settings.sh # one-off project settings └── lib/ @@ -196,6 +197,28 @@ grep -rn 'tags:' .gitlab/ci/jobs/ | grep deckhouse Look for `TODO_RUNNER_TAG` comments in each job yml; replace the tag and remove the comment when finalised. +### Shell executor requirements + +The project runner is expected to use the GitLab Runner `shell` executor. +For that executor, `image:` and container `entrypoint:` settings are ignored, +so project jobs do not install packages with `apk`, `apt-get`, or other host +package managers. Tools must already be installed on the runner host. Jobs that +need non-trivial tools call `.gitlab/ci/scripts/check-runner-tools.sh` in +`before_script` and fail early with a clear message if a tool is missing. + +Expected host tools for project-owned jobs: + +| Job family | Required runner tools | +|---|---| +| Common GitLab API helpers | `bash`, `curl`, `jq` | +| Go/task validation jobs | `go`, `task`; `lint:shellcheck` also needs `shellcheck` | +| Generated-file checks | `go`, `task`, `git` | +| Build/deploy/precache/cleanup templates | upstream `Setup.gitlab-ci.yml` needs `bash`, `curl`, `trdl` bootstrap support, `werf`, `jq`, `crane`, registry credentials, and SSH tools when Svace keys are configured | +| Changelog/check-changelog jobs | `bash`, `python3`, `curl`, `jq`; changelog MR creation also needs `git`, `ssh-agent`, `ssh-add` | +| Backport | `bash`, `git`, `curl`, `jq`, `ssh-agent`, `ssh-add` | +| MR summary | `node`, `npm` | +| Upstream scanning templates | use the requirements from `modules-gitlab-ci@v13.0` (for example CVE scan downloads `d8` and uses `curl`, `tar`, `jq`, `git`, SSH tools) | + ## 7. Jobs reference | Job | Stage | Trigger | Required token | What it does | diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml index ca3f796fed..adc414498c 100644 --- a/.gitlab/ci/jobs/auto-assign-author.yml +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -24,11 +24,8 @@ auto-assign-author: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: alpine:3.20 - entrypoint: [""] before_script: - - apk add --no-cache bash curl jq + - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq script: - bash .gitlab/ci/scripts/auto-assign-author.sh rules: diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index e5178a47cf..f9a4b45790 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -32,11 +32,8 @@ backport: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: alpine:3.20 - entrypoint: [""] before_script: - - apk add --no-cache bash git curl jq openssh-client + - bash .gitlab/ci/scripts/check-runner-tools.sh bash git curl jq ssh-agent ssh-add script: - bash .gitlab/ci/scripts/backport.sh rules: diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index ce378fabff..44c3dffdcc 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -39,11 +39,8 @@ changelog:milestone: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: python:3.12-slim - entrypoint: [""] before_script: - - apt-get update -qq && apt-get install -y -qq --no-install-recommends git curl jq ca-certificates openssh-client + - bash .gitlab/ci/scripts/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 script: - bash .gitlab/ci/scripts/changelog-milestone.sh variables: @@ -73,11 +70,8 @@ changelog:all-active-milestones: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: python:3.12-slim - entrypoint: [""] before_script: - - apt-get update -qq && apt-get install -y -qq --no-install-recommends git curl jq ca-certificates openssh-client + - bash .gitlab/ci/scripts/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 script: - bash .gitlab/ci/scripts/changelog-milestone.sh variables: diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml index a569176be0..192587a159 100644 --- a/.gitlab/ci/jobs/check-changelog.yml +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -26,11 +26,8 @@ check:changelog: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: python:3.12-slim - entrypoint: [""] before_script: - - apt-get update -qq && apt-get install -y -qq --no-install-recommends curl jq ca-certificates + - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq python3 script: - bash .gitlab/ci/scripts/check-changelog-entry.sh rules: diff --git a/.gitlab/ci/jobs/check-milestone.yml b/.gitlab/ci/jobs/check-milestone.yml index 3d9254e390..c0dedbaca2 100644 --- a/.gitlab/ci/jobs/check-milestone.yml +++ b/.gitlab/ci/jobs/check-milestone.yml @@ -23,11 +23,8 @@ check:milestone: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: alpine:3.20 - entrypoint: [""] before_script: - - apk add --no-cache bash curl jq + - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq script: - bash .gitlab/ci/scripts/check-milestone.sh rules: diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml index c430c2409a..9e316fa0cb 100644 --- a/.gitlab/ci/jobs/cleanup.yml +++ b/.gitlab/ci/jobs/cleanup.yml @@ -26,4 +26,5 @@ cleanup: rules: - if: $CI_PIPELINE_SOURCE == "schedule" script: + - bash .gitlab/ci/scripts/check-runner-tools.sh werf - werf cleanup --repo dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization --without-kube=true diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 4f46f36de5..7142dc9345 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -39,11 +39,8 @@ lint:no-cyrillic: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true - image: - name: golang:1.25.11 - entrypoint: [''] before_script: - - go install github.com/go-task/task/v3/cmd/task@latest + - bash .gitlab/ci/scripts/check-runner-tools.sh go task script: - task validation:no-cyrillic rules: @@ -62,11 +59,8 @@ lint:doc-changes: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true - image: - name: golang:1.25.11 - entrypoint: [''] before_script: - - go install github.com/go-task/task/v3/cmd/task@latest + - bash .gitlab/ci/scripts/check-runner-tools.sh go task script: - task validation:doc-changes rules: @@ -88,11 +82,8 @@ lint:shellcheck: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true - image: - name: golang:1.25.11 - entrypoint: [''] before_script: - - go install github.com/go-task/task/v3/cmd/task@latest + - bash .gitlab/ci/scripts/check-runner-tools.sh go task shellcheck script: - task lint:shellcheck rules: @@ -110,11 +101,8 @@ lint:yaml: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true - image: - name: golang:1.25.11 - entrypoint: [''] before_script: - - go install github.com/go-task/task/v3/cmd/task@latest + - bash .gitlab/ci/scripts/check-runner-tools.sh go task script: - task lint:prettier:yaml rules: @@ -144,11 +132,8 @@ lint:helm-templates: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true - image: - name: golang:1.25.11 - entrypoint: [''] before_script: - - go install github.com/go-task/task/v3/cmd/task@latest + - bash .gitlab/ci/scripts/check-runner-tools.sh go task script: - task validation:helm-templates rules: @@ -189,12 +174,8 @@ check:gens-files: # TODO_RUNNER_TAG: heavy jobs may need [deckhouse, large] once runners # are registered. interruptible: true - image: - name: golang:1.25.11 - entrypoint: [''] before_script: - - go install github.com/go-task/task/v3/cmd/task@latest - - apt-get update && apt-get install -y -qq git + - bash .gitlab/ci/scripts/check-runner-tools.sh go task git script: - | set -e @@ -263,11 +244,8 @@ lint:gitlab-ci: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true - image: - name: alpine:3.20 - entrypoint: [''] before_script: - - apk add --no-cache bash curl jq + - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq script: - bash .gitlab/ci/scripts/gitlab-ci-lint.sh rules: diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index a574f6f75b..bb73943482 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -34,10 +34,8 @@ mrs:summary: # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse - image: - name: node:24-alpine - entrypoint: [""] before_script: + - bash .gitlab/ci/scripts/check-runner-tools.sh node npm - corepack enable || true - cd .gitlab/scripts/js - npm ci --omit=dev || npm install --omit=dev diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index c34dd6dc28..ff937efd9b 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -42,11 +42,8 @@ svace:set-vars: stage: info # TODO_RUNNER_TAG: short-lived helper job, [deckhouse] is fine. interruptible: true - image: - name: alpine:3.20 - entrypoint: [''] before_script: - - apk add --no-cache bash + - bash .gitlab/ci/scripts/check-runner-tools.sh bash script: - | set -euo pipefail @@ -129,11 +126,8 @@ svace:notify: stage: cleanup # TODO_RUNNER_TAG: short-lived helper, [deckhouse] is fine. interruptible: true - image: - name: alpine:3.20 - entrypoint: [''] before_script: - - apk add --no-cache bash curl + - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl variables: GIT_STRATEGY: none needs: diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml index 817d83272d..e4cc0f2c6a 100644 --- a/.gitlab/ci/jobs/test.yml +++ b/.gitlab/ci/jobs/test.yml @@ -15,6 +15,7 @@ lint:virtualization-controller: stage: lint script: + - bash .gitlab/ci/scripts/check-runner-tools.sh task - task virtualization-controller:init - task virtualization-controller:lint # TODO: needs follow-up — dvcr lint target has known issues @@ -25,6 +26,7 @@ lint:virtualization-controller: test:virtualization-controller: stage: test script: + - bash .gitlab/ci/scripts/check-runner-tools.sh task - task virtualization-controller:init - task virtualization-controller:test:unit extends: @@ -33,6 +35,7 @@ test:virtualization-controller: test:hooks: stage: test script: + - bash .gitlab/ci/scripts/check-runner-tools.sh task - task hooks:test extends: - .dev diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index cc527a1c81..05f835abc2 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -44,6 +44,12 @@ translate:changelog: stage: notify tags: - deckhouse + before_script: + - bash .gitlab/ci/scripts/check-runner-tools.sh git curl jq python3 + - python3 -m venv /tmp/venv + - /tmp/venv/bin/pip install --no-cache-dir deep-translator packaging + - git -C "${CI_PROJECT_DIR}" config user.email "gitlab-ci@gitlab.com" + - git -C "${CI_PROJECT_DIR}" config user.name "GitLab CI" variables: # Path to CHANGELOG dir (upstream default; explicit for clarity). TRANSLATE_CHANGELOG_PATH: "CHANGELOG" @@ -53,3 +59,5 @@ translate:changelog: # script. GitLab does not evaluate shell parameter expansion in `variables:`; # set RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN # does not have enough permissions to push and create merge requests. + # The local before_script intentionally replaces the upstream template's + # container/package-manager setup so the job works on shell executors. diff --git a/.gitlab/ci/scripts/check-runner-tools.sh b/.gitlab/ci/scripts/check-runner-tools.sh new file mode 100644 index 0000000000..2b5d2c0c70 --- /dev/null +++ b/.gitlab/ci/scripts/check-runner-tools.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ "$#" -eq 0 ]]; then + echo "Usage: $0 [...]" >&2 + exit 2 +fi + +missing=() +for tool in "$@"; do + if ! command -v "$tool" >/dev/null 2>&1; then + missing+=("$tool") + fi +done + +if [[ "${#missing[@]}" -gt 0 ]]; then + echo "ERROR: required runner tool(s) are missing: ${missing[*]}" >&2 + echo "This pipeline is designed for GitLab Runner shell executor." >&2 + echo "Container images and package-manager installs are not used by these jobs; install the tools on the runner host." >&2 + exit 1 +fi + +printf 'Runner tools OK:' +printf ' %s' "$@" +printf '\n' diff --git a/.gitlab/ci/templates/build.yml b/.gitlab/ci/templates/build.yml index d7c68a26a4..7ce98acd04 100644 --- a/.gitlab/ci/templates/build.yml +++ b/.gitlab/ci/templates/build.yml @@ -31,6 +31,7 @@ .local_build: stage: build script: + - bash .gitlab/ci/scripts/check-runner-tools.sh werf jq crane # Use gitlab ci job token - | SOURCE_REPO=${SOURCE_REPO#git@} diff --git a/.gitlab/ci/templates/deploy.yml b/.gitlab/ci/templates/deploy.yml index 8626f48e66..e44ff2d7ff 100644 --- a/.gitlab/ci/templates/deploy.yml +++ b/.gitlab/ci/templates/deploy.yml @@ -20,6 +20,7 @@ .local_deploy: stage: deploy script: + - bash .gitlab/ci/scripts/check-runner-tools.sh crane - | REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" From e77ec99ae6d217d3b635c5ee0eb21c0fe81a8c33 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 16:27:08 +0300 Subject: [PATCH 07/60] fix(ci): keep gitlab api logs off stdout Signed-off-by: Nikita Korolev --- .gitlab/ci/scripts/lib/api.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/scripts/lib/api.sh b/.gitlab/ci/scripts/lib/api.sh index 3fa55da965..debd04490b 100755 --- a/.gitlab/ci/scripts/lib/api.sh +++ b/.gitlab/ci/scripts/lib/api.sh @@ -53,7 +53,7 @@ gl_required_env() { gl_log_call() { local method="$1" local path="$2" - echo ">>> ${method} ${CI_API_V4_URL}${path}" + echo ">>> ${method} ${CI_API_V4_URL}${path}" >&2 } # api METHOD PATH [extra curl args] @@ -91,7 +91,7 @@ api() { rm -f "$response_file" if [[ "$http_code" =~ ^2 ]]; then - echo "<<< status=${http_code}" + echo "<<< status=${http_code}" >&2 return 0 fi From 3449008e9b5bf2a4445a7276ce921c6f62c72da5 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 16:36:17 +0300 Subject: [PATCH 08/60] fix(ci): harden gitlab helper scripts Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/svace.yml | 4 +-- .gitlab/ci/scripts/lib/api.sh | 7 ++-- .gitlab/ci/scripts/setup-mr-settings.sh | 46 +++++++++++++------------ 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index ff937efd9b..6998c1e5f7 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -127,7 +127,7 @@ svace:notify: # TODO_RUNNER_TAG: short-lived helper, [deckhouse] is fine. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl + - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq variables: GIT_STRATEGY: none needs: @@ -149,7 +149,7 @@ svace:notify: curl --silent --show-error --fail \ --request POST \ --header 'Content-Type: application/json' \ - --data "$(printf '{"text": "%s"}' "${MESSAGE}")" \ + --data "$(jq -n --arg text "${MESSAGE}" '{text: $text}')" \ "${LOOP_WEBHOOK_URL}" || echo "Loop webhook failed (non-fatal)" rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' diff --git a/.gitlab/ci/scripts/lib/api.sh b/.gitlab/ci/scripts/lib/api.sh index debd04490b..ded8d0674a 100755 --- a/.gitlab/ci/scripts/lib/api.sh +++ b/.gitlab/ci/scripts/lib/api.sh @@ -87,14 +87,15 @@ api() { --header "Accept: application/json" \ "${CI_API_V4_URL}${path}" "$@")" - cat "$response_file" - rm -f "$response_file" - if [[ "$http_code" =~ ^2 ]]; then + cat "$response_file" + rm -f "$response_file" echo "<<< status=${http_code}" >&2 return 0 fi + cat "$response_file" >&2 + rm -f "$response_file" echo "<<< status=${http_code} (FAILED)" >&2 return 1 } diff --git a/.gitlab/ci/scripts/setup-mr-settings.sh b/.gitlab/ci/scripts/setup-mr-settings.sh index f7ad132126..f6101af37f 100644 --- a/.gitlab/ci/scripts/setup-mr-settings.sh +++ b/.gitlab/ci/scripts/setup-mr-settings.sh @@ -112,32 +112,34 @@ EOF SETTINGS_PATH="/projects/${PROJECT_ID}" -run() { - if [[ "$DRY_RUN" == "true" ]]; then - echo "DRY-RUN: $@" - else - eval "$@" - fi -} +curl_args=( + curl --silent --show-error --request PUT + --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" + --header "Content-Type: application/json" + --data "${SETTINGS_BODY}" + "${API_BASE}${SETTINGS_PATH}" +) echo "Applying project MR settings to project_id=${PROJECT_ID}..." # Push the settings payload via PUT. -run curl --silent --show-error --request PUT \ - --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" \ - --header "Content-Type: application/json" \ - --data "${SETTINGS_BODY}" \ - "${API_BASE}${SETTINGS_PATH}" \ - | jq '{ - id, name, path_with_namespace, - merge_method, squash_option, - remove_source_branch_after_merge, - only_allow_merge_if_pipeline_succeeds, - only_allow_merge_if_all_discussions_are_resolved, - allow_merge_on_skipped_pipeline, - resolve_outdated_diff_discussions, - printing_merge_request_link_enabled - }' +if [[ "$DRY_RUN" == "true" ]]; then + printf 'DRY-RUN:' + printf ' %q' "${curl_args[@]}" + printf '\n' +else + "${curl_args[@]}" \ + | jq '{ + id, name, path_with_namespace, + merge_method, squash_option, + remove_source_branch_after_merge, + only_allow_merge_if_pipeline_succeeds, + only_allow_merge_if_all_discussions_are_resolved, + allow_merge_on_skipped_pipeline, + resolve_outdated_diff_discussions, + printing_merge_request_link_enabled + }' +fi # Approvers: leave empty unless team decides on a default approver group. # Push rules: file_size_limit and other settings are configured via UI From 7a19ac035bc31260753b673a72bc1b286f1faf2d Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 16:59:01 +0300 Subject: [PATCH 09/60] fix(ci): fetch diff base for validation jobs Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 7142dc9345..98bea46984 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -39,8 +39,11 @@ lint:no-cyrillic: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true + variables: + GIT_DEPTH: "0" before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task + - bash .gitlab/ci/scripts/check-runner-tools.sh go task git + - git fetch --no-tags origin +main:refs/remotes/origin/main script: - task validation:no-cyrillic rules: @@ -59,8 +62,11 @@ lint:doc-changes: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true + variables: + GIT_DEPTH: "0" before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task + - bash .gitlab/ci/scripts/check-runner-tools.sh go task git + - git fetch --no-tags origin +main:refs/remotes/origin/main script: - task validation:doc-changes rules: From 2f09cdd2e433ea0400037d84300d95146f9fa088 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 17:18:34 +0300 Subject: [PATCH 10/60] fix(ci): correct auto-assign rules and dedupe pipelines - auto-assign-author: run on every MR event instead of gating on changes to the job's own files, so the MR author is actually auto-assigned when no assignee is set (the script is idempotent and skips when an assignee already exists) - workflow: skip redundant branch pipelines when an open MR exists (CI_OPEN_MERGE_REQUESTS guard) to avoid duplicate branch + MR pipelines - lint-validate: drop unused RUN_ON_CHANGE matrix var from check:gens-files and fold vm-route-forge into the single COMPONENT matrix Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/auto-assign-author.yml | 12 +++++------- .gitlab/ci/jobs/lint-validate.yml | 4 +--- .gitlab/ci/workflow.yml | 10 ++++++++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml index adc414498c..efcf4b3de1 100644 --- a/.gitlab/ci/jobs/auto-assign-author.yml +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -29,11 +29,9 @@ auto-assign-author: script: - bash .gitlab/ci/scripts/auto-assign-author.sh rules: - # Only run on MR open/reopen events; do not re-run on every push. + # Run on every MR pipeline. The script is idempotent: it skips when the MR + # already has an assignee, so re-running on each push is harmless. Gating on + # changes: to this job's own files (the previous behaviour) was wrong — it + # meant the author was only auto-assigned when the auto-assign files changed, + # which never happens on normal MRs. - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - paths: - - .gitlab/ci/scripts/auto-assign-author.sh - - .gitlab/ci/jobs/auto-assign-author.yml - # Open MR without a triggering push (e.g. from Run pipeline UI) is also covered - # by the merge_request_event pipeline source above; no manual trigger needed. diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 98bea46984..d6d9126e05 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -216,9 +216,7 @@ check:gens-files: esac parallel: matrix: - - COMPONENT: [virtualization-artifact, api] - - COMPONENT: vm-route-forge - RUN_ON_CHANGE: "true" + - COMPONENT: [virtualization-artifact, api, vm-route-forge] rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: diff --git a/.gitlab/ci/workflow.yml b/.gitlab/ci/workflow.yml index f4ba4fcbdd..8acfacd676 100644 --- a/.gitlab/ci/workflow.yml +++ b/.gitlab/ci/workflow.yml @@ -18,9 +18,19 @@ workflow: rules: + # MR pipelines always run. - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Avoid duplicate pipelines: when a branch push has an open MR, GitLab + # would otherwise create both a branch pipeline and an MR pipeline. Skip + # the redundant branch pipeline and keep only the MR one. (The default + # branch and release/tag pushes are handled by the explicit rules below, + # which take precedence because rules are first-match.) - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(-dev.*)?$/ - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_PIPELINE_SOURCE == "web" + # Plain branch pushes (non-default, non-tag) only create a pipeline when + # there is no open MR for the branch, preventing duplicate pipelines. + - if: $CI_PIPELINE_SOURCE == "push" && $CI_OPEN_MERGE_REQUESTS + when: never - if: $CI_PIPELINE_SOURCE == "push" From 36b899660699fa3e893381fb782b4485b22a3193 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 18:00:24 +0300 Subject: [PATCH 11/60] fix(ci): use gohooks task name and enable Go toolchain auto-download test:hooks job ran 'task hooks:test' which does not exist (the task is gohooks:test), matching the migration plan and Taskfile. lint/test:virtualization-controller jobs failed at 'task virtualization-controller:init' because runner go (1.25.9, GOTOOLCHAIN=go1.25.9) is older than go.mod's 'go 1.25.11'. Add global GOTOOLCHAIN=auto so the host toolchain auto-downloads the version required by go.mod, mirroring actions/setup-go@v5. GitLab CI variables override runner environment variables. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/test.yml | 4 ++-- .gitlab/ci/variables.yml | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml index e4cc0f2c6a..6be58072df 100644 --- a/.gitlab/ci/jobs/test.yml +++ b/.gitlab/ci/jobs/test.yml @@ -3,7 +3,7 @@ # Carries forward the lint/test jobs from the previous root .gitlab-ci.yml: # lint:virtualization-controller -> task virtualization-controller:init + lint # test:virtualization-controller -> task virtualization-controller:init + test:unit -# test:hooks -> task hooks:test +# test:hooks -> task gohooks:test # # All three extend .dev so they only run on MR pipelines with the right # MODULES_MODULE_TAG/registry context. The other lint checks (yaml-lint, @@ -36,6 +36,6 @@ test:hooks: stage: test script: - bash .gitlab/ci/scripts/check-runner-tools.sh task - - task hooks:test + - task gohooks:test extends: - .dev diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index 0fb25e55e6..f504b3dabf 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -57,4 +57,12 @@ variables: # --- Build/dev tooling (kept from previous root) --- GO_VERSION: "1.25.11" GOLANGCI_LINT_VERSION: "2.11.1" + # Let the host Go toolchain auto-download the version required by go.mod + # (currently go 1.25.11). Runner hosts may ship an older Go with + # GOTOOLCHAIN pinned to that older version (e.g. GOTOOLCHAIN=go1.25.9), + # which otherwise makes `go mod download` / `go install` fail with + # "go.mod requires go >= 1.25.11". `auto` overrides the runner env var + # (GitLab CI variables take precedence over runner environment variables) + # and matches the behavior of GitHub's actions/setup-go@v5. + GOTOOLCHAIN: "auto" WERF_EXPERIMENTAL_IMPORT_BY_SOURCE_IMAGE_TAG: "true" From 88485e93c12fafc952ec728f2740b4477f6bba8e Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 19:09:45 +0300 Subject: [PATCH 12/60] fix(ci): stop svace dotenv leaking svace tag into MR build jobs build_dev pushed tag 'chore_ci_add-gitlab-ci-svace' instead of mr because svace:set-vars ran on every MR (no rules) and emitted a dotenv MODULES_MODULE_TAG=-svace, and build_dev had no needs: so GitLab's legacy artifact inheritance loaded that dotenv and overrode .dev's mr${CI_MERGE_REQUEST_IID} tag. Add needs: [] to build_dev/build_dev_tags/build_main to opt into DAG mode with no artifact downloads (their tag/registry vars come from their own templates). Gate svace:set-vars to the same SVACE_PIPELINE_ENABLED / schedule / web contexts as svace:build so it no longer runs on ordinary MR pipelines. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/build-dev.yml | 11 +++++++++++ .gitlab/ci/jobs/svace.yml | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml index 68abdf8f8d..49fc4a2a5e 100644 --- a/.gitlab/ci/jobs/build-dev.yml +++ b/.gitlab/ci/jobs/build-dev.yml @@ -23,14 +23,24 @@ # older main pipeline. build_dev and build_dev_tags inherit the project # default (interruptible not set, so they can be cancelled manually). +# `needs: []` is intentional: it opts these jobs into DAG mode with no +# artifact downloads. Without it, GitLab's legacy behavior downloads ALL +# artifacts from every prior-stage job, which leaks unrelated dotenv +# artifacts (e.g. svace:set-vars writes MODULES_MODULE_TAG=-svace) +# and overrides the per-template MODULES_MODULE_TAG from .dev / .dev_tags / +# .main. These builds derive their tag and registry vars from their own +# templates and need no upstream artifacts. + build_dev: stage: build + needs: [] extends: - .local_build - .dev build_dev_tags: stage: build + needs: [] extends: - .local_build - .dev_tags @@ -38,6 +48,7 @@ build_dev_tags: build_main: stage: build interruptible: true + needs: [] extends: - .local_build - .main diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index 6998c1e5f7..32dc221a41 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -42,6 +42,19 @@ svace:set-vars: stage: info # TODO_RUNNER_TAG: short-lived helper job, [deckhouse] is fine. interruptible: true + # Only emit the svace tag dotenv in contexts where svace actually runs. + # Without these rules this job ran on EVERY MR pipeline and leaked a + # MODULES_MODULE_TAG=-svace dotenv artifact into build jobs, + # overriding their own mr tag. Runs automatically (no `when: manual`) + # so the manual svace:build job can consume its artifact. + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' + when: always + - when: never before_script: - bash .gitlab/ci/scripts/check-runner-tools.sh bash script: From 4a6c88a9f1de8c88ea31456108ea96c920141f4f Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 19:37:38 +0300 Subject: [PATCH 13/60] fix(ci): auto-install shellcheck on runner when missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lint:shellcheck failed on the shell executor because shellcheck is not preinstalled on the runner host, and check-runner-tools.sh hard-required it. Add .gitlab/ci/scripts/install-shellcheck.sh which downloads the official koalaman/shellcheck pre-compiled static binary from GitHub Releases into ~/.local/bin when shellcheck is missing (no-op when already on PATH). This is distro-agnostic — works on both apt- and rpm-based runners without root. lint:shellcheck before_script drops shellcheck from the check-runner-tools guard and runs install-shellcheck.sh instead, so 'task lint:shellcheck' in script: finds it on PATH (the shell executor shares one bash session between before_script and script). Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 3 +- .gitlab/ci/scripts/install-shellcheck.sh | 99 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 .gitlab/ci/scripts/install-shellcheck.sh diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index d6d9126e05..77bfbbae62 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -89,7 +89,8 @@ lint:shellcheck: # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task shellcheck + - bash .gitlab/ci/scripts/check-runner-tools.sh go task + - bash .gitlab/ci/scripts/install-shellcheck.sh script: - task lint:shellcheck rules: diff --git a/.gitlab/ci/scripts/install-shellcheck.sh b/.gitlab/ci/scripts/install-shellcheck.sh new file mode 100644 index 0000000000..5b4ba57589 --- /dev/null +++ b/.gitlab/ci/scripts/install-shellcheck.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install ShellCheck into a writable directory if it is not already on $PATH. +# +# Used by the lint:shellcheck CI job on GitLab Runner shell executors where +# the tool is not preinstalled on the host (the runner pool mixes apt- and +# rpm-based hosts, so a distro-agnostic install is required). Downloads the +# official pre-compiled static binary from the koalaman/shellcheck GitHub +# Releases — works on any Linux with curl/wget + tar + xz, no root needed. +# +# Usage: +# bash .gitlab/ci/scripts/install-shellcheck.sh [version] [install_dir] +# +# Defaults: +# version SHELLCHECK_VERSION env var, or "v0.11.0" (latest stable as +# of 2026-06). Pin via env var to bump without editing YAML. +# install_dir SHELLCHECK_INSTALL_DIR env var, or "${HOME}/.local/bin" +# (created if missing). +# +# Environment: +# SHELLCHECK_VERSION override the version to install +# SHELLCHECK_INSTALL_DIR override the install directory +# SHELLCHECK_ARCH override the release arch (x86_64 or aarch64; +# auto-detected from uname -m when unset) +# +# Exit codes: +# 0 - shellcheck is available (preinstalled or just installed) +# 1 - download/extract failed +# 2 - missing prerequisites (curl/wget, tar, xz) + +set -euo pipefail + +log() { printf '[install-shellcheck] %s\n' "$*"; } +fail() { printf '[install-shellcheck] ERROR: %s\n' "$*" >&2; exit 1; } + +# If shellcheck is already on PATH, nothing to do. +if command -v shellcheck >/dev/null 2>&1; then + log "shellcheck already installed: $(command -v shellcheck)" + shellcheck --version | sed -n '1,2p' + exit 0 +fi + +VERSION="${1:-${SHELLCHECK_VERSION:-v0.11.0}}" +INSTALL_DIR="${2:-${SHELLCHECK_INSTALL_DIR:-${HOME}/.local/bin}}" + +# Resolve the release arch suffix (default x86_64; aarch64 on arm64). +if [[ -z "${SHELLCHECK_ARCH:-}" ]]; then + case "$(uname -m)" in + aarch64|arm64) SHELLCHECK_ARCH="aarch64" ;; + *) SHELLCHECK_ARCH="x86_64" ;; + esac +fi +ARCH_DIR="shellcheck-${VERSION}.linux.${SHELLCHECK_ARCH}" + +# Prerequisites: a downloader + tar with xz support. +if command -v curl >/dev/null 2>&1; then + fetch() { curl -fsSL "$1"; } +elif command -v wget >/dev/null 2>&1; then + fetch() { wget -qO- "$1"; } +else + fail "neither curl nor wget is available; cannot download shellcheck" +fi +command -v tar >/dev/null 2>&1 || fail "tar is required to extract the release tarball" +# tar -xJ needs xz; check it explicitly so the error is readable. +command -v xz >/dev/null 2>&1 || fail "xz is required to decompress the .tar.xz release; install the xz-utils/xz package on the runner" + +mkdir -p "${INSTALL_DIR}" + +URL="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/shellcheck-${VERSION}.linux.${SHELLCHECK_ARCH}" + +log "Installing shellcheck ${VERSION} -> ${INSTALL_DIR}" +log "Downloading ${URL}" +fetch "${URL}" | tar -xJ --strip-components=1 -C "${INSTALL_DIR}" "${ARCH_DIR}/shellcheck" + +chmod +x "${INSTALL_DIR}/shellcheck" + +# Ensure the install dir is on PATH for the rest of the job. On the GitLab +# shell executor before_script and script share one bash session, so this +# export carries over to the task invocation in the script: section. +case ":${PATH}:" in + *":${INSTALL_DIR}:"*) ;; + *) export PATH="${INSTALL_DIR}:${PATH}" ;; +esac + +log "Installed: ${INSTALL_DIR}/shellcheck" +"${INSTALL_DIR}/shellcheck" --version | sed -n '1,2p' From 99bac2ee0a0dc699e10b9275b19fb4a1cdf23333 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 19:45:03 +0300 Subject: [PATCH 14/60] fix(ci): fix shellcheck download URL and tarball extraction install-shellcheck.sh failed with HTTP 404: the release asset is named shellcheck-.linux..tar.xz but the URL was missing the .tar.xz suffix. The inner tarball directory is also shellcheck- (no arch suffix), so the previous --strip-components path was wrong too. Append .tar.xz to the URL and extract into a temp dir, then locate the binary via 'find -name shellcheck' instead of hard-coding the inner directory name. Verified: curl returns 200, tar extracts the binary. Signed-off-by: Nikita Korolev --- .gitlab/ci/scripts/install-shellcheck.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/scripts/install-shellcheck.sh b/.gitlab/ci/scripts/install-shellcheck.sh index 5b4ba57589..1e0c7370e0 100644 --- a/.gitlab/ci/scripts/install-shellcheck.sh +++ b/.gitlab/ci/scripts/install-shellcheck.sh @@ -63,7 +63,6 @@ if [[ -z "${SHELLCHECK_ARCH:-}" ]]; then *) SHELLCHECK_ARCH="x86_64" ;; esac fi -ARCH_DIR="shellcheck-${VERSION}.linux.${SHELLCHECK_ARCH}" # Prerequisites: a downloader + tar with xz support. if command -v curl >/dev/null 2>&1; then @@ -79,13 +78,22 @@ command -v xz >/dev/null 2>&1 || fail "xz is required to decompress the .tar.xz mkdir -p "${INSTALL_DIR}" -URL="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/shellcheck-${VERSION}.linux.${SHELLCHECK_ARCH}" +# Release asset name is shellcheck-.linux..tar.xz. The inner +# directory of the tarball is just shellcheck- (no arch suffix), so +# extract into a temp dir and move the binary instead of hard-coding the path. +URL="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/shellcheck-${VERSION}.linux.${SHELLCHECK_ARCH}.tar.xz" log "Installing shellcheck ${VERSION} -> ${INSTALL_DIR}" log "Downloading ${URL}" -fetch "${URL}" | tar -xJ --strip-components=1 -C "${INSTALL_DIR}" "${ARCH_DIR}/shellcheck" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT +fetch "${URL}" | tar -xJ -C "${TMP_DIR}" -chmod +x "${INSTALL_DIR}/shellcheck" +# The archive contains a single shellcheck-/ directory. +BINARY="$(find "${TMP_DIR}" -type f -name shellcheck -print -quit)" +[[ -n "${BINARY}" ]] || fail "shellcheck binary not found in the extracted archive" + +install -m 0755 "${BINARY}" "${INSTALL_DIR}/shellcheck" # Ensure the install dir is on PATH for the rest of the job. On the GitLab # shell executor before_script and script share one bash session, so this From ba339e69efd8e7f58e2e6e39af1277d90e52bee9 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 19:57:44 +0300 Subject: [PATCH 15/60] fix(ci): use docker for lint:shellcheck, drop dead install-shellcheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit task lint:shellcheck runs shellcheck inside the koalaman/shellcheck-alpine Docker image (Taskfile.yaml), not via a host-installed binary, so install-shellcheck.sh was dead code — the v0.11.0 binary it installed was never used. Remove install-shellcheck.sh and have lint:shellcheck before_script check for docker instead. The job already ran successfully via the docker image; this just stops installing an unused binary on every run. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 6 +- .gitlab/ci/scripts/install-shellcheck.sh | 107 ----------------------- 2 files changed, 4 insertions(+), 109 deletions(-) delete mode 100644 .gitlab/ci/scripts/install-shellcheck.sh diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 77bfbbae62..5e9eb0ce5d 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -88,9 +88,11 @@ lint:shellcheck: stage: lint # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true + # The `task lint:shellcheck` target runs shellcheck inside the + # koalaman/shellcheck-alpine Docker image, so the runner needs docker — + # not a host-installed shellcheck binary. before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task - - bash .gitlab/ci/scripts/install-shellcheck.sh + - bash .gitlab/ci/scripts/check-runner-tools.sh go task docker script: - task lint:shellcheck rules: diff --git a/.gitlab/ci/scripts/install-shellcheck.sh b/.gitlab/ci/scripts/install-shellcheck.sh deleted file mode 100644 index 1e0c7370e0..0000000000 --- a/.gitlab/ci/scripts/install-shellcheck.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2026 Flant JSC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install ShellCheck into a writable directory if it is not already on $PATH. -# -# Used by the lint:shellcheck CI job on GitLab Runner shell executors where -# the tool is not preinstalled on the host (the runner pool mixes apt- and -# rpm-based hosts, so a distro-agnostic install is required). Downloads the -# official pre-compiled static binary from the koalaman/shellcheck GitHub -# Releases — works on any Linux with curl/wget + tar + xz, no root needed. -# -# Usage: -# bash .gitlab/ci/scripts/install-shellcheck.sh [version] [install_dir] -# -# Defaults: -# version SHELLCHECK_VERSION env var, or "v0.11.0" (latest stable as -# of 2026-06). Pin via env var to bump without editing YAML. -# install_dir SHELLCHECK_INSTALL_DIR env var, or "${HOME}/.local/bin" -# (created if missing). -# -# Environment: -# SHELLCHECK_VERSION override the version to install -# SHELLCHECK_INSTALL_DIR override the install directory -# SHELLCHECK_ARCH override the release arch (x86_64 or aarch64; -# auto-detected from uname -m when unset) -# -# Exit codes: -# 0 - shellcheck is available (preinstalled or just installed) -# 1 - download/extract failed -# 2 - missing prerequisites (curl/wget, tar, xz) - -set -euo pipefail - -log() { printf '[install-shellcheck] %s\n' "$*"; } -fail() { printf '[install-shellcheck] ERROR: %s\n' "$*" >&2; exit 1; } - -# If shellcheck is already on PATH, nothing to do. -if command -v shellcheck >/dev/null 2>&1; then - log "shellcheck already installed: $(command -v shellcheck)" - shellcheck --version | sed -n '1,2p' - exit 0 -fi - -VERSION="${1:-${SHELLCHECK_VERSION:-v0.11.0}}" -INSTALL_DIR="${2:-${SHELLCHECK_INSTALL_DIR:-${HOME}/.local/bin}}" - -# Resolve the release arch suffix (default x86_64; aarch64 on arm64). -if [[ -z "${SHELLCHECK_ARCH:-}" ]]; then - case "$(uname -m)" in - aarch64|arm64) SHELLCHECK_ARCH="aarch64" ;; - *) SHELLCHECK_ARCH="x86_64" ;; - esac -fi - -# Prerequisites: a downloader + tar with xz support. -if command -v curl >/dev/null 2>&1; then - fetch() { curl -fsSL "$1"; } -elif command -v wget >/dev/null 2>&1; then - fetch() { wget -qO- "$1"; } -else - fail "neither curl nor wget is available; cannot download shellcheck" -fi -command -v tar >/dev/null 2>&1 || fail "tar is required to extract the release tarball" -# tar -xJ needs xz; check it explicitly so the error is readable. -command -v xz >/dev/null 2>&1 || fail "xz is required to decompress the .tar.xz release; install the xz-utils/xz package on the runner" - -mkdir -p "${INSTALL_DIR}" - -# Release asset name is shellcheck-.linux..tar.xz. The inner -# directory of the tarball is just shellcheck- (no arch suffix), so -# extract into a temp dir and move the binary instead of hard-coding the path. -URL="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/shellcheck-${VERSION}.linux.${SHELLCHECK_ARCH}.tar.xz" - -log "Installing shellcheck ${VERSION} -> ${INSTALL_DIR}" -log "Downloading ${URL}" -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "${TMP_DIR}"' EXIT -fetch "${URL}" | tar -xJ -C "${TMP_DIR}" - -# The archive contains a single shellcheck-/ directory. -BINARY="$(find "${TMP_DIR}" -type f -name shellcheck -print -quit)" -[[ -n "${BINARY}" ]] || fail "shellcheck binary not found in the extracted archive" - -install -m 0755 "${BINARY}" "${INSTALL_DIR}/shellcheck" - -# Ensure the install dir is on PATH for the rest of the job. On the GitLab -# shell executor before_script and script share one bash session, so this -# export carries over to the task invocation in the script: section. -case ":${PATH}:" in - *":${INSTALL_DIR}:"*) ;; - *) export PATH="${INSTALL_DIR}:${PATH}" ;; -esac - -log "Installed: ${INSTALL_DIR}/shellcheck" -"${INSTALL_DIR}/shellcheck" --version | sed -n '1,2p' From 0ead41ae0c59527f8718dda6e1b93f0a3b5ef47c Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 19:58:57 +0300 Subject: [PATCH 16/60] chore: formatting yaml Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 80 ++++++++++++------------- .gitlab/ci/jobs/deploy-dev.yml | 2 +- .gitlab/ci/jobs/deploy-prod.yml | 10 ++-- .gitlab/ci/jobs/lint-validate.yml | 70 +++++++++++----------- .gitlab/ci/jobs/translate-changelog.yml | 6 +- 5 files changed, 84 insertions(+), 84 deletions(-) diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index ad0b8b241e..e99f632de7 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -32,16 +32,16 @@ include: # --- Upstream modules-gitlab-ci (deckhouse/3p, ref v13.0) --- # TODO: pin to SHA after first green pipeline on virt-test; see migration # plan §11.2. Until then, branch ref v13.0 keeps fixes flowing. - - project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' - ref: 'v13.0' + - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" + ref: "v13.0" file: # Setup.gitlab-ci.yml provides trdl + werf setup + dual-registry # `werf cr login` in before_script. See templates/Setup.gitlab-ci.yml. - - '/templates/Setup.gitlab-ci.yml' + - "/templates/Setup.gitlab-ci.yml" # Validation/scanning jobs extend these upstream templates. - - '/templates/CVE_Scan.gitlab-ci.yml' - - '/templates/gitleaks.gitlab-ci.yml' - - '/templates/Svace_Analayze.gitlab-ci.yml' + - "/templates/CVE_Scan.gitlab-ci.yml" + - "/templates/gitleaks.gitlab-ci.yml" + - "/templates/Svace_Analayze.gitlab-ci.yml" # Build.gitlab-ci.yml and Deploy.gitlab-ci.yml are intentionally NOT # included directly. Their hidden jobs (.build, .deploy) ship with # `rules:` baked in (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`, @@ -51,45 +51,45 @@ include: # `.local_build` and `.local_deploy` and extend those instead. # --- Local structural fragments (order: stages, vars, defaults, then jobs) --- - - local: '.gitlab/ci/stages.yml' - - local: '.gitlab/ci/variables.yml' - - local: '.gitlab/ci/defaults.yml' - - local: '.gitlab/ci/workflow.yml' + - local: ".gitlab/ci/stages.yml" + - local: ".gitlab/ci/variables.yml" + - local: ".gitlab/ci/defaults.yml" + - local: ".gitlab/ci/workflow.yml" # --- Local shared templates (extends: ...) --- - - local: '.gitlab/ci/templates/dev_vars.yml' - - local: '.gitlab/ci/templates/prod_vars.yml' - - local: '.gitlab/ci/templates/dev.yml' - - local: '.gitlab/ci/templates/dev_tags.yml' - - local: '.gitlab/ci/templates/main.yml' - - local: '.gitlab/ci/templates/prod_manual.yml' - - local: '.gitlab/ci/templates/prod_always.yml' - - local: '.gitlab/ci/templates/info.yml' - - local: '.gitlab/ci/templates/dual_registry_login.yml' - - local: '.gitlab/ci/templates/build.yml' - - local: '.gitlab/ci/templates/deploy.yml' + - local: ".gitlab/ci/templates/dev_vars.yml" + - local: ".gitlab/ci/templates/prod_vars.yml" + - local: ".gitlab/ci/templates/dev.yml" + - local: ".gitlab/ci/templates/dev_tags.yml" + - local: ".gitlab/ci/templates/main.yml" + - local: ".gitlab/ci/templates/prod_manual.yml" + - local: ".gitlab/ci/templates/prod_always.yml" + - local: ".gitlab/ci/templates/info.yml" + - local: ".gitlab/ci/templates/dual_registry_login.yml" + - local: ".gitlab/ci/templates/build.yml" + - local: ".gitlab/ci/templates/deploy.yml" # --- Local job files owned by this issue --- - - local: '.gitlab/ci/jobs/info.yml' - - local: '.gitlab/ci/jobs/test.yml' - - local: '.gitlab/ci/jobs/build-dev.yml' - - local: '.gitlab/ci/jobs/build-prod.yml' - - local: '.gitlab/ci/jobs/deploy-dev.yml' - - local: '.gitlab/ci/jobs/deploy-prod.yml' - - local: '.gitlab/ci/jobs/cleanup.yml' + - local: ".gitlab/ci/jobs/info.yml" + - local: ".gitlab/ci/jobs/test.yml" + - local: ".gitlab/ci/jobs/build-dev.yml" + - local: ".gitlab/ci/jobs/build-prod.yml" + - local: ".gitlab/ci/jobs/deploy-dev.yml" + - local: ".gitlab/ci/jobs/deploy-prod.yml" + - local: ".gitlab/ci/jobs/cleanup.yml" # --- Local validation and scanning jobs --- - - local: '.gitlab/ci/jobs/lint-validate.yml' - - local: '.gitlab/ci/jobs/precache.yml' - - local: '.gitlab/ci/jobs/svace.yml' - - local: '.gitlab/ci/jobs/cve-scan.yml' - - local: '.gitlab/ci/jobs/gitleaks.yml' + - local: ".gitlab/ci/jobs/lint-validate.yml" + - local: ".gitlab/ci/jobs/precache.yml" + - local: ".gitlab/ci/jobs/svace.yml" + - local: ".gitlab/ci/jobs/cve-scan.yml" + - local: ".gitlab/ci/jobs/gitleaks.yml" # --- Local GitLab API automation, changelog, backport, and manual tools --- - - local: '.gitlab/ci/jobs/auto-assign-author.yml' - - local: '.gitlab/ci/jobs/backport.yml' - - local: '.gitlab/ci/jobs/changelog.yml' - - local: '.gitlab/ci/jobs/check-changelog.yml' - - local: '.gitlab/ci/jobs/check-milestone.yml' - - local: '.gitlab/ci/jobs/manual-tools.yml' - - local: '.gitlab/ci/jobs/translate-changelog.yml' + - local: ".gitlab/ci/jobs/auto-assign-author.yml" + - local: ".gitlab/ci/jobs/backport.yml" + - local: ".gitlab/ci/jobs/changelog.yml" + - local: ".gitlab/ci/jobs/check-changelog.yml" + - local: ".gitlab/ci/jobs/check-milestone.yml" + - local: ".gitlab/ci/jobs/manual-tools.yml" + - local: ".gitlab/ci/jobs/translate-changelog.yml" diff --git a/.gitlab/ci/jobs/deploy-dev.yml b/.gitlab/ci/jobs/deploy-dev.yml index 2873b36c65..dbf864a866 100644 --- a/.gitlab/ci/jobs/deploy-dev.yml +++ b/.gitlab/ci/jobs/deploy-dev.yml @@ -17,7 +17,7 @@ deploy_for_dev_tag: stage: deploy_dev - needs: ['build_dev_tags'] + needs: ["build_dev_tags"] extends: - .local_deploy - .dev_tags diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index 8a83675174..eea7e907c3 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -29,7 +29,7 @@ deploy_to_prod_alpha: stage: deploy_prod_alpha variables: RELEASE_CHANNEL: alpha - needs: ['build_prod'] + needs: ["build_prod"] extends: - .local_deploy - .prod_manual @@ -45,7 +45,7 @@ deploy_to_prod_beta: stage: deploy_prod_beta variables: RELEASE_CHANNEL: beta - needs: ['deploy_to_prod_alpha'] + needs: ["deploy_to_prod_alpha"] extends: - .local_deploy - .prod_manual @@ -61,7 +61,7 @@ deploy_to_prod_ea: stage: deploy_prod_ea variables: RELEASE_CHANNEL: early-access - needs: ['deploy_to_prod_beta'] + needs: ["deploy_to_prod_beta"] extends: - .local_deploy - .prod_manual @@ -77,7 +77,7 @@ deploy_to_prod_stable: stage: deploy_prod_stable variables: RELEASE_CHANNEL: stable - needs: ['deploy_to_prod_ea'] + needs: ["deploy_to_prod_ea"] extends: - .local_deploy - .prod_manual @@ -93,7 +93,7 @@ deploy_to_prod_rock_solid: stage: deploy_prod_rock_solid variables: RELEASE_CHANNEL: rock-solid - needs: ['deploy_to_prod_stable'] + needs: ["deploy_to_prod_stable"] extends: - .local_deploy - .prod_manual diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 5e9eb0ce5d..1df4201b7f 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -51,7 +51,7 @@ lint:no-cyrillic: when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - - if: '$CI_COMMIT_BRANCH =~ /^release-/' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" - if: '$CI_PIPELINE_SOURCE == "schedule"' # --------------------------------------------------------------------------- @@ -74,7 +74,7 @@ lint:doc-changes: when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - - if: '$CI_COMMIT_BRANCH =~ /^release-/' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" # --------------------------------------------------------------------------- # shellcheck @@ -100,7 +100,7 @@ lint:shellcheck: when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - - if: '$CI_COMMIT_BRANCH =~ /^release-/' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" # --------------------------------------------------------------------------- # yaml (prettier) @@ -118,17 +118,17 @@ lint:yaml: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: paths: - - '**/*.yaml' - - '**/*.yml' - - '.gitlab-ci.yml' - - '.gitlab/**/*' + - "**/*.yaml" + - "**/*.yml" + - ".gitlab-ci.yml" + - ".gitlab/**/*" - if: '$CI_COMMIT_BRANCH == "main"' changes: paths: - - '**/*.yaml' - - '**/*.yml' - - '.gitlab-ci.yml' - - '.gitlab/**/*' + - "**/*.yaml" + - "**/*.yml" + - ".gitlab-ci.yml" + - ".gitlab/**/*" # --------------------------------------------------------------------------- # helm_templates @@ -151,23 +151,23 @@ lint:helm-templates: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: paths: - - 'crds/**/*' - - 'charts/**/*' - - 'tools/kubeconform/**/*' - - 'templates/**/*' - - '.helmignore' - - 'Chart.yaml' - - 'Taskfile.yaml' + - "crds/**/*" + - "charts/**/*" + - "tools/kubeconform/**/*" + - "templates/**/*" + - ".helmignore" + - "Chart.yaml" + - "Taskfile.yaml" - if: '$CI_COMMIT_BRANCH == "main"' changes: paths: - - 'crds/**/*' - - 'charts/**/*' - - 'tools/kubeconform/**/*' - - 'templates/**/*' - - '.helmignore' - - 'Chart.yaml' - - 'Taskfile.yaml' + - "crds/**/*" + - "charts/**/*" + - "tools/kubeconform/**/*" + - "templates/**/*" + - ".helmignore" + - "Chart.yaml" + - "Taskfile.yaml" # --------------------------------------------------------------------------- # check_gens_files @@ -224,15 +224,15 @@ check:gens-files: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: paths: - - 'api/**/*' - - 'images/virtualization-artifact/**/*' - - 'go.mod' - - 'go.sum' + - "api/**/*" + - "images/virtualization-artifact/**/*" + - "go.mod" + - "go.sum" - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: paths: - - 'images/vm-route-forge/bpf/route_watcher.c' - - 'images/vm-route-forge/**/*' + - "images/vm-route-forge/bpf/route_watcher.c" + - "images/vm-route-forge/**/*" - if: '$CI_COMMIT_BRANCH == "main"' # --------------------------------------------------------------------------- @@ -259,11 +259,11 @@ lint:gitlab-ci: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: paths: - - '.gitlab-ci.yml' - - '.gitlab/**/*' + - ".gitlab-ci.yml" + - ".gitlab/**/*" - if: '$CI_COMMIT_BRANCH == "main"' changes: paths: - - '.gitlab-ci.yml' - - '.gitlab/**/*' + - ".gitlab-ci.yml" + - ".gitlab/**/*" - if: '$CI_PIPELINE_SOURCE == "schedule"' diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index 05f835abc2..7c98d07a2f 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -30,9 +30,9 @@ # (works in GitLab 15.9+ for push and MR create). include: - - project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' - ref: 'v13.0' - file: '/templates/Translate_Changelog.gitlab-ci.yml' + - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" + ref: "v13.0" + file: "/templates/Translate_Changelog.gitlab-ci.yml" # Local job definition that extends the upstream hidden job. # Tag override is intentional: we want our project's runner tag, not the From bd34f4b8614ec17b813bade3fba7794c2019e57f Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 20:07:22 +0300 Subject: [PATCH 17/60] ci(shellcheck): scan .gitlab/ci/scripts and make lint logs informative The lint:shellcheck task only scanned a hardcoded list of 5 paths and missed .gitlab/ci/scripts entirely, and produced no output when clean, so a passing job looked like nothing was checked. Collect files via 'find .gitlab/ci/scripts -type f -name *.sh' (whole tree, recursive) plus the existing explicit list, and log the file count, the file list, and a 'no issues found' footer. Use find instead of ** glob so it works under plain sh (task default shell, no globstar). Fix SC1091 in the 5 scripts that source lib/api.sh by making the '# shellcheck source=' directives repo-root-relative so shellcheck follows them. Suppress SC2154 for CI_* / GITLAB_API_TOKEN (runtime-injected by the GitLab Runner) with a file-level disable + explanation. Signed-off-by: Nikita Korolev --- .gitlab/ci/scripts/auto-assign-author.sh | 4 +++- .gitlab/ci/scripts/backport.sh | 4 +++- .gitlab/ci/scripts/check-milestone.sh | 4 +++- .gitlab/ci/scripts/lib/api.sh | 2 ++ .gitlab/ci/scripts/set-vars.sh | 4 +++- .gitlab/ci/scripts/setup-mr-settings.sh | 2 +- Taskfile.yaml | 24 +++++++++++++++++++----- 7 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.gitlab/ci/scripts/auto-assign-author.sh b/.gitlab/ci/scripts/auto-assign-author.sh index afc66f1f44..6fbbd33022 100644 --- a/.gitlab/ci/scripts/auto-assign-author.sh +++ b/.gitlab/ci/scripts/auto-assign-author.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + # Auto-assign MR author as the MR assignee. # # Migration of .github/workflows/dev_auto-pr-author-assign.yml which used the @@ -31,7 +33,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib/api.sh +# shellcheck source=.gitlab/ci/scripts/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_MERGE_REQUEST_IID diff --git a/.gitlab/ci/scripts/backport.sh b/.gitlab/ci/scripts/backport.sh index 17a780406c..9fea53f1e0 100644 --- a/.gitlab/ci/scripts/backport.sh +++ b/.gitlab/ci/scripts/backport.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + # Backport a merged MR to a release branch by opening a backport MR. # # Migration of .github/workflows/on-pull-request-backport.yml which used @@ -38,7 +40,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib/api.sh +# shellcheck source=.gitlab/ci/scripts/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_SERVER_HOST CI_PROJECT_PATH diff --git a/.gitlab/ci/scripts/check-milestone.sh b/.gitlab/ci/scripts/check-milestone.sh index a24dd9533d..75409e59ea 100644 --- a/.gitlab/ci/scripts/check-milestone.sh +++ b/.gitlab/ci/scripts/check-milestone.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + # Check that the current MR has a milestone assigned. # # Migration of .github/workflows/check-pr-milestone.yml which used @@ -29,7 +31,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib/api.sh +# shellcheck source=.gitlab/ci/scripts/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" if [[ "${CI_PIPELINE_SOURCE:-}" != "merge_request_event" ]]; then diff --git a/.gitlab/ci/scripts/lib/api.sh b/.gitlab/ci/scripts/lib/api.sh index ded8d0674a..20587d0cb6 100755 --- a/.gitlab/ci/scripts/lib/api.sh +++ b/.gitlab/ci/scripts/lib/api.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + # Shared GitLab API helpers for migration-era jobs. # # Source from a job's script: diff --git a/.gitlab/ci/scripts/set-vars.sh b/.gitlab/ci/scripts/set-vars.sh index 689ce10c44..02aa83542f 100755 --- a/.gitlab/ci/scripts/set-vars.sh +++ b/.gitlab/ci/scripts/set-vars.sh @@ -31,11 +31,13 @@ # reports: # dotenv: set_vars.env +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + set -euo pipefail # Source the api() helper for the GitLab API call below. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib/api.sh +# shellcheck source=.gitlab/ci/scripts/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" OUTPUT="${CI_PROJECT_DIR:-.}/set_vars.env" diff --git a/.gitlab/ci/scripts/setup-mr-settings.sh b/.gitlab/ci/scripts/setup-mr-settings.sh index f6101af37f..8ae138ecac 100644 --- a/.gitlab/ci/scripts/setup-mr-settings.sh +++ b/.gitlab/ci/scripts/setup-mr-settings.sh @@ -36,7 +36,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib/api.sh +# shellcheck source=.gitlab/ci/scripts/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" PROJECT_ID="${CI_PROJECT_ID:-}" diff --git a/Taskfile.yaml b/Taskfile.yaml index cc83d74680..0ac91d8501 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -188,16 +188,30 @@ tasks: desc: "Run shellcheck for CI shell scripts." cmds: - | + set -e + # Collect the explicit per-file list plus two recursive trees. + # Use find (not ** glob) so this works under plain sh / task default + # shell, which does not enable globstar. + EXPLICIT=".github/scripts/bash/e2e/*.sh + api/scripts/update-codegen.sh + images/virtualization-artifact/hack/args.sh + images/virtualization-artifact/hack/dlv.sh + images/virtualization-artifact/hack/pyroscope.sh" + EXPANDED="$( { for f in $EXPLICIT; do [ -f "$f" ] && echo "$f"; done; \ + find .gitlab/ci/scripts -type f -name '*.sh'; } | sort -u)" + if [ -z "$EXPANDED" ]; then + echo "No shell scripts found to lint."; exit 0 + fi + echo "==> shellcheck will scan $(printf '%s\n' "$EXPANDED" | wc -l | tr -d ' ') files:" + printf ' %s\n' $EXPANDED + echo "==> running shellcheck (koalaman/shellcheck-alpine:v0.10.0)" docker run --rm \ -v "$PWD:/mnt" \ -w /mnt \ koalaman/shellcheck-alpine:v0.10.0 \ shellcheck \ - .github/scripts/bash/e2e/*.sh \ - api/scripts/update-codegen.sh \ - images/virtualization-artifact/hack/args.sh \ - images/virtualization-artifact/hack/dlv.sh \ - images/virtualization-artifact/hack/pyroscope.sh + $EXPANDED + echo "==> shellcheck: no issues found" lint:actionlint: desc: "Run actionlint for E2E GitHub workflows." From 27972541946316ffd1c6bc5074210316ef077874 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 20:12:38 +0300 Subject: [PATCH 18/60] ci(shellcheck): bump koalaman/shellcheck-alpine v0.10.0 -> v0.11.0 v0.11.0 is the latest stable ShellCheck release (2025-08-04), available as the koalaman/shellcheck-alpine:v0.11.0 Docker Hub tag. Pin the version tag for reproducibility. Signed-off-by: Nikita Korolev --- Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 0ac91d8501..de728f99df 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -204,11 +204,11 @@ tasks: fi echo "==> shellcheck will scan $(printf '%s\n' "$EXPANDED" | wc -l | tr -d ' ') files:" printf ' %s\n' $EXPANDED - echo "==> running shellcheck (koalaman/shellcheck-alpine:v0.10.0)" + echo "==> running shellcheck (koalaman/shellcheck-alpine:v0.11.0)" docker run --rm \ -v "$PWD:/mnt" \ -w /mnt \ - koalaman/shellcheck-alpine:v0.10.0 \ + koalaman/shellcheck-alpine:v0.11.0 \ shellcheck \ $EXPANDED echo "==> shellcheck: no issues found" From eb5d67465871a6046ffc93038b2dea693f2a2e44 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 20:22:00 +0300 Subject: [PATCH 19/60] refactor(ci): split scripts into bash/ and python/ subfolders The .gitlab/ci/scripts directory mixed .sh and .py files together. Split into language-specific subfolders: .gitlab/ci/scripts/bash/ .sh wrappers + lib/api.sh .gitlab/ci/scripts/python/ .py scripts Update all references: job/template YAML invocations, the shellcheck source= directives, the wrapper -> python path resolution (/../python/), and the README tree. Taskfile lint:shellcheck now points directly at .gitlab/ci/scripts/bash/ (and bash/lib/) instead of scanning the whole scripts tree with find, since the bash scripts live in their own folder now. Signed-off-by: Nikita Korolev --- .gitlab/README.md | 34 +++++++++++-------- .gitlab/ci/changelog-sections.txt | 4 +-- .gitlab/ci/jobs/auto-assign-author.yml | 4 +-- .gitlab/ci/jobs/backport.yml | 4 +-- .gitlab/ci/jobs/changelog.yml | 8 ++--- .gitlab/ci/jobs/check-changelog.yml | 4 +-- .gitlab/ci/jobs/check-milestone.yml | 4 +-- .gitlab/ci/jobs/cleanup.yml | 2 +- .gitlab/ci/jobs/lint-validate.yml | 18 +++++----- .gitlab/ci/jobs/manual-tools.yml | 2 +- .gitlab/ci/jobs/svace.yml | 4 +-- .gitlab/ci/jobs/test.yml | 6 ++-- .gitlab/ci/jobs/translate-changelog.yml | 2 +- .../scripts/{ => bash}/auto-assign-author.sh | 2 +- .gitlab/ci/scripts/{ => bash}/backport.sh | 2 +- .../scripts/{ => bash}/changelog-milestone.sh | 2 +- .../{ => bash}/check-changelog-entry.sh | 2 +- .../ci/scripts/{ => bash}/check-milestone.sh | 2 +- .../scripts/{ => bash}/check-runner-tools.sh | 0 .../ci/scripts/{ => bash}/gitlab-ci-lint.sh | 4 +-- .gitlab/ci/scripts/{ => bash}/lib/api.sh | 2 +- .gitlab/ci/scripts/{ => bash}/set-vars.sh | 4 +-- .../scripts/{ => bash}/setup-mr-settings.sh | 2 +- .../scripts/{ => python}/changelog_collect.py | 0 .../{ => python}/check_changelog_entry.py | 0 .gitlab/ci/templates/build.yml | 2 +- .gitlab/ci/templates/deploy.yml | 2 +- Taskfile.yaml | 10 +++--- 28 files changed, 68 insertions(+), 64 deletions(-) rename .gitlab/ci/scripts/{ => bash}/auto-assign-author.sh (98%) rename .gitlab/ci/scripts/{ => bash}/backport.sh (99%) rename .gitlab/ci/scripts/{ => bash}/changelog-milestone.sh (94%) rename .gitlab/ci/scripts/{ => bash}/check-changelog-entry.sh (94%) rename .gitlab/ci/scripts/{ => bash}/check-milestone.sh (97%) rename .gitlab/ci/scripts/{ => bash}/check-runner-tools.sh (100%) rename .gitlab/ci/scripts/{ => bash}/gitlab-ci-lint.sh (98%) rename .gitlab/ci/scripts/{ => bash}/lib/api.sh (98%) rename .gitlab/ci/scripts/{ => bash}/set-vars.sh (97%) rename .gitlab/ci/scripts/{ => bash}/setup-mr-settings.sh (98%) rename .gitlab/ci/scripts/{ => python}/changelog_collect.py (100%) rename .gitlab/ci/scripts/{ => python}/check_changelog_entry.py (100%) diff --git a/.gitlab/README.md b/.gitlab/README.md index ed505af6ab..f703661016 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -39,7 +39,7 @@ For a release engineer: 1. Verify all CI/CD variables from [§3](#3-required-cicd-variables) exist in the project's `Settings -> CI/CD -> Variables`. Add missing ones — they are not auto-provisioned. -2. Run `bash .gitlab/ci/scripts/setup-mr-settings.sh --dry-run` to preview +2. Run `bash .gitlab/ci/scripts/bash/setup-mr-settings.sh --dry-run` to preview the project MR settings that the script will apply, then drop `--dry-run` to apply them once. 3. For a release: see [§8](#8-manual-pipelines) (`backport`, `changelog:milestone`, @@ -61,24 +61,28 @@ For a release engineer: │ ├── manual-tools.yml # mrs:summary (Loop notification) │ └── translate-changelog.yml # ru -> en changelog + MR └── scripts/ - ├── auto-assign-author.sh - ├── backport.sh - ├── changelog-milestone.sh # wrapper for changelog_collect.py - ├── changelog_collect.py - ├── check-changelog-entry.sh # wrapper for check_changelog_entry.py - ├── check-milestone.sh - ├── check-runner-tools.sh # shell-executor tool preflight - ├── check_changelog_entry.py - ├── setup-mr-settings.sh # one-off project settings - └── lib/ - └── api.sh # shared GitLab API helper + ├── bash/ + │ ├── auto-assign-author.sh + │ ├── backport.sh + │ ├── changelog-milestone.sh # wrapper for ../python/changelog_collect.py + │ ├── check-changelog-entry.sh # wrapper for ../python/check_changelog_entry.py + │ ├── check-milestone.sh + │ ├── check-runner-tools.sh # shell-executor tool preflight + │ ├── gitlab-ci-lint.sh + │ ├── set-vars.sh + │ ├── setup-mr-settings.sh # one-off project settings + │ └── lib/ + │ └── api.sh # shared GitLab API helper + └── python/ + ├── changelog_collect.py + └── check_changelog_entry.py .gitlab/scripts/js/ ├── package.json └── mrs_notifier.mjs # GitLab counterpart of prs_notifier.mjs ``` -Every job `extends` (or `include`s) a script in `.gitlab/ci/scripts/`. -Scripts source `.gitlab/ci/scripts/lib/api.sh` for the `api GET / POST / PUT` +Every job `extends` (or `include`s) a script in `.gitlab/ci/scripts/bash/`. +Scripts source `.gitlab/ci/scripts/bash/lib/api.sh` for the `api GET / POST / PUT` helper. ## 3. Required CI/CD variables @@ -203,7 +207,7 @@ The project runner is expected to use the GitLab Runner `shell` executor. For that executor, `image:` and container `entrypoint:` settings are ignored, so project jobs do not install packages with `apk`, `apt-get`, or other host package managers. Tools must already be installed on the runner host. Jobs that -need non-trivial tools call `.gitlab/ci/scripts/check-runner-tools.sh` in +need non-trivial tools call `.gitlab/ci/scripts/bash/check-runner-tools.sh` in `before_script` and fail early with a clear message if a tool is missing. Expected host tools for project-owned jobs: diff --git a/.gitlab/ci/changelog-sections.txt b/.gitlab/ci/changelog-sections.txt index 2cb45bbb8e..62480f4fbe 100644 --- a/.gitlab/ci/changelog-sections.txt +++ b/.gitlab/ci/changelog-sections.txt @@ -15,8 +15,8 @@ # Allowed `section` values for ```changes fenced blocks in MR descriptions. # # This list is shared between: -# - .gitlab/ci/scripts/check-changelog-entry.sh (validates MR description blocks) -# - .gitlab/ci/scripts/changelog_collect.py (groups changes by section) +# - .gitlab/ci/scripts/bash/check-changelog-entry.sh (validates MR description blocks) +# - .gitlab/ci/scripts/python/changelog_collect.py (groups changes by section) # - .github/actions/milestone-changelog/action.yml (kept in sync during migration) # # Suffix `:low` pins the section to low impact_level (impact_level field becomes optional). diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml index efcf4b3de1..206ee24a60 100644 --- a/.gitlab/ci/jobs/auto-assign-author.yml +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -25,9 +25,9 @@ auto-assign-author: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq script: - - bash .gitlab/ci/scripts/auto-assign-author.sh + - bash .gitlab/ci/scripts/bash/auto-assign-author.sh rules: # Run on every MR pipeline. The script is idempotent: it skips when the MR # already has an assignee, so re-running on each push is harmless. Gating on diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index f9a4b45790..1b070c5157 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -33,9 +33,9 @@ backport: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash git curl jq ssh-agent ssh-add + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add script: - - bash .gitlab/ci/scripts/backport.sh + - bash .gitlab/ci/scripts/bash/backport.sh rules: # Mode 1: explicit manual run with TARGET_BRANCH provided via UI. - if: $TARGET_BRANCH diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index 44c3dffdcc..c8a2c8a163 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -40,9 +40,9 @@ changelog:milestone: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 script: - - bash .gitlab/ci/scripts/changelog-milestone.sh + - bash .gitlab/ci/scripts/bash/changelog-milestone.sh variables: MILESTONE_TITLE: "" OPEN_CHANGELOG_MR: "false" @@ -71,9 +71,9 @@ changelog:all-active-milestones: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add python3 script: - - bash .gitlab/ci/scripts/changelog-milestone.sh + - bash .gitlab/ci/scripts/bash/changelog-milestone.sh variables: MILESTONE_TITLE: "" OPEN_CHANGELOG_MR: "false" diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml index 192587a159..d9dc4a26a6 100644 --- a/.gitlab/ci/jobs/check-changelog.yml +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -27,9 +27,9 @@ check:changelog: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq python3 + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq python3 script: - - bash .gitlab/ci/scripts/check-changelog-entry.sh + - bash .gitlab/ci/scripts/bash/check-changelog-entry.sh rules: # Skip-label must be first because GitLab rules are evaluated first-match. - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_changelog/ diff --git a/.gitlab/ci/jobs/check-milestone.yml b/.gitlab/ci/jobs/check-milestone.yml index c0dedbaca2..2a051229e9 100644 --- a/.gitlab/ci/jobs/check-milestone.yml +++ b/.gitlab/ci/jobs/check-milestone.yml @@ -24,9 +24,9 @@ check:milestone: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq script: - - bash .gitlab/ci/scripts/check-milestone.sh + - bash .gitlab/ci/scripts/bash/check-milestone.sh rules: # Skip-label must be first because GitLab rules are evaluated first-match. - if: $CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/check_milestone/ diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml index 9e316fa0cb..9702efaf65 100644 --- a/.gitlab/ci/jobs/cleanup.yml +++ b/.gitlab/ci/jobs/cleanup.yml @@ -26,5 +26,5 @@ cleanup: rules: - if: $CI_PIPELINE_SOURCE == "schedule" script: - - bash .gitlab/ci/scripts/check-runner-tools.sh werf + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf - werf cleanup --repo dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization --without-kube=true diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 1df4201b7f..3a59659cab 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -42,7 +42,7 @@ lint:no-cyrillic: variables: GIT_DEPTH: "0" before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task git + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git - git fetch --no-tags origin +main:refs/remotes/origin/main script: - task validation:no-cyrillic @@ -65,7 +65,7 @@ lint:doc-changes: variables: GIT_DEPTH: "0" before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task git + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git - git fetch --no-tags origin +main:refs/remotes/origin/main script: - task validation:doc-changes @@ -92,7 +92,7 @@ lint:shellcheck: # koalaman/shellcheck-alpine Docker image, so the runner needs docker — # not a host-installed shellcheck binary. before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task docker + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task docker script: - task lint:shellcheck rules: @@ -111,7 +111,7 @@ lint:yaml: # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task script: - task lint:prettier:yaml rules: @@ -142,7 +142,7 @@ lint:helm-templates: # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task script: - task validation:helm-templates rules: @@ -184,7 +184,7 @@ check:gens-files: # are registered. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh go task git + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git script: - | set -e @@ -244,7 +244,7 @@ check:gens-files: # # Authentication: PRIVATE-TOKEN via GITLAB_API_TOKEN (project access # token, scope api). The script lives in -# .gitlab/ci/scripts/gitlab-ci-lint.sh (owned by this issue). +# .gitlab/ci/scripts/bash/gitlab-ci-lint.sh (owned by this issue). # --------------------------------------------------------------------------- lint:gitlab-ci: @@ -252,9 +252,9 @@ lint:gitlab-ci: # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq script: - - bash .gitlab/ci/scripts/gitlab-ci-lint.sh + - bash .gitlab/ci/scripts/bash/gitlab-ci-lint.sh rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index bb73943482..54876fbcb5 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -35,7 +35,7 @@ mrs:summary: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh node npm + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh node npm - corepack enable || true - cd .gitlab/scripts/js - npm ci --omit=dev || npm install --omit=dev diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index 32dc221a41..b3bb6b0f29 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -56,7 +56,7 @@ svace:set-vars: when: always - when: never before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash script: - | set -euo pipefail @@ -140,7 +140,7 @@ svace:notify: # TODO_RUNNER_TAG: short-lived helper, [deckhouse] is fine. interruptible: true before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh bash curl jq + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq variables: GIT_STRATEGY: none needs: diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml index 6be58072df..4c84b750de 100644 --- a/.gitlab/ci/jobs/test.yml +++ b/.gitlab/ci/jobs/test.yml @@ -15,7 +15,7 @@ lint:virtualization-controller: stage: lint script: - - bash .gitlab/ci/scripts/check-runner-tools.sh task + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task - task virtualization-controller:init - task virtualization-controller:lint # TODO: needs follow-up — dvcr lint target has known issues @@ -26,7 +26,7 @@ lint:virtualization-controller: test:virtualization-controller: stage: test script: - - bash .gitlab/ci/scripts/check-runner-tools.sh task + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task - task virtualization-controller:init - task virtualization-controller:test:unit extends: @@ -35,7 +35,7 @@ test:virtualization-controller: test:hooks: stage: test script: - - bash .gitlab/ci/scripts/check-runner-tools.sh task + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task - task gohooks:test extends: - .dev diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index 7c98d07a2f..3993ed7d47 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -45,7 +45,7 @@ translate:changelog: tags: - deckhouse before_script: - - bash .gitlab/ci/scripts/check-runner-tools.sh git curl jq python3 + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh git curl jq python3 - python3 -m venv /tmp/venv - /tmp/venv/bin/pip install --no-cache-dir deep-translator packaging - git -C "${CI_PROJECT_DIR}" config user.email "gitlab-ci@gitlab.com" diff --git a/.gitlab/ci/scripts/auto-assign-author.sh b/.gitlab/ci/scripts/bash/auto-assign-author.sh similarity index 98% rename from .gitlab/ci/scripts/auto-assign-author.sh rename to .gitlab/ci/scripts/bash/auto-assign-author.sh index 6fbbd33022..c560891eb8 100644 --- a/.gitlab/ci/scripts/auto-assign-author.sh +++ b/.gitlab/ci/scripts/bash/auto-assign-author.sh @@ -33,7 +33,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=.gitlab/ci/scripts/lib/api.sh +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_MERGE_REQUEST_IID diff --git a/.gitlab/ci/scripts/backport.sh b/.gitlab/ci/scripts/bash/backport.sh similarity index 99% rename from .gitlab/ci/scripts/backport.sh rename to .gitlab/ci/scripts/bash/backport.sh index 9fea53f1e0..2361b974e3 100644 --- a/.gitlab/ci/scripts/backport.sh +++ b/.gitlab/ci/scripts/bash/backport.sh @@ -40,7 +40,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=.gitlab/ci/scripts/lib/api.sh +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_SERVER_HOST CI_PROJECT_PATH diff --git a/.gitlab/ci/scripts/changelog-milestone.sh b/.gitlab/ci/scripts/bash/changelog-milestone.sh similarity index 94% rename from .gitlab/ci/scripts/changelog-milestone.sh rename to .gitlab/ci/scripts/bash/changelog-milestone.sh index 758b302d0e..1e95490f9c 100644 --- a/.gitlab/ci/scripts/changelog-milestone.sh +++ b/.gitlab/ci/scripts/bash/changelog-milestone.sh @@ -31,4 +31,4 @@ else exit 1 fi -exec "${PYTHON_BIN}" "${SCRIPT_DIR}/changelog_collect.py" +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/../python/changelog_collect.py" diff --git a/.gitlab/ci/scripts/check-changelog-entry.sh b/.gitlab/ci/scripts/bash/check-changelog-entry.sh similarity index 94% rename from .gitlab/ci/scripts/check-changelog-entry.sh rename to .gitlab/ci/scripts/bash/check-changelog-entry.sh index bbe180867d..ba603ef871 100644 --- a/.gitlab/ci/scripts/check-changelog-entry.sh +++ b/.gitlab/ci/scripts/bash/check-changelog-entry.sh @@ -34,4 +34,4 @@ else exit 1 fi -exec "${PYTHON_BIN}" "${SCRIPT_DIR}/check_changelog_entry.py" +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/../python/check_changelog_entry.py" diff --git a/.gitlab/ci/scripts/check-milestone.sh b/.gitlab/ci/scripts/bash/check-milestone.sh similarity index 97% rename from .gitlab/ci/scripts/check-milestone.sh rename to .gitlab/ci/scripts/bash/check-milestone.sh index 75409e59ea..be96739a57 100644 --- a/.gitlab/ci/scripts/check-milestone.sh +++ b/.gitlab/ci/scripts/bash/check-milestone.sh @@ -31,7 +31,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=.gitlab/ci/scripts/lib/api.sh +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" if [[ "${CI_PIPELINE_SOURCE:-}" != "merge_request_event" ]]; then diff --git a/.gitlab/ci/scripts/check-runner-tools.sh b/.gitlab/ci/scripts/bash/check-runner-tools.sh similarity index 100% rename from .gitlab/ci/scripts/check-runner-tools.sh rename to .gitlab/ci/scripts/bash/check-runner-tools.sh diff --git a/.gitlab/ci/scripts/gitlab-ci-lint.sh b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh similarity index 98% rename from .gitlab/ci/scripts/gitlab-ci-lint.sh rename to .gitlab/ci/scripts/bash/gitlab-ci-lint.sh index cac252a07f..8d854e5d81 100755 --- a/.gitlab/ci/scripts/gitlab-ci-lint.sh +++ b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# .gitlab/ci/scripts/gitlab-ci-lint.sh +# .gitlab/ci/scripts/bash/gitlab-ci-lint.sh # # Calls the GitLab CI Lint API # POST ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint @@ -17,7 +17,7 @@ # Authentication: PRIVATE-TOKEN via GITLAB_API_TOKEN (Project Access # Token, scope api). Falls back to CI_JOB_TOKEN for read-only pipeline # scope when GITLAB_API_TOKEN is unset (matches the convention in -# .gitlab/ci/scripts/lib/api.sh). +# .gitlab/ci/scripts/bash/lib/api.sh). # # Exit codes: # 0 - CI config is valid. diff --git a/.gitlab/ci/scripts/lib/api.sh b/.gitlab/ci/scripts/bash/lib/api.sh similarity index 98% rename from .gitlab/ci/scripts/lib/api.sh rename to .gitlab/ci/scripts/bash/lib/api.sh index 20587d0cb6..96c4213b55 100755 --- a/.gitlab/ci/scripts/lib/api.sh +++ b/.gitlab/ci/scripts/bash/lib/api.sh @@ -18,7 +18,7 @@ # Shared GitLab API helpers for migration-era jobs. # # Source from a job's script: -# source .gitlab/ci/scripts/lib/api.sh +# source .gitlab/ci/scripts/bash/lib/api.sh # # Provides: # api METHOD PATH [curl-args...] -- REST call with PRIVATE-TOKEN, prints body, returns exit code. diff --git a/.gitlab/ci/scripts/set-vars.sh b/.gitlab/ci/scripts/bash/set-vars.sh similarity index 97% rename from .gitlab/ci/scripts/set-vars.sh rename to .gitlab/ci/scripts/bash/set-vars.sh index 02aa83542f..a0575e491d 100755 --- a/.gitlab/ci/scripts/set-vars.sh +++ b/.gitlab/ci/scripts/bash/set-vars.sh @@ -26,7 +26,7 @@ # set_vars: # stage: info # script: -# - bash .gitlab/ci/scripts/set-vars.sh +# - bash .gitlab/ci/scripts/bash/set-vars.sh # artifacts: # reports: # dotenv: set_vars.env @@ -37,7 +37,7 @@ set -euo pipefail # Source the api() helper for the GitLab API call below. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=.gitlab/ci/scripts/lib/api.sh +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" OUTPUT="${CI_PROJECT_DIR:-.}/set_vars.env" diff --git a/.gitlab/ci/scripts/setup-mr-settings.sh b/.gitlab/ci/scripts/bash/setup-mr-settings.sh similarity index 98% rename from .gitlab/ci/scripts/setup-mr-settings.sh rename to .gitlab/ci/scripts/bash/setup-mr-settings.sh index 8ae138ecac..34d754cd29 100644 --- a/.gitlab/ci/scripts/setup-mr-settings.sh +++ b/.gitlab/ci/scripts/bash/setup-mr-settings.sh @@ -36,7 +36,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=.gitlab/ci/scripts/lib/api.sh +# shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" PROJECT_ID="${CI_PROJECT_ID:-}" diff --git a/.gitlab/ci/scripts/changelog_collect.py b/.gitlab/ci/scripts/python/changelog_collect.py similarity index 100% rename from .gitlab/ci/scripts/changelog_collect.py rename to .gitlab/ci/scripts/python/changelog_collect.py diff --git a/.gitlab/ci/scripts/check_changelog_entry.py b/.gitlab/ci/scripts/python/check_changelog_entry.py similarity index 100% rename from .gitlab/ci/scripts/check_changelog_entry.py rename to .gitlab/ci/scripts/python/check_changelog_entry.py diff --git a/.gitlab/ci/templates/build.yml b/.gitlab/ci/templates/build.yml index 7ce98acd04..042880df84 100644 --- a/.gitlab/ci/templates/build.yml +++ b/.gitlab/ci/templates/build.yml @@ -31,7 +31,7 @@ .local_build: stage: build script: - - bash .gitlab/ci/scripts/check-runner-tools.sh werf jq crane + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf jq crane # Use gitlab ci job token - | SOURCE_REPO=${SOURCE_REPO#git@} diff --git a/.gitlab/ci/templates/deploy.yml b/.gitlab/ci/templates/deploy.yml index e44ff2d7ff..42b103839f 100644 --- a/.gitlab/ci/templates/deploy.yml +++ b/.gitlab/ci/templates/deploy.yml @@ -20,7 +20,7 @@ .local_deploy: stage: deploy script: - - bash .gitlab/ci/scripts/check-runner-tools.sh crane + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh crane - | REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" diff --git a/Taskfile.yaml b/Taskfile.yaml index de728f99df..501eca036f 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -189,16 +189,16 @@ tasks: cmds: - | set -e - # Collect the explicit per-file list plus two recursive trees. - # Use find (not ** glob) so this works under plain sh / task default - # shell, which does not enable globstar. + # The .gitlab/ci/scripts tree is split into bash/ and python/ folders, + # so lint:shellcheck points directly at the bash folder (no find scan). EXPLICIT=".github/scripts/bash/e2e/*.sh + .gitlab/ci/scripts/bash/*.sh + .gitlab/ci/scripts/bash/lib/*.sh api/scripts/update-codegen.sh images/virtualization-artifact/hack/args.sh images/virtualization-artifact/hack/dlv.sh images/virtualization-artifact/hack/pyroscope.sh" - EXPANDED="$( { for f in $EXPLICIT; do [ -f "$f" ] && echo "$f"; done; \ - find .gitlab/ci/scripts -type f -name '*.sh'; } | sort -u)" + EXPANDED="$(for f in $EXPLICIT; do [ -f "$f" ] && echo "$f"; done | sort -u)" if [ -z "$EXPANDED" ]; then echo "No shell scripts found to lint."; exit 0 fi From 1ebbb3dea3a2c90f6b2a1c01887ad7b6c819f510 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 20:57:16 +0300 Subject: [PATCH 20/60] fix(ci): install helm/jq in lint:helm-templates and migrate lint_go to lint:go Fixes two failing GitLab CI jobs and restores the lost "lint all Go modules" step from the GitHub Actions migration: - lint:helm-templates: install helm v3 and jq as portable prebuilt binaries in before_script (arch-aware, no sudo) instead of relying on the runner host; require docker/git/curl/python3 for kubeconform.sh. HELM_VERSION=3.16.3 and JQ_VERSION=1.7.1 added to variables.yml. kubeconform.sh itself is untouched. - lint:go: new job in lint-validate.yml, direct migration of the GH lint_go job from dev_module_build.yml. Installs the pinned prebuilt golangci-lint v${GOLANGCI_LINT_VERSION} via install.sh (fixes the flaky install.sh-latest checksum failure) and runs golangci-lint run in every directory shipping a .golangci.yaml, pruning the same upstream modules as GH (images/cdi-cloner/cloner-startup, images/dvcr-artifact, test/performance/shatal). - Remove the redundant single-dir lint:virtualization-controller job from test.yml (now covered by lint:go). - Taskfile.init.yaml: bump golangci-lint install from v1.64.8 to v2.11.1 via the v2 module path, aligning with GOLANGCI_LINT_VERSION and the v2-format .golangci.yaml configs in every subproject. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 98 ++++++++++++++++++- .gitlab/ci/jobs/test.yml | 28 ++---- .gitlab/ci/variables.yml | 5 + .../Taskfile.init.yaml | 5 +- 4 files changed, 114 insertions(+), 22 deletions(-) diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 3a59659cab..d350527008 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -130,6 +130,74 @@ lint:yaml: - ".gitlab-ci.yml" - ".gitlab/**/*" +# --------------------------------------------------------------------------- +# go (golangci-lint across every Go module) +# +# Migration of the GitHub Actions `lint_go` job from +# .github/workflows/dev_module_build.yml. It installs the pinned +# golangci-lint v${GOLANGCI_LINT_VERSION} (same method/version as the GH job) +# and runs `golangci-lint run` in every directory that ships a .golangci.yaml, +# excluding the upstream/vendored modules that GH also skipped: +# images/cdi-cloner/cloner-startup, images/dvcr-artifact, test/performance/shatal. +# This supersedes the old single-dir `lint:virtualization-controller` job. +# --------------------------------------------------------------------------- + +lint:go: + stage: lint + # TODO_RUNNER_TAG: lints every Go module; may need [deckhouse, large] once + # runners are registered. + interruptible: true + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go curl + - | + set -e + # Install the pinned prebuilt golangci-lint v2 binary. + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b "$(go env GOPATH)/bin" "v${GOLANGCI_LINT_VERSION}" + export PATH="$(go env GOPATH)/bin:${PATH}" + golangci-lint --version + script: + - | + set -e + export PATH="$(go env GOPATH)/bin:${PATH}" + + # Directories containing a .golangci.yaml, excluding the same upstream / + # vendored modules as .github/workflows/dev_module_build.yml (lint_go). + mapfile -t config_dirs < <( + find . \ + -path ./images/cdi-cloner/cloner-startup -prune -o \ + -path ./images/dvcr-artifact -prune -o \ + -path ./test/performance/shatal -prune -o \ + -type f -name '.golangci.yaml' -not -type l -printf '%h\0' | \ + xargs -0 -n1 | sort -u + ) + count=${#config_dirs[@]} + echo "Found ${count} directories with golangci-lint configurations" + + error_count=0 + for dir in "${config_dirs[@]}"; do + echo "------------------------------------------------------------" + echo "Linting: ${dir}" + pushd "$dir" >/dev/null + if ! golangci-lint run; then + error_count=$(( error_count + 1 )) + fi + popd >/dev/null + done + + if [ "$error_count" -gt 0 ]; then + echo "${error_count} directory/directories failed golangci-lint" + exit 1 + fi + echo "All golangci-lint checks passed" + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /validation\/skip\/go/' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + - if: '$CI_PIPELINE_SOURCE == "schedule"' + # --------------------------------------------------------------------------- # helm_templates # @@ -142,7 +210,35 @@ lint:helm-templates: # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task + # go + task are host tools; docker runs kubeconform; git/curl/python3 are + # needed by tools/kubeconform/kubeconform.sh (clone kubeconform.git, fetch + # CRD schemas, run openapi2jsonschema.py). + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task docker git curl python3 + - | + # Helm and jq are not guaranteed on the runner host. Install portable + # prebuilt binaries into a user-writable dir on PATH (no sudo needed). + set -e + case "$(uname -m)" in + x86_64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + *) arch="amd64" ;; + esac + bin_dir="$HOME/.local/bin" + mkdir -p "$bin_dir" + export PATH="$bin_dir:$PATH" + if ! command -v helm >/dev/null 2>&1; then + echo "Installing Helm v${HELM_VERSION} (${arch})..." + curl -sSfL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${arch}.tar.gz" -o /tmp/helm.tar.gz + tar -xzf /tmp/helm.tar.gz -C /tmp "linux-${arch}/helm" + install -m 0755 "/tmp/linux-${arch}/helm" "$bin_dir/helm" + helm version + fi + if ! command -v jq >/dev/null 2>&1; then + echo "Installing jq v${JQ_VERSION} (${arch})..." + curl -sSfL -o "$bin_dir/jq" "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${arch}" + chmod +x "$bin_dir/jq" + jq --version + fi script: - task validation:helm-templates rules: diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml index 4c84b750de..988e1aeb77 100644 --- a/.gitlab/ci/jobs/test.yml +++ b/.gitlab/ci/jobs/test.yml @@ -1,27 +1,15 @@ -# Lint + unit-test jobs. +# Unit-test jobs. # -# Carries forward the lint/test jobs from the previous root .gitlab-ci.yml: -# lint:virtualization-controller -> task virtualization-controller:init + lint +# Carries forward the test jobs from the previous root .gitlab-ci.yml: # test:virtualization-controller -> task virtualization-controller:init + test:unit # test:hooks -> task gohooks:test # -# All three extend .dev so they only run on MR pipelines with the right -# MODULES_MODULE_TAG/registry context. The other lint checks (yaml-lint, -# no-cyrillic, shellcheck, helm-templates, gen-files-check, dmt-lint, -# gitlab-ci-lint) live in .gitlab/ci/jobs/lint-validate.yml, which is owned -# by a different subagent issue — keep the include line for it commented -# here as a hint for the integration step. - -lint:virtualization-controller: - stage: lint - script: - - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task - - task virtualization-controller:init - - task virtualization-controller:lint - # TODO: needs follow-up — dvcr lint target has known issues - # - task virtualization-controller:dvcr:lint - extends: - - .dev +# Go linting of every module (.golangci.yaml dirs) lives in +# .gitlab/ci/jobs/lint-validate.yml -> lint:go (migration of the GH lint_go job); +# the old single-dir lint:virtualization-controller job was removed as redundant. +# +# All test jobs extend .dev so they only run on MR pipelines with the right +# MODULES_MODULE_TAG/registry context. test:virtualization-controller: stage: test diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index f504b3dabf..882893e704 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -57,6 +57,11 @@ variables: # --- Build/dev tooling (kept from previous root) --- GO_VERSION: "1.25.11" GOLANGCI_LINT_VERSION: "2.11.1" + # Helm v3 + jq are needed by `task validation:helm-templates` (tools/kubeconform). + # They are not assumed to be on the runner host; lint:helm-templates installs + # portable binaries from these versions in its before_script. + HELM_VERSION: "3.16.3" + JQ_VERSION: "1.7.1" # Let the host Go toolchain auto-download the version required by go.mod # (currently go 1.25.11). Runner hosts may ship an older Go with # GOTOOLCHAIN pinned to that older version (e.g. GOTOOLCHAIN=go1.25.9), diff --git a/images/virtualization-artifact/Taskfile.init.yaml b/images/virtualization-artifact/Taskfile.init.yaml index 73cbf043b4..c702352320 100644 --- a/images/virtualization-artifact/Taskfile.init.yaml +++ b/images/virtualization-artifact/Taskfile.init.yaml @@ -9,7 +9,10 @@ tasks: golangci-lint: cmds: - - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 + # Install golangci-lint v2. Keep in sync with GOLANGCI_LINT_VERSION in + # .gitlab/ci/variables.yml and GOLANGCI_LINT_MIN_VERSION in the + # _ensure:golangci-lint tasks. v2 lives under the /v2 module path. + - go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.11.1 ginkgo: cmds: From d2d047027b406dfab68f20967a1327b4702b7878 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 21:38:48 +0300 Subject: [PATCH 21/60] fix(ci): fix check:gens-files matrix for moq, bpf2go and api generate Three matrix legs of check:gens-files failed on the GitLab shell runner: - virtualization-artifact: go generate needs the moq binary. "go install tool" puts it in $(go env GOPATH)/bin, which the shell runner does not add to PATH (GitHub setup-go@v5 did this automatically). Export PATH with GOPATH/bin so //go:generate moq resolves. - vm-route-forge: bpf2go (go tool github.com/cilium/ebpf/cmd/bpf2go) shells out to clang and llvm-strip, which are absent on the runner. Install the same apt packages as the GH workflow dev_validation.yaml (llvm, linux-headers, clang, libbpf-dev, uuid-runtime, gcc-multilib), guarded by command presence checks and sudo -n to fail fast instead of hanging on a password prompt. - api: "task generate" is defined in api/Taskfile.dist.yaml, not the root Taskfile, so it must run from inside api/ (mirrors the GH job which does cd ./api first). Run it as cd api && go install tool && task generate. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 40 +++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index d350527008..b79741d7e0 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -280,10 +280,22 @@ check:gens-files: # are registered. interruptible: true before_script: + # git is needed by check_diffs (git diff). go/task are host tools. - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git + - | + # `go install tool` (used by all components) puts binaries in + # $(go env GOPATH)/bin, but the shell runner does not add that dir to + # PATH. GitHub's setup-go@v5 does this automatically; replicate it here + # so //go:generate directives invoking `moq`, `ginkgo`, etc. resolve. + # (PATH is re-exported in the script block too, since each top-level + # script line is a fresh shell.) + export PATH="$(go env GOPATH)/bin:${PATH}" + echo "GOPATH/bin added to PATH: $(go env GOPATH)/bin" script: - | set -e + # Restore PATH augmented in before_script for this shell too. + export PATH="$(go env GOPATH)/bin:${PATH}" function check_diffs() { local folder="$1" if ! git diff --exit-code -- "$folder"; then @@ -297,17 +309,41 @@ check:gens-files: } case "$COMPONENT" in virtualization-artifact) + # go.mod `tool` directives include moq + ginkgo; go install tool + # puts them in $GOPATH/bin (already on PATH above). Mirrors GH: + # cd images/virtualization-artifact && go install tool, then run + # generation from the root Taskfile namespace. (cd images/virtualization-artifact && go install tool) task controller:dev:gogenerate check_diffs images/virtualization-artifact ;; vm-route-forge) + # bpf2go (go tool github.com/cilium/ebpf/cmd/bpf2go) shells out to + # `clang` and `llvm-strip`. Install the same apt packages as the + # GitHub workflow (.github/workflows/dev_validation.yaml, + # check_gens_files -> Install dependencies). Prefer already-installed + # tools; only apt-get when missing, and use sudo -n so a password + # prompt fails fast instead of hanging the job. + if ! command -v clang >/dev/null 2>&1 || ! command -v llvm-strip >/dev/null 2>&1; then + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + sudo -n apt-get update + sudo -n apt-get install -y -qq \ + llvm "linux-headers-$(uname -r)" clang \ + libbpf-dev uuid-runtime gcc-multilib + else + echo "ERROR: clang/llvm-strip missing and apt-get unavailable on the runner" >&2 + exit 1 + fi + fi task vm-route-forge:gen check_diffs images/vm-route-forge ;; api) - (cd api && go install tool) - task generate + # `task generate` is defined in api/Taskfile.dist.yaml, not the + # root Taskfile — it must run from inside api/. Mirrors GH: + # cd ./api && go install tool && task generate. + (cd api && go install tool && task generate) check_diffs api ;; *) From ab9e20edef2d54fd42c01eff435259bc566a20a8 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 22:30:49 +0300 Subject: [PATCH 22/60] fix(ci): run vm-route-forge bpf2go generation in docker image The vm-route-forge leg of check:gens-files needs clang, llvm-strip and the libbpf/kernel headers to run bpf2go (go tool github.com/cilium/ebpf/cmd/bpf2go). The GitLab shell runner runs as the gitlab-runner user, which has no passwordless sudo, so the apt-get install used by the GitHub workflow fails with "sudo: a password is required". Run the generation inside the public ghcr.io/cilium/ebpf-builder image instead, bind-mounting the repo at /src. The image ships clang-22, llvm-strip-22, libbpf-dev and linux headers. Point bpf2go at the versioned binaries via BPF2GO_CC/BPF2GO_STRIP (the image has no bare clang/llvm-strip), and add -I/usr/include/x86_64-linux-gnu via BPF2GO_CFLAGS so resolves (the image symlinks /usr/include/asm to asm-generic, which lacks ptrace.h). Added docker to check-runner-tools for the check:gens-files matrix job, since the vm-route-forge leg now requires it. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 46 +++++++++++++++++-------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index b79741d7e0..f32f2bd4e1 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -281,7 +281,11 @@ check:gens-files: interruptible: true before_script: # git is needed by check_diffs (git diff). go/task are host tools. - - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git + # docker is needed by the vm-route-forge matrix leg, which runs bpf2go + # inside the cilium/ebpf-builder image (the runner has no sudo to apt-get + # clang/llvm-strip). The other legs don't use docker but it must be + # present for the matrix job to start. + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task git docker - | # `go install tool` (used by all components) puts binaries in # $(go env GOPATH)/bin, but the shell runner does not add that dir to @@ -318,25 +322,27 @@ check:gens-files: check_diffs images/virtualization-artifact ;; vm-route-forge) - # bpf2go (go tool github.com/cilium/ebpf/cmd/bpf2go) shells out to - # `clang` and `llvm-strip`. Install the same apt packages as the - # GitHub workflow (.github/workflows/dev_validation.yaml, - # check_gens_files -> Install dependencies). Prefer already-installed - # tools; only apt-get when missing, and use sudo -n so a password - # prompt fails fast instead of hanging the job. - if ! command -v clang >/dev/null 2>&1 || ! command -v llvm-strip >/dev/null 2>&1; then - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - sudo -n apt-get update - sudo -n apt-get install -y -qq \ - llvm "linux-headers-$(uname -r)" clang \ - libbpf-dev uuid-runtime gcc-multilib - else - echo "ERROR: clang/llvm-strip missing and apt-get unavailable on the runner" >&2 - exit 1 - fi - fi - task vm-route-forge:gen + # bpf2go (go tool github.com/cilium/ebpf/cmd/bpf2go) needs clang, + # llvm-strip and the libbpf/kernel headers. The GitLab shell runner + # has no sudo, so instead of apt-get (as the GH workflow did) run + # generation inside the public cilium/ebpf-builder image, which ships + # clang-22, llvm-strip-22, libbpf-dev and linux headers. go generate + # runs in the container against the repo bind-mounted at /src. + # + # BPF2GO_CC/STRIP point bpf2go at the versioned binaries (the image + # has no bare `clang`). BPF2GO_CFLAGS adds the arch-specific include + # dir so resolves (the image symlinks /usr/include/asm + # to asm-generic, which lacks ptrace.h; the real one lives under + # x86_64-linux-gnu on amd64 / aarch64-linux-gnu on arm64). + docker run --rm --platform linux/amd64 \ + -v "${PWD}:/src" \ + -w /src/images/vm-route-forge \ + -e BPF2GO_CC=clang-22 \ + -e BPF2GO_STRIP=llvm-strip-22 \ + -e BPF2GO_CFLAGS="-I/usr/include/x86_64-linux-gnu" \ + -e GOTOOLCHAIN=auto \ + ghcr.io/cilium/ebpf-builder:1777990914 \ + sh -c 'go generate ./...' check_diffs images/vm-route-forge ;; api) From 7b27d2083fb5a18d3d9d6f7d1a6503f371f203e5 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 23:00:15 +0300 Subject: [PATCH 23/60] fix(ci): use clang-14 in vm-route-forge bpf2go generation Switch the bpf2go generation inside ghcr.io/cilium/ebpf-builder from clang-22 to clang-14, matching the toolchain that produced the committed ebpf_x86_bpfel.{go,o} files (GitHub ran on ubuntu-22.04 where `apt-get install clang` pulls clang-14). The image ships clang-14/17/22, so only the BPF2GO_CC/BPF2GO_STRIP env vars change. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index f32f2bd4e1..b7ad3fe0c3 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -326,8 +326,11 @@ check:gens-files: # llvm-strip and the libbpf/kernel headers. The GitLab shell runner # has no sudo, so instead of apt-get (as the GH workflow did) run # generation inside the public cilium/ebpf-builder image, which ships - # clang-22, llvm-strip-22, libbpf-dev and linux headers. go generate - # runs in the container against the repo bind-mounted at /src. + # clang-14/17/22, llvm-strip-14/17/22, libbpf-dev and linux headers. + # go generate runs in the container against the repo bind-mounted at + # /src. Use clang-14 to match the toolchain that produced the + # committed ebpf_x86_bpfel.{go,o} files (GH ran on ubuntu-22.04 where + # apt-get install clang pulls clang-14). # # BPF2GO_CC/STRIP point bpf2go at the versioned binaries (the image # has no bare `clang`). BPF2GO_CFLAGS adds the arch-specific include @@ -337,8 +340,8 @@ check:gens-files: docker run --rm --platform linux/amd64 \ -v "${PWD}:/src" \ -w /src/images/vm-route-forge \ - -e BPF2GO_CC=clang-22 \ - -e BPF2GO_STRIP=llvm-strip-22 \ + -e BPF2GO_CC=clang-14 \ + -e BPF2GO_STRIP=llvm-strip-14 \ -e BPF2GO_CFLAGS="-I/usr/include/x86_64-linux-gnu" \ -e GOTOOLCHAIN=auto \ ghcr.io/cilium/ebpf-builder:1777990914 \ From 58356a41cd46d5e63242c20ba6070441564ea195 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 23 Jun 2026 23:00:38 +0300 Subject: [PATCH 24/60] chore(core, vm-route-forge): regenerate bpf2go output for cilium/ebpf v0.17.1 The committed ebpf_x86_bpfel.{go,o} were generated with cilium/ebpf v0.16.0, but go.mod was bumped to v0.17.1 without regenerating. bpf2go v0.17.1 emits ebpfVariableSpecs/ebpfVariables for the `unused` const variable (used in route_watcher.c to force the route_event struct into the ELF), so the regenerated output adds those types. go build ./... passes against the new files. Regenerated with clang-14 + llvm-strip-14 inside ghcr.io/cilium/ebpf-builder (BPF2GO_CFLAGS=-I/usr/include/x86_64-linux-gnu). Signed-off-by: Nikita Korolev --- .../controller/route/ebpf_x86_bpfel.go | 18 +++++++++++++++++- .../controller/route/ebpf_x86_bpfel.o | Bin 7808 -> 8648 bytes 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go index 8f5464236b..11ec2949a3 100644 --- a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go +++ b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.go @@ -54,9 +54,10 @@ func loadEbpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { type ebpfSpecs struct { ebpfProgramSpecs ebpfMapSpecs + ebpfVariableSpecs } -// ebpfSpecs contains programs before they are loaded into the kernel. +// ebpfProgramSpecs contains programs before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type ebpfProgramSpecs struct { @@ -71,12 +72,20 @@ type ebpfMapSpecs struct { RouteEventsMap *ebpf.MapSpec `ebpf:"route_events_map"` } +// ebpfVariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type ebpfVariableSpecs struct { + Unused *ebpf.VariableSpec `ebpf:"unused"` +} + // ebpfObjects contains all objects after they have been loaded into the kernel. // // It can be passed to loadEbpfObjects or ebpf.CollectionSpec.LoadAndAssign. type ebpfObjects struct { ebpfPrograms ebpfMaps + ebpfVariables } func (o *ebpfObjects) Close() error { @@ -99,6 +108,13 @@ func (m *ebpfMaps) Close() error { ) } +// ebpfVariables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to loadEbpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type ebpfVariables struct { + Unused *ebpf.Variable `ebpf:"unused"` +} + // ebpfPrograms contains all programs after they have been loaded into the kernel. // // It can be passed to loadEbpfObjects or ebpf.CollectionSpec.LoadAndAssign. diff --git a/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.o b/images/vm-route-forge/internal/controller/route/ebpf_x86_bpfel.o index 1adcd61bd29e4505fa2901c3962676fd8b767a04..a9d7a95041d1cf73680d305b91fde85327926b7d 100644 GIT binary patch literal 8648 zcmeHMU2I%O6`r-@`Y#Ph8j4Aw+|VR!s$^q3E^)mU675{8^ zuZ>NFa3v}t@jyiRiG&J)RLD|Q6agYZs$@UWNC+tpeTYOKES`)|r91@14@l!p0s-rQ)+IM}713x@vc`uh5f24h+vdL89el=*9`*6vG<+pJ=T z7Yy|epry%l+jjch3C{Iw&?M64M~oQ; zcKcaz;@5F%fAiN?>c;WBI!8_a2T?cKUTXcW!7s!i&Jq_{#LZr0KX)e7c6p-pA$e0GMaiJO;6{2q7{w`Fb!+l$* zp-nkc+qZ^<{{IwO1*{JiiB;J+#SGWge29~1m- z)d&9_)d&Aw)d&AQ)yM3;q59x|toq=8qWa+9QhmHsj>pCi{%zF<|7+C;|2x&kMExE- zDAj^TmK+EnKZ-ESdnk{h{4tSj*Z1eQzkwJa+rArmtKg{)%4sjB76XT2Y4P=vM9TS9|r$8cu>m|c_Rs3PXSQ%0Llrm3BWs4qsOrfir%LkC9pMmJR^jnw|@&g z##a_S+K^uN@D*T;!KNzL4vFviO-If%2Putm4RWM})!%2O$@rcE&v9}LthvV6C)Z#B zg|X0{=L7O@JO1S}Zs*EiXEb4W^7QGG=fg8EeWgD6thRqgk|6LWoT%4$Bc6x}HJlK{5RZ6B(F*8$@;@A|n`-$nP zF_VwyOm)8xi&K@T9GeTJ^7Ke_v`(Hmn$f?J(&*77^sndQ{CHH(qKAf~V2pq2QtkK+@#QKoHHw!E@4Fj>w~<(1wFq$rZv2QF$twNFR(x4hNjz>CtpN za_Df7o}Dfhf~OB196B4G?tk%2ICS##7rWE!?a&F^TX8fB-7i)h&4VMG#`YWjKgCV?UWy^Qr_+5bcrY?FZ9|h5numIyIgx-RDO);sXDm@gA!i zFO?r=IvFiK$u-dfGj_N*^89E%Bur*<%TVbFU-24Gz^TRYx6?E1hz8S8O z(oAJMj3%PV^g$<0>_S0k=<9IMH*_jI`SN*q+sM}fOk%7;KaWLTo2ZHKL5Z3l&y~&K zOE2`le5T)BtR|O_i_=razIB7iF&L7WIo6F$fzRqPOZJJ8$jW9aQ330Ot1Mab8KhaA zAk$SZ>aG_Zs~7c{%m~(a26KmJdAg8`b7rELkI-durV6m9k@idBH{u)$iTmjXDEh6~ z1^zAU=!zc$=C_MFx~JX){}>g{rfeJW=YgM$jS0XLe+~G=4~*#*jLK3LPi=2U`;g#w zpx<@{aXGjZcv$eeXm7o0%!1$#f%|`pGooPaA12m6+uPB;EciGM1g$rXxh2^8L*uI8 zL9{>jzAkHPrz|_~wON(I53rwAD*SttOn0e7D z*!o{V-U~)ImWBmuUW_}Kd7*ird2z+{hv2Sz{+0!6UbuO{ytwVOGcPm`G%t+2XB#gD zx6{juw3`(Qr1z3n;;N34F6-5(vXS+bzXba;OLD)MChHZ1-u za^|Z`uZn->olAeRpsDV?oDhBPA1<|tocZKaqx0UijrW$_PX3IG-3iEd*;{m?nHGat ztKTHZn3Cwc7*o=lXZsxn5m7;9&ZfJEOhF@UGb_17^kcA;H|fIlfmsJnrGx!wVkf z-HpVGch~&qUo%u26v=d+Q@D&d)dibV?Z+ZBRhgUtk=HV7D6p+T(;o-E0dp$hh z;d34y_wbyDFM0TihZjA3)5Etse8f4{03zlYNvcJ~{;zX7jZ?;{kmUH2Qs zPaQb8e_L~F`wo+dqq!ItooFJ{clK09LEXhqe%Np>NB(uP?k;6A6BDzOVXjcX?UDCkudgdSZcue5Rae-XAX{~XpG}ah zy273W_0)x$u`OL8ekroPQ2TexjBV-)yAust!9XDDI)hC_T{MPOr*zZFm;N81AHj#D zRE@V3KNg_ZWWPyJ^3715HJe!;A#f<`@GdLqRu%CZx0U_4tamlgbAaMKVECJTRs~Ye z$q$CZf_I|kIMtq*G>x)Z|GIDKImIgDq4tU|fZ447JDc>+(~sI~|NQNq{rm5rLyk)R zd{NgW<^g^B_JP=Ie`@37XV9=&|7xFBL#ciHUy8l=n#_Ro|9#KikDvcf@MSdlKJg1(YhSJ&~~9@atmzDb)PAr{yAlFaq6{MvjqpE$yG{j`UB{F4XqS@YNhjItPyE3Z;sldR*&V*A+PFX=BY zKY0aSj4{R;X@0<}C|guazNx?Nyq8St&IhUQcUIlbWJOkm(L^{=iLOW^?fM=yR|h*uv#`E z%v!DrTP@cj%v!DsTPfSZR?2T8%+l?K94M>Zo+Mc9uv-JISUW6!z^vU#VJjULYvNYA zy0DdQP1s6@O+Y^MX9sI{RT8Xr?dSlQ%}>HsyWI%0bbAqI=@KG^m2O^jfV%0{w zrdAC}SfZ`W3_Z;FZ}kz?Dyfd4W}6Hv^&4vT&k5Ua!_VTsB!2#~u-&3N**uNseA>wj z(}UdGx=F82%m)d?dM3kM4aLkyo{cN!O^5pq^NC|)cM^pU0-No>XK0`6`~3s+0^Vmf z+ky8f%DgR>`30r$1IV9hHHuP;Fvk~#`K0p#&N;j~;1|P$%L7NqO@~{;T&3x|{InkF z{F47c2foaV10yI^hgo4Z8$au?U73xqI`MUfHx6>3S_d7f?QqxOp2NEi-*vd}aDw-A zuprzW|9}HK;Bd|1vko^LUUhig;f*N2f|<0U4shGyuEUw!L^ghCXn2J5(Oaa)GW6i^ z({#R=qmPQOQLpHG&&wrC(Y@l0SR+f@Q*Yq9+?$$-b+h!;^qj8Jh3V7!ByE}J9XuOU zE}i!N9OAuEYKh)2&BXj1ZI|Yfb@P{uYZj+JN~e;oT$r;NrxWE Date: Wed, 24 Jun 2026 12:13:18 +0300 Subject: [PATCH 25/60] ci(gitlab): remove TODO_RUNNER_TAG placeholders, confirm deckhouse tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deckhouse runner tag is confirmed as registered for the project. Remove all TODO_RUNNER_TAG placeholder comments across the .gitlab/ migration artifacts and rewrite README §6 / §10 to describe the tag as confirmed instead of a placeholder. Signed-off-by: Nikita Korolev --- .gitlab/README.md | 18 ++---------------- .gitlab/ci/defaults.yml | 9 --------- .gitlab/ci/jobs/auto-assign-author.yml | 2 -- .gitlab/ci/jobs/backport.yml | 2 -- .gitlab/ci/jobs/changelog.yml | 4 ---- .gitlab/ci/jobs/check-changelog.yml | 2 -- .gitlab/ci/jobs/check-milestone.yml | 2 -- .gitlab/ci/jobs/cve-scan.yml | 3 --- .gitlab/ci/jobs/gitleaks.yml | 4 ---- .gitlab/ci/jobs/lint-validate.yml | 13 ------------- .gitlab/ci/jobs/manual-tools.yml | 2 -- .gitlab/ci/jobs/precache.yml | 4 ---- .gitlab/ci/jobs/svace.yml | 5 ----- .gitlab/ci/jobs/translate-changelog.yml | 2 -- .gitlab/ci/scripts/bash/setup-mr-settings.sh | 3 --- .gitlab/ci/templates/dev.yml | 3 --- 16 files changed, 2 insertions(+), 76 deletions(-) diff --git a/.gitlab/README.md b/.gitlab/README.md index f703661016..d84315b57f 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -186,21 +186,10 @@ For local debugging (e.g. `setup-mr-settings.sh`) you can export ## 6. Runner tags -All jobs in this directory specify `tags: [deckhouse]`. This is a placeholder -until concrete runner tags are registered at +All jobs in this directory specify `tags: [deckhouse]`. This is the agreed +runner tag for the project, registered at . -Once registration is complete, update the value: - -```bash -# Find every "tags:" line in this directory that says "deckhouse". -grep -rn 'tags:' .gitlab/ci/jobs/ | grep deckhouse -# Update each to the real runner tag, e.g. "deckhouse-large" or "dvp". -``` - -Look for `TODO_RUNNER_TAG` comments in each job yml; replace the tag and -remove the comment when finalised. - ### Shell executor requirements The project runner is expected to use the GitLab Runner `shell` executor. @@ -278,9 +267,6 @@ maintainer hasn't pre-approved them. These are intentional gaps from the first-iteration migration. Track them under the `virtualization-m9e.3` Beads issue. -- **`TODO_RUNNER_TAG`** — every job uses `tags: [deckhouse]` as a - placeholder. Replace with the real registered runner tag once - available. Search for `TODO_RUNNER_TAG` in this directory. - **Webhook listener for slash-commands** — GitLab does not natively start pipelines on MR comment creation or label change. See [§12](#12-slash-commands-and-webhook-listener). diff --git a/.gitlab/ci/defaults.yml b/.gitlab/ci/defaults.yml index 8529475f2c..308068516e 100644 --- a/.gitlab/ci/defaults.yml +++ b/.gitlab/ci/defaults.yml @@ -5,12 +5,6 @@ # resources are not pinned to a specific tag set (we leave GitLab's # scheduling to runner tags only). # -# TODO_RUNNER_TAG: the `deckhouse` tag is the agreed default across the -# migration plan (decision 7 in tmp/ai-summary/gitlab-ci-migration-plan.md -# §0). The real runner set on fox.flant.com/deckhouse/virtualization may use -# `large` / `regular` / similar — once runners are registered, narrow this -# to the actual tag list. Until then, `deckhouse` is the safe placeholder. -# # Per-job `interruptible` is intentionally NOT set here; long-running jobs # (build_prod and the deploy chain) need `interruptible: false`, while # short-lived lint/test/build_dev jobs benefit from `interruptible: true`. @@ -19,6 +13,3 @@ default: tags: - deckhouse - # TODO_RUNNER_TAG: confirm registered runner tags on - # fox.flant.com/deckhouse/virtualization and replace [deckhouse] above - # with the real tag list (e.g. [deckhouse, large] for heavy builds). diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml index 206ee24a60..defe0749ee 100644 --- a/.gitlab/ci/jobs/auto-assign-author.yml +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -20,8 +20,6 @@ auto-assign-author: stage: info - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index 1b070c5157..2a4117c97f 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -28,8 +28,6 @@ backport: stage: lint - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index c8a2c8a163..06b7e81b08 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -35,8 +35,6 @@ changelog:milestone: stage: lint - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: @@ -66,8 +64,6 @@ changelog:milestone: # MRs are opened. changelog:all-active-milestones: stage: lint - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml index d9dc4a26a6..04091bbb36 100644 --- a/.gitlab/ci/jobs/check-changelog.yml +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -22,8 +22,6 @@ check:changelog: stage: lint - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/check-milestone.yml b/.gitlab/ci/jobs/check-milestone.yml index 2a051229e9..a20fa97f71 100644 --- a/.gitlab/ci/jobs/check-milestone.yml +++ b/.gitlab/ci/jobs/check-milestone.yml @@ -19,8 +19,6 @@ check:milestone: stage: lint - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml index f0f0e492d0..b186fe85a6 100644 --- a/.gitlab/ci/jobs/cve-scan.yml +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -32,8 +32,6 @@ cve:scan:daily: extends: - .cve_scan stage: scan - # TODO_RUNNER_TAG: Trivy scan over the full module image set is heavy; - # narrow to [deckhouse, large] once runners are registered. interruptible: false variables: SOURCE_TAG: "main" @@ -56,7 +54,6 @@ cve:scan:manual: extends: - .cve_scan stage: scan - # TODO_RUNNER_TAG: same as cve:scan:daily. interruptible: false variables: SOURCE_TAG: "${SCAN_TAG:-main}" diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index 116dc6c8f3..5af74bed7d 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -39,7 +39,6 @@ gitleaks_full_scheduled: gitleaks:diff: extends: .gitleaks_scan stage: scan - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true variables: SCAN_MODE: "diff" @@ -50,8 +49,6 @@ gitleaks:diff: gitleaks:full:scheduled: extends: .gitleaks_scan stage: scan - # TODO_RUNNER_TAG: full-history scan is heavier than diff; narrow to - # [deckhouse, large] if available. interruptible: false variables: SCAN_MODE: "full" @@ -62,7 +59,6 @@ gitleaks:full:scheduled: gitleaks:full:manual: extends: .gitleaks_scan stage: scan - # TODO_RUNNER_TAG: same as gitleaks:full:scheduled. interruptible: false variables: SCAN_MODE: "full" diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index b7ad3fe0c3..7a99ead161 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -12,9 +12,6 @@ # # Each job: # - inherits `tags: [deckhouse]` from .gitlab/ci/defaults.yml; -# - emits a `# TODO_RUNNER_TAG:` comment where the real runner tag (e.g. -# large / regular) might need to replace `deckhouse` once runners are -# registered on fox.flant.com/deckhouse/virtualization; # - has `interruptible: true` where short-lived (safe to cancel on a new # push) and `interruptible: false` where a long run is expected; # - honors `validation/skip/` labels via `when: never` rules. @@ -37,7 +34,6 @@ lint:no-cyrillic: stage: lint - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true variables: GIT_DEPTH: "0" @@ -60,7 +56,6 @@ lint:no-cyrillic: lint:doc-changes: stage: lint - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true variables: GIT_DEPTH: "0" @@ -86,7 +81,6 @@ lint:doc-changes: lint:shellcheck: stage: lint - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true # The `task lint:shellcheck` target runs shellcheck inside the # koalaman/shellcheck-alpine Docker image, so the runner needs docker — @@ -108,7 +102,6 @@ lint:shellcheck: lint:yaml: stage: lint - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task @@ -144,8 +137,6 @@ lint:yaml: lint:go: stage: lint - # TODO_RUNNER_TAG: lints every Go module; may need [deckhouse, large] once - # runners are registered. interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go curl @@ -207,7 +198,6 @@ lint:go: lint:helm-templates: stage: lint - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: # go + task are host tools; docker runs kubeconform; git/curl/python3 are @@ -276,8 +266,6 @@ lint:helm-templates: check:gens-files: stage: lint - # TODO_RUNNER_TAG: heavy jobs may need [deckhouse, large] once runners - # are registered. interruptible: true before_script: # git is needed by check_diffs (git diff). go/task are host tools. @@ -390,7 +378,6 @@ check:gens-files: lint:gitlab-ci: stage: lint - # TODO_RUNNER_TAG: confirm real runner tag on fox.flant.com runner pool. interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index 54876fbcb5..c43ecc1247 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -30,8 +30,6 @@ mrs:summary: stage: notify - # TODO_RUNNER_TAG: confirm registered runner tag at - # https://fox.flant.com/deckhouse/virtualization/-/runners after registration. tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/precache.yml b/.gitlab/ci/jobs/precache.yml index a38a63a841..5d3ab958ce 100644 --- a/.gitlab/ci/jobs/precache.yml +++ b/.gitlab/ci/jobs/precache.yml @@ -20,9 +20,6 @@ precache:build:main: - .local_build - .main stage: build - # TODO_RUNNER_TAG: scheduled builds should run on a runner pool separate - # from interactive MR builds. Once runners are registered, narrow this - # to the dedicated precache tag (e.g. [deckhouse, precache]). interruptible: false variables: MODULES_MODULE_TAG: "${CI_COMMIT_REF_NAME:-main}" @@ -40,7 +37,6 @@ precache:build:branch: - .local_build - .dev_vars stage: build - # TODO_RUNNER_TAG: same as precache:build:main. interruptible: false variables: MODULES_MODULE_TAG: "${REF_BRANCH}" diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index b3bb6b0f29..e55a00511e 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -40,7 +40,6 @@ svace:set-vars: extends: - .info stage: info - # TODO_RUNNER_TAG: short-lived helper job, [deckhouse] is fine. interruptible: true # Only emit the svace tag dotenv in contexts where svace actually runs. # Without these rules this job ran on EVERY MR pipeline and leaked a @@ -90,8 +89,6 @@ svace:build: - .local_build - .dev_vars stage: build - # TODO_RUNNER_TAG: build with svace instrumentation requires more CPU/RAM; - # narrow to [deckhouse, large] once runners are registered. interruptible: false needs: - svace:set-vars @@ -116,7 +113,6 @@ svace:analyze: extends: - .svace_analyze stage: scan - # TODO_RUNNER_TAG: SSH-heavy; narrow to [deckhouse, large] if available. interruptible: false needs: - job: svace:set-vars @@ -137,7 +133,6 @@ svace:analyze: svace:notify: stage: cleanup - # TODO_RUNNER_TAG: short-lived helper, [deckhouse] is fine. interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index 3993ed7d47..ba3a3ccac4 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -37,8 +37,6 @@ include: # Local job definition that extends the upstream hidden job. # Tag override is intentional: we want our project's runner tag, not the # upstream default. -# TODO_RUNNER_TAG: confirm registered runner tag at -# https://fox.flant.com/deckhouse/virtualization/-/runners after registration. translate:changelog: extends: .translate_and_create_mr stage: notify diff --git a/.gitlab/ci/scripts/bash/setup-mr-settings.sh b/.gitlab/ci/scripts/bash/setup-mr-settings.sh index 34d754cd29..06d6072448 100644 --- a/.gitlab/ci/scripts/bash/setup-mr-settings.sh +++ b/.gitlab/ci/scripts/bash/setup-mr-settings.sh @@ -29,9 +29,6 @@ # --project-id override CI_PROJECT_ID # --api-base default $CI_API_V4_URL or https://fox.flant.com/api/v4 # --dry-run print curl commands instead of executing them -# -# TODO_RUNNER_TAG: this script is intended to be run by a human from a -# workstation, not from CI. No runner tag applies. set -euo pipefail diff --git a/.gitlab/ci/templates/dev.yml b/.gitlab/ci/templates/dev.yml index adfafa672c..4ab89a9694 100644 --- a/.gitlab/ci/templates/dev.yml +++ b/.gitlab/ci/templates/dev.yml @@ -4,9 +4,6 @@ # that uses `extends: .dev` runs only on MR pipelines, derives its # MODULES_MODULE_TAG from the MR iid, and inherits the DEV registry # credentials via .dev_vars. -# -# TODO_RUNNER_TAG: runner tags come from .gitlab/ci/defaults.yml; this file -# only sets logic-level defaults. .dev: variables: From b19dc3107512208d2668ed9c710ce1f288da910b Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 13:24:14 +0300 Subject: [PATCH 26/60] fix(ci): wire edition/delve into dev builds, restore werf cleanup config, fix changelog yml schema - build_dev/build_dev_tags/build_main: add set_vars job in info stage that derives MODULE_EDITION (edition/ce label), DEBUG_COMPONENT (first delve/* label), RELEASE_IN_DEV from MR labels; consume its dotenv via needs and set WERF_VIRTUAL_MERGE=0, matching the GitHub dev_setup_build behavior. set-vars.sh is now wired in (dead code removed); MODULES_MODULE_TAG stays per-template. - cleanup: pass --config werf_cleanup.yaml and compose --repo from MODULES_MODULE_SOURCE/MODULES_MODULE_NAME (via .dev_vars) instead of hardcoding the path, restoring the GitHub retention behavior. - changelog_collect.py: render_yaml now emits the deckhouse CHANGELOG schema (sections -> features/fixes -> summary + pull_request), validated against existing CHANGELOG-*.yml; strip :low section suffix. - check_changelog_entry.py & changelog_collect.py: narrow ALLOWED_TYPES to {feature, fix} (deckhouse/changelog-action only renders features/fixes), with a clear error for unsupported types. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/build-dev.yml | 36 +++++--- .gitlab/ci/jobs/cleanup.yml | 43 ++++++---- .gitlab/ci/jobs/info.yml | 40 +++++++++ .gitlab/ci/scripts/bash/set-vars.sh | 57 ++++-------- .../ci/scripts/python/changelog_collect.py | 86 ++++++++++++++++--- .../scripts/python/check_changelog_entry.py | 10 ++- 6 files changed, 189 insertions(+), 83 deletions(-) diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml index 49fc4a2a5e..b2510e54a7 100644 --- a/.gitlab/ci/jobs/build-dev.yml +++ b/.gitlab/ci/jobs/build-dev.yml @@ -23,24 +23,36 @@ # older main pipeline. build_dev and build_dev_tags inherit the project # default (interruptible not set, so they can be cancelled manually). -# `needs: []` is intentional: it opts these jobs into DAG mode with no -# artifact downloads. Without it, GitLab's legacy behavior downloads ALL -# artifacts from every prior-stage job, which leaks unrelated dotenv -# artifacts (e.g. svace:set-vars writes MODULES_MODULE_TAG=-svace) -# and overrides the per-template MODULES_MODULE_TAG from .dev / .dev_tags / -# .main. These builds derive their tag and registry vars from their own -# templates and need no upstream artifacts. +# `needs: [set_vars]` (with artifacts) is intentional: it opts these jobs +# into DAG mode and pulls ONLY the set_vars dotenv (MODULE_EDITION, +# DEBUG_COMPONENT, RELEASE_IN_DEV). It replaces the previous bare `needs: []`, +# which existed to stop GitLab's legacy behavior of downloading ALL prior-stage +# artifacts and leaking unrelated dotenv artifacts (e.g. svace:set-vars writes +# MODULES_MODULE_TAG=-svace) that override the per-template +# MODULES_MODULE_TAG from .dev / .dev_tags / .main. Listing set_vars explicitly +# keeps that protection: svace.env is still not downloaded. set_vars does NOT +# emit MODULES_MODULE_TAG, so the per-template tag is preserved. +# +# WERF_VIRTUAL_MERGE=0 mirrors the GitHub dev_setup_build job env. build_dev: stage: build - needs: [] + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" extends: - .local_build - .dev build_dev_tags: stage: build - needs: [] + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" extends: - .local_build - .dev_tags @@ -48,7 +60,11 @@ build_dev_tags: build_main: stage: build interruptible: true - needs: [] + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" extends: - .local_build - .main diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml index 9702efaf65..c9d1870ae7 100644 --- a/.gitlab/ci/jobs/cleanup.yml +++ b/.gitlab/ci/jobs/cleanup.yml @@ -1,30 +1,39 @@ # Cleanup job. # -# Carries forward the cleanup job from the previous root .gitlab-ci.yml. -# Runs only on scheduled pipelines and prunes old module images from the -# DEV registry using werf cleanup --without-kube=true. +# Carries forward the cleanup job from the previous root .gitlab-ci.yml and +# the GitHub .github/workflows/dev_registry-cleanup.yml workflow. Runs only +# on scheduled pipelines and prunes old module images from the DEV registry +# using werf cleanup --without-kube=true --config werf_cleanup.yaml. # -# Behavior preserved: -# - registry host: dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization -# (kept as a literal because we are cleaning exactly this repo's -# namespace, regardless of DEV_REGISTRY variable contents). -# - tag: v0.0.0-main (matches the build_main tag). -# - resource: not pinned (single schedule at a time is enough). -# - MODULES_REGISTRY is set so any inherited .dev_vars does not override -# the hardcoded path — we deliberately do NOT extend .dev_vars here. +# The weekly cadence (GitHub cron "12 0 * * 6") is configured in the +# GitLab UI under CI/CD -> Schedules, not here. This job only gates on +# $CI_PIPELINE_SOURCE == "schedule" so it runs under that schedule. # -# TODO: parameterize the registry path via DEV_MODULE_SOURCE / MODULE_NAME -# once the previous GH cleanup workflow (dev_registry-cleanup.yml) is fully -# analyzed — see migration plan §2 row for dev_registry-cleanup.yml. +# Registry path is composed from the DEV Project Variables the rest of the +# pipeline uses: +# MODULES_MODULE_SOURCE <- DEV_MODULE_SOURCE (e.g. dev-registry.deckhouse.io/sys/deckhouse-oss/modules), via .dev_vars +# MODULES_MODULE_NAME <- MODULE_NAME (virtualization), set in .gitlab/ci/variables.yml +# This matches the GitHub workflow's +# --repo ${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} +# exactly. Extending .dev_vars also provides MODULES_REGISTRY + +# MODULES_REGISTRY_LOGIN/_PASSWORD so the upstream Setup.gitlab-ci.yml +# before_script `werf cr login` authenticates against the DEV registry +# before cleanup runs. +# +# Retention policies live in werf_cleanup.yaml at the repo root +# (tag /.*/ -> 72h, branch /.*/ -> 168h, branch /main|release-[0-9]+.*/ -> +# last 5 with imagesPerReference last 1). --config is mandatory: without +# it werf cleanup falls back to default git-history policies and +# over/under-prunes dev images. cleanup: stage: cleanup + extends: + - .dev_vars variables: MODULES_MODULE_TAG: v0.0.0-main - MODULES_REGISTRY: dev-registry.deckhouse.io - MODULES_MODULE_SOURCE: dev-registry.deckhouse.io/sys/deckhouse-oss/modules rules: - if: $CI_PIPELINE_SOURCE == "schedule" script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf - - werf cleanup --repo dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization --without-kube=true + - werf cleanup --repo ${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} --without-kube=true --config werf_cleanup.yaml diff --git a/.gitlab/ci/jobs/info.yml b/.gitlab/ci/jobs/info.yml index b07e1fa07a..80a00aad7f 100644 --- a/.gitlab/ci/jobs/info.yml +++ b/.gitlab/ci/jobs/info.yml @@ -19,3 +19,43 @@ show_main_manifest: extends: - .info - .main + +# set_vars: derive label-driven build variables for the dev build jobs. +# +# Mirrors the GitHub Actions `set_vars` job from dev_module_build.yml: +# - MODULE_EDITION CE when the MR carries the `edition/ce` label, else EE. +# - DEBUG_COMPONENT the first `delve/*` label (empty if none). +# - RELEASE_IN_DEV true on release-X.Y branches, false otherwise. +# +# The dotenv artifact is consumed by build_dev / build_dev_tags / build_main +# via `needs: [{job: set_vars, artifacts: true}]`. werf reads MODULE_EDITION +# (.werf/consts.yaml: `env "MODULE_EDITION" "EE"`) and DEBUG_COMPONENT +# (werf.yaml: `env "DEBUG_COMPONENT" ""`) from the process environment, so +# the dotenv vars flow straight into the werf build. +# +# MODULES_MODULE_TAG is intentionally NOT emitted here (see set-vars.sh); +# each build job still derives its tag from .dev / .dev_tags / .main. +# +# Rules mirror the union of the dev build jobs' rules (.dev / .dev_tags / +# .main) so set_vars always runs when any of them runs and the `needs:` edge +# is satisfied. Specifying an explicit `needs:` on the build jobs also keeps +# them in DAG mode, so the svace:set-vars dotenv still does not leak in. + +set_vars: + stage: info + interruptible: true + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash + script: + - bash .gitlab/ci/scripts/bash/set-vars.sh + artifacts: + reports: + dotenv: set_vars.env + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: always + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-dev.*$/' + when: always + - when: never diff --git a/.gitlab/ci/scripts/bash/set-vars.sh b/.gitlab/ci/scripts/bash/set-vars.sh index a0575e491d..aafc3b23da 100755 --- a/.gitlab/ci/scripts/bash/set-vars.sh +++ b/.gitlab/ci/scripts/bash/set-vars.sh @@ -8,28 +8,27 @@ # `artifacts.reports.dotenv`. # # Outputs (written to set_vars.env in $CI_PROJECT_DIR): -# MODULES_MODULE_TAG mrNNN for MR pipelines, main for default branch, -# release-X.Y for release branches, mrNNN for manual -# PR_NUMBER override, fail otherwise. # MODULE_EDITION CE if MR carries label edition/ce, otherwise EE. # RELEASE_IN_DEV true if $CI_COMMIT_BRANCH matches release-X.Y, # false otherwise. # DEBUG_COMPONENT first delve/* label (empty if none). # +# MODULES_MODULE_TAG is intentionally NOT emitted here. The dev build jobs +# (build_dev / build_dev_tags / build_main) derive their own tag from the +# .dev / .dev_tags / .main templates, and a dotenv variable would override +# those per-template tags (see the `needs:` note in jobs/build-dev.yml). +# # Required env (provided by the job context): # CI_API_V4_URL, CI_PROJECT_ID, CI_PIPELINE_SOURCE, CI_COMMIT_BRANCH, # CI_MERGE_REQUEST_IID, CI_MERGE_REQUEST_LABELS, GITLAB_API_TOKEN. # -# This script is not yet wired into a job by this issue — the -# info.yml / set-vars integration lands in a follow-up because the previous -# GitLab config did not have an equivalent job. Child issue can call it via: -# set_vars: -# stage: info -# script: -# - bash .gitlab/ci/scripts/bash/set-vars.sh -# artifacts: -# reports: -# dotenv: set_vars.env +# GITLAB_API_TOKEN is only required for the manual PR_NUMBER fallback below; +# normal MR pipelines use $CI_MERGE_REQUEST_LABELS and need no token. +# +# Wired into the `set_vars` job in .gitlab/ci/jobs/info.yml, which runs in +# the info stage on MR, default-branch, and dev-tag pipelines and is +# consumed by the dev build jobs via +# `needs: [{job: set_vars, artifacts: true}]`. # shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. @@ -42,35 +41,14 @@ source "${SCRIPT_DIR}/lib/api.sh" OUTPUT="${CI_PROJECT_DIR:-.}/set_vars.env" -# 1) MODULES_MODULE_TAG ------------------------------------------------------ -# Mirrors the GH set_vars job: prefer MR iid for MR pipelines, then main for -# pushes to the default branch, then release-X.Y for release branches, then -# PR_NUMBER for manual triggers, fail otherwise. -if [[ "${CI_PIPELINE_SOURCE:-}" == "merge_request_event" ]]; then - if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then - echo "ERROR: merge_request_event pipeline without CI_MERGE_REQUEST_IID" >&2 - exit 1 - fi - MODULES_MODULE_TAG="mr${CI_MERGE_REQUEST_IID}" -elif [[ "${CI_COMMIT_BRANCH:-}" == "${CI_DEFAULT_BRANCH:-main}" ]]; then - MODULES_MODULE_TAG="main" -elif [[ "${CI_COMMIT_BRANCH:-}" =~ ^release-([0-9]+\.[0-9]+) ]]; then - MODULES_MODULE_TAG="${CI_COMMIT_BRANCH}" -elif [[ -n "${PR_NUMBER:-}" ]]; then - MODULES_MODULE_TAG="mr${PR_NUMBER}" -else - echo "ERROR: cannot derive MODULES_MODULE_TAG (source=${CI_PIPELINE_SOURCE:-?}, branch=${CI_COMMIT_BRANCH:-empty})" >&2 - exit 1 -fi - -# 2) RELEASE_IN_DEV ---------------------------------------------------------- +# 1) RELEASE_IN_DEV ---------------------------------------------------------- if [[ "${CI_COMMIT_BRANCH:-}" =~ ^release-[0-9]+\.[0-9]+ ]]; then RELEASE_IN_DEV="true" else RELEASE_IN_DEV="false" fi -# 3) Labels via GitLab API --------------------------------------------------- +# 2) Labels via GitLab API --------------------------------------------------- # GitLab exposes $CI_MERGE_REQUEST_LABELS automatically for MR pipelines, but # we keep the explicit API fetch as a safety net for manual/web pipelines that # target a specific MR via PR_NUMBER. @@ -82,14 +60,14 @@ elif [[ -n "${PR_NUMBER:-}" && -n "${GITLAB_API_TOKEN:-}" ]]; then | jq -r '.labels | join(",")')" fi -# 4) MODULE_EDITION ---------------------------------------------------------- +# 3) MODULE_EDITION ---------------------------------------------------------- if [[ ",${LABELS}," == *,edition/ce,* ]]; then MODULE_EDITION="CE" else MODULE_EDITION="EE" fi -# 5) DEBUG_COMPONENT --------------------------------------------------------- +# 4) DEBUG_COMPONENT --------------------------------------------------------- DEBUG_COMPONENT="" DELVE_COUNT=0 if [[ -n "${LABELS}" ]]; then @@ -101,9 +79,8 @@ if [[ "${DELVE_COUNT}" -gt 1 ]]; then exit 1 fi -# 6) Persist ----------------------------------------------------------------- +# 5) Persist ----------------------------------------------------------------- cat > "${OUTPUT}" < features) and 'fix' +# (-> fixes) sections in CHANGELOG-*.yml. Keep in sync with +# check_changelog_entry.py. +ALLOWED_TYPES = {"feature", "fix"} +TYPE_TO_SECTION = { + "feature": "features", + "fix": "fixes", +} def log(message: str) -> None: @@ -189,20 +196,73 @@ def group_entries(entries: list[dict]) -> dict[str, list[dict]]: return grouped +def yaml_summary_scalar(value: str) -> str: + """Emit a YAML scalar for a changelog summary line. + + Plain style when safe (matches deckhouse/changelog-action output for the + common case); double-quoted otherwise to avoid YAML injection. + """ + if value == "": + return '""' + if ( + re.search(r"[:#]", value) + or value[0] in "-?,[]{}'\"&*!|>%@`" + or value.endswith(" ") + or ": " in value + or " #" in value + ): + return json.dumps(value, ensure_ascii=False) + return value + + def render_yaml(entries: list[dict], milestone_title: str) -> str: - grouped = group_entries(entries) - lines = [f"# Changelog for {milestone_title}", ""] - for section in sorted(grouped.keys()): - section_entries = grouped[section] - lines.append(f"## {section}") - lines.append("") - for entry in section_entries: - lines.append( - f"- **{entry['type']}** ({entry['impact_level']}): {entry['summary']} " - f"(MR !{entry['mr_iid']})" + """Render CHANGELOG-.yml in the deckhouse schema. + + Schema (matches deckhouse/changelog-action@v2.6.0 release_yaml):: + +
: + features: + - summary: + pull_request: + fixes: + - summary: + pull_request: + + Sections are sorted alphabetically and emitted compactly (no blank lines + between sections). The ':low' impact_level suffix is stripped from the + section key: it only pins impact_level during validation and is not + represented in the YAML. Within each section, entries are ordered by MR iid + descending, matching the historical generator output. An empty milestone + yields '{}' (same as the historical generator). + """ + grouped: dict[str, dict[str, list[dict]]] = {} + for entry in entries: + section_key = entry["section"].split(":", 1)[0] + bucket = TYPE_TO_SECTION.get(entry["type"]) + if bucket is None: + log( + f"WARN: MR !{entry['mr_iid']} has unsupported type " + f"'{entry['type']}' (allowed: {sorted(ALLOWED_TYPES)}), skipping." ) - lines.append("") - return "\n".join(lines).rstrip() + "\n" + continue + grouped.setdefault(section_key, {"features": [], "fixes": []})[bucket].append(entry) + + if not grouped: + return "{}\n\n" + + lines: list[str] = [] + for section in sorted(grouped.keys()): + buckets = grouped[section] + lines.append(f"{section}:") + for bucket in ("features", "fixes"): + items = sorted(buckets[bucket], key=lambda e: e["mr_iid"], reverse=True) + if not items: + continue + lines.append(f" {bucket}:") + for entry in items: + lines.append(f" - summary: {yaml_summary_scalar(entry['summary'])}") + lines.append(f" pull_request: {entry['mr_url']}") + return "\n".join(lines) + "\n\n" def render_markdown(entries: list[dict], milestone_title: str, minor_version: str) -> str: diff --git a/.gitlab/ci/scripts/python/check_changelog_entry.py b/.gitlab/ci/scripts/python/check_changelog_entry.py index 873baecc8f..68a848dd35 100644 --- a/.gitlab/ci/scripts/python/check_changelog_entry.py +++ b/.gitlab/ci/scripts/python/check_changelog_entry.py @@ -52,7 +52,9 @@ re.DOTALL, ) KEY_VALUE_RE = re.compile(r"^([A-Za-z_]+)\s*:\s*(.*)$") -ALLOWED_TYPES = {"feature", "fix", "breaking", "chore", "docs", "refactor", "test"} +# deckhouse/changelog-action@v2.6.0 only renders 'feature' (-> features) and +# 'fix' (-> fixes) in CHANGELOG-*.yml. Keep in sync with changelog_collect.py. +ALLOWED_TYPES = {"feature", "fix"} def log(message: str) -> None: @@ -133,8 +135,10 @@ def validate_block( errors.append(f"block #{block_index}: missing required key 'type'") elif change_type not in ALLOWED_TYPES: errors.append( - f"block #{block_index}: type '{change_type}' is not one of " - f"{sorted(ALLOWED_TYPES)}" + f"block #{block_index}: type '{change_type}' is not supported; " + f"allowed types are {sorted(ALLOWED_TYPES)} " + f"(deckhouse changelog only supports 'feature' and 'fix', " + f"rendered as the 'features'/'fixes' sections)" ) summary = fields.get("summary", "") From 0dd21277c887ff95a8050bb8ee0c3c4b8ca057a3 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 14:59:13 +0300 Subject: [PATCH 27/60] fix(ci): port remaining GH jobs, fix translate-changelog direction, add CVE per-MR scan, cleanup - translate:changelog: override script with EN->RU implementation (upstream template is hardcoded RU->EN), add rules for push to main/release-X.Y, fix header comment. - cve-scan: add cve:scan:mr (MR-gated, needs build_dev); document VAULT_ROLE as required Project CI/CD var in variables.yml; drop redundant CUSTOM_PATH vars. - lint:go: remove dead prune entry for ./test/performance/shatal (path never existed; real module test/performance/tools/shatal is intentionally linted). - changelog_collect.py: move MR-body temp file out of CHANGELOG/ to project root so git add CHANGELOG/ cannot stage it. - dev_vars/prod_vars: remove unused FALLBACK_* vars. - Port dropped GH jobs: lint:dmt (allow_failure), test:scripts:js (fix package.json + add mrs_notifier.test.mjs smoke test), test:scripts:python (py_compile smoke), test:build:d8v-cli (build + --help). All wired into includes.yml. - README: document port/drop decisions, CVE scan jobs, VAULT_ROLE/RELEASE_TOKEN, merge translate-changelog and CVE snippets. Signed-off-by: Nikita Korolev --- .gitlab/README.md | 38 +++- .gitlab/ci/includes.yml | 4 + .gitlab/ci/jobs/cve-scan.yml | 58 +++++- .gitlab/ci/jobs/lint-dmt.yml | 43 +++++ .gitlab/ci/jobs/lint-validate.yml | 13 +- .gitlab/ci/jobs/test-d8v-cli.yml | 36 ++++ .gitlab/ci/jobs/test-scripts-js.yml | 41 ++++ .gitlab/ci/jobs/test-scripts-python.yml | 36 ++++ .gitlab/ci/jobs/translate-changelog.yml | 181 +++++++++++++++++- .../ci/scripts/python/changelog_collect.py | 6 +- .gitlab/ci/templates/dev_vars.yml | 5 - .gitlab/ci/templates/prod_vars.yml | 3 - .gitlab/ci/variables.yml | 15 ++ .gitlab/scripts/js/mrs_notifier.test.mjs | 65 +++++++ 14 files changed, 510 insertions(+), 34 deletions(-) create mode 100644 .gitlab/ci/jobs/lint-dmt.yml create mode 100644 .gitlab/ci/jobs/test-d8v-cli.yml create mode 100644 .gitlab/ci/jobs/test-scripts-js.yml create mode 100644 .gitlab/ci/jobs/test-scripts-python.yml create mode 100644 .gitlab/scripts/js/mrs_notifier.test.mjs diff --git a/.gitlab/README.md b/.gitlab/README.md index d84315b57f..ca3d7f4bd0 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -59,6 +59,10 @@ For a release engineer: │ ├── check-changelog.yml # validate ```changes blocks │ ├── check-milestone.yml # MR has a milestone │ ├── manual-tools.yml # mrs:summary (Loop notification) + │ ├── lint-dmt.yml # DMT linter (allow_failure: true) + │ ├── test-scripts-js.yml # JS smoke tests (.gitlab/scripts/js) + │ ├── test-scripts-python.yml # python py_compile smoke check + │ ├── test-d8v-cli.yml # d8v CLI build + --help check │ └── translate-changelog.yml # ru -> en changelog + MR └── scripts/ ├── bash/ @@ -78,7 +82,8 @@ For a release engineer: └── check_changelog_entry.py .gitlab/scripts/js/ ├── package.json -└── mrs_notifier.mjs # GitLab counterpart of prs_notifier.mjs +├── mrs_notifier.mjs # GitLab counterpart of prs_notifier.mjs +└── mrs_notifier.test.mjs # node:test smoke test ``` Every job `extends` (or `include`s) a script in `.gitlab/ci/scripts/bash/`. @@ -107,6 +112,7 @@ files. The full list (including build/deploy) is in | `LOOP_TOKEN` | Loop API (optional) | Only needed if Loop API is used in addition to the webhook. | | `DMT_METRICS_TOKEN` | DMT linter | Auth token for DMT metrics endpoint. | | `DMT_METRICS_URL` | DMT linter | Endpoint URL for DMT metrics. | +| `VAULT_ROLE` | cve scan | Vault role at `seguro.flant.com`; value = repository name (`virtualization`). Required by the upstream `.cve_scan` template for JWT-based Vault login. Without it, `cve:scan:*` jobs fail at Vault login. | ### Plain variables (`Masked = off`) @@ -210,6 +216,10 @@ Expected host tools for project-owned jobs: | Changelog/check-changelog jobs | `bash`, `python3`, `curl`, `jq`; changelog MR creation also needs `git`, `ssh-agent`, `ssh-add` | | Backport | `bash`, `git`, `curl`, `jq`, `ssh-agent`, `ssh-add` | | MR summary | `node`, `npm` | +| `test:scripts:js` | `node`, `npm` | +| `test:scripts:python` | `python3` | +| `test:build:d8v-cli` | `go`, `task` | +| `lint:dmt` | upstream Setup (trdl-installed `dmt`) | | Upstream scanning templates | use the requirements from `modules-gitlab-ci@v13.0` (for example CVE scan downloads `d8` and uses `curl`, `tar`, `jq`, `git`, SSH tools) | ## 7. Jobs reference @@ -219,12 +229,22 @@ Expected host tools for project-owned jobs: | `auto-assign-author` | info | MR opened / reopened | `GITLAB_API_TOKEN` | Assigns the MR author via API. Skips silently if the MR already has an assignee (plan §0(4)). | | `check:milestone` | lint | MR open / synchronize | `GITLAB_API_TOKEN` | Fails if MR has no `milestone` assigned. | | `check:changelog` | lint | MR open / synchronize | `GITLAB_API_TOKEN` | Validates ` ```changes ` blocks in MR description against `.gitlab/ci/changelog-sections.txt`. | -| `translate:changelog` | (template) | push to any branch except default | `RELEASE_TOKEN` (or `GITLAB_API_TOKEN`) | Extends upstream `.translate_and_create_mr` from `modules-gitlab-ci@v13.0`. Translates `CHANGELOG/v*.ru.yml` to English and opens an MR. | +| `translate:changelog` | pre | push to `main` / `release-X.Y` | `RELEASE_TOKEN` (falls back to `CI_JOB_TOKEN`) | Translates the latest English `CHANGELOG/CHANGELOG-v*.yml` to Russian (`.ru.yml`) and opens an MR. | +| `lint:dmt` | lint | MR (`merge_request_event`) | `DMT_METRICS_URL`, `DMT_METRICS_TOKEN` (optional) | Runs `dmt lint ./` (Deckhouse Module Tester). Non-blocking (`allow_failure: true`), mirroring GH `continue-on-error: true`. | +| `test:scripts:js` | test | MR (`merge_request_event`) | — | Runs `npm test` (node:test smoke test) in `.gitlab/scripts/js`. | +| `test:scripts:python` | test | MR (`merge_request_event`) | — | `python3 -m py_compile` syntax smoke check over `.gitlab/ci/scripts/python/*.py`. Non-blocking; real unit tests are a TODO. | +| `test:build:d8v-cli` | test | MR (`merge_request_event`) | — | Builds the `d8v` CLI (`task d8v-cli:build`) and runs `./src/cli/d8v --help`. | | `changelog:milestone` | lint | manual / scheduled | `GITLAB_API_TOKEN` | Re-generates `CHANGELOG/CHANGELOG-.yml` and `CHANGELOG/CHANGELOG-.md` from MRs with a milestone. Optionally opens a changelog MR. | | `changelog:all-active-milestones` | lint | manual / scheduled | `GITLAB_API_TOKEN` | Same as above, but iterates over all active milestones. | | `backport` | lint | manual with `TARGET_BRANCH` OR MR labelled `backport-release-X.Y` | `GITLAB_API_TOKEN` | Cherry-picks the merged MR into a new `backport//` branch, pushes it, and opens an MR to the release branch. | | `mrs:summary` | notify | manual / scheduled | `GITLAB_API_TOKEN`, `LOOP_WEBHOOK_URL` | Posts a markdown summary of open MRs to Loop (replaces `prs_notifier.mjs`). | +### CVE (Trivy) scan +- `cve:scan:mr` — per-MR scan of the dev build image (`mr${CI_MERGE_REQUEST_IID}`), runs after `build_dev`, gated on `merge_request_event`; non-blocking (`allow_failure: true`). +- `cve:scan:daily` — scheduled daily scan of `main` (`0 02 * * *`). +- `cve:scan:manual` — web-triggered manual scan of `${SCAN_TAG:-main}`. +All extend the upstream `.cve_scan` template. **Required Project CI/CD variable:** `VAULT_ROLE` (Vault role at seguro.flant.com; value = repository name, `virtualization`). Without it, scan jobs fail at Vault login. + ## 8. Manual pipelines GitLab has no native equivalent of `workflow_dispatch`; instead, jobs are @@ -299,6 +319,20 @@ under the `virtualization-m9e.3` Beads issue. If a secret remains in Vault only, the CVE-scan job needs a JWT-auth sidecar (out of scope for the first iteration). +### Ported/dropped GitHub jobs (issue `virtualization-m9e.5.8`) + +The GitHub workflow `.github/workflows/dev_module_build.yml` defined four jobs +with no direct GitLab counterpart. Decisions: + +| GH job | Decision | Notes | +|---|---|---| +| `lint_dmt` | **Ported** → `lint:dmt` (`.gitlab/ci/jobs/lint-dmt.yml`) | The upstream `Build.gitlab-ci.yml` ships a hidden `.lint` template running `dmt lint ./` with `allow_failure: true`. We mirror its script (we do NOT include `Build.gitlab-ci.yml` directly, to avoid its `.build`/`.deploy` rules bypassing our `.dev` gating) and rely on Setup's `before_script` to install `dmt` via `trdl`. `allow_failure: true` mirrors GH `continue-on-error: true`. `DMT_METRICS_URL`/`DMT_METRICS_TOKEN` are optional CI/CD vars. | +| `test_scripts_js` | **Ported** → `test:scripts:js` (`.gitlab/ci/jobs/test-scripts-js.yml`) | Fixed `.gitlab/scripts/js/package.json` which referenced a missing `mrs_notifier.test.mjs`; added that file as a minimal node:test smoke test (syntax check + structural assertions). `mrs_notifier.mjs` auto-runs at import, so full unit tests are a TODO pending a refactor exporting pure helpers. | +| `test_scripts_python` | **Ported (smoke)** → `test:scripts:python` (`.gitlab/ci/jobs/test-scripts-python.yml`) | `.gitlab/ci/scripts/python/` has no tests; this is a non-blocking `python3 -m py_compile` syntax check. Real unit tests are a TODO. | +| `test_build_d8v_cli` | **Ported** → `test:build:d8v-cli` (`.gitlab/ci/jobs/test-d8v-cli.yml`) | Runs `task d8v-cli:build` then `./src/cli/d8v --help`. MR-gated via `.dev`, mirroring the GH PR-only trigger. | + +All four jobs are wired into `.gitlab/ci/includes.yml`. + ## 11. Updating upstream templates (`modules-gitlab-ci`) The `include: project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' ref: 'v13.0'` diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index e99f632de7..0007f9d315 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -72,6 +72,9 @@ include: # --- Local job files owned by this issue --- - local: ".gitlab/ci/jobs/info.yml" - local: ".gitlab/ci/jobs/test.yml" + - local: ".gitlab/ci/jobs/test-scripts-js.yml" + - local: ".gitlab/ci/jobs/test-scripts-python.yml" + - local: ".gitlab/ci/jobs/test-d8v-cli.yml" - local: ".gitlab/ci/jobs/build-dev.yml" - local: ".gitlab/ci/jobs/build-prod.yml" - local: ".gitlab/ci/jobs/deploy-dev.yml" @@ -79,6 +82,7 @@ include: - local: ".gitlab/ci/jobs/cleanup.yml" # --- Local validation and scanning jobs --- + - local: ".gitlab/ci/jobs/lint-dmt.yml" - local: ".gitlab/ci/jobs/lint-validate.yml" - local: ".gitlab/ci/jobs/precache.yml" - local: ".gitlab/ci/jobs/svace.yml" diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml index b186fe85a6..ea07b1b007 100644 --- a/.gitlab/ci/jobs/cve-scan.yml +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -1,8 +1,10 @@ # CVE (Trivy) scan via the upstream `.cve_scan` template. # -# Migration of .github/workflows/cve_scan_daily.yml. The GH workflow -# pulled CVE_TEST_REPO_GIT, CODEOWNERS_REPO_TOKEN, DD_URL, DD_TOKEN, -# DECKHOUSE_PRIVATE_REPO, etc. from HashiCorp Vault via +# Migration of .github/workflows/cve_scan_daily.yml and the +# `cve_scan_on_pr` job from .github/workflows/dev_module_build.yml. +# +# The GH workflows pulled CVE_TEST_REPO_GIT, CODEOWNERS_REPO_TOKEN, +# DD_URL, DD_TOKEN, DECKHOUSE_PRIVATE_REPO, etc. from HashiCorp Vault via # `hashicorp/vault-action@v2`. The upstream # deckhouse/3p/deckhouse/modules-gitlab-ci@v13.0 `.cve_scan` template # does the same thing but via GitLab `id_tokens` + d8-cli. Both paths @@ -15,13 +17,24 @@ # Required CI/CD variables (declared in the project). The upstream # template provides defaults via `vault:` references, but the # operator must set: -# VAULT_ROLE - role to authenticate against in seguro.flant.com +# VAULT_ROLE - role to authenticate against in seguro.flant.com. +# Set as a Project CI/CD variable (see +# .gitlab/ci/variables.yml, "CVE scan" section). +# In the GH workflow this was the repo name +# (virtualization); reuse the same Vault role. # SOURCE_TAG - tag/branch to scan (e.g. "main" or a release tag) # CASE - case label forwarded to cve_scan.sh # EXTERNAL_MODULE_NAME - the module name (default: "virtualization") # -# The job runs on schedule + manual, matching the GH `on: schedule` and -# `on: workflow_dispatch` triggers. +# The upstream `.cve_scan` template sets `allow_failure: true`, so a scan +# failure never blocks the pipeline (mirrors the GH behavior where the +# action result did not fail the workflow job). +# +# MODULE_PROD_REGISTRY_CUSTOM_PATH and MODULE_DEV_REGISTRY_CUSTOM_PATH +# are intentionally NOT set here: the upstream `.cve_scan` template +# already defaults them to "deckhouse/fe/modules" and +# "sys/deckhouse-oss/modules" respectively. Restating them verbatim +# would be redundant. # --------------------------------------------------------------------------- # Scheduled daily scan against main. @@ -40,8 +53,6 @@ cve:scan:daily: SCAN_SEVERAL_LATEST_RELEASES: "True" LATEST_RELEASES_AMOUNT: "5" RELEASE_IN_DEV: "false" - MODULE_PROD_REGISTRY_CUSTOM_PATH: "deckhouse/fe/modules" - MODULE_DEV_REGISTRY_CUSTOM_PATH: "sys/deckhouse-oss/modules" rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' @@ -62,9 +73,36 @@ cve:scan:manual: SCAN_SEVERAL_LATEST_RELEASES: "${SCAN_SEVERAL:-False}" LATEST_RELEASES_AMOUNT: "5" RELEASE_IN_DEV: "${SCAN_TAG:-false}" - MODULE_PROD_REGISTRY_CUSTOM_PATH: "deckhouse/fe/modules" - MODULE_DEV_REGISTRY_CUSTOM_PATH: "sys/deckhouse-oss/modules" rules: - if: '$CI_PIPELINE_SOURCE == "web"' when: manual - when: never + +# --------------------------------------------------------------------------- +# Per-MR scan against the MR dev build. +# Mirrors the `cve_scan_on_pr` job in dev_module_build.yml, which ran +# `needs: [set_vars, dev_setup_build]` and scanned the dev build tag for +# the PR. Here it runs after build_dev (which tags images +# `mr${CI_MERGE_REQUEST_IID}` via the .dev template) and scans that tag. +# `allow_failure: true` is inherited from `.cve_scan`, matching the GH +# job (which had no continue-on-error but whose action result did not +# block the PR). +# --------------------------------------------------------------------------- + +cve:scan:mr: + extends: + - .cve_scan + stage: scan + interruptible: true + needs: + - build_dev + variables: + SOURCE_TAG: "mr${CI_MERGE_REQUEST_IID}" + CASE: "External Modules" + EXTERNAL_MODULE_NAME: "virtualization" + SCAN_SEVERAL_LATEST_RELEASES: "False" + LATEST_RELEASES_AMOUNT: "3" + RELEASE_IN_DEV: "false" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - when: never diff --git a/.gitlab/ci/jobs/lint-dmt.yml b/.gitlab/ci/jobs/lint-dmt.yml new file mode 100644 index 0000000000..99113b372c --- /dev/null +++ b/.gitlab/ci/jobs/lint-dmt.yml @@ -0,0 +1,43 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DMT (Deckhouse Module Tester) linter. +# +# Ports the GitHub Actions `lint_dmt` job from +# .github/workflows/dev_module_build.yml, which ran +# deckhouse/modules-actions/lint@v2 with continue-on-error: true. +# +# The upstream modules-gitlab-ci Build.gitlab-ci.yml ships a hidden `.lint` +# template that runs `dmt lint ./` with allow_failure: true. We do NOT +# include Build.gitlab-ci.yml directly: its `.build`/`.deploy` hidden jobs +# carry baked-in `rules:` that would bypass our strict .dev/.main/.prod +# gating (see .gitlab/ci/includes.yml). Instead this job mirrors the `.lint` +# script body and relies on the global `before_script` from +# Setup.gitlab-ci.yml (already included) to install dmt via trdl. +# +# dmt is NOT a host tool (it is installed per-job via trdl), so this job +# intentionally does NOT override `before_script` with check-runner-tools.sh; +# it inherits Setup's before_script, which performs the trdl+dmt bootstrap. +# +# DMT_METRICS_URL / DMT_METRICS_TOKEN are optional CI/CD variables consumed +# by dmt for metrics reporting; see .gitlab/README.md §3. + +lint:dmt: + stage: lint + interruptible: true + allow_failure: true + script: + - dmt lint ./ + extends: + - .dev diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 7a99ead161..2fe460dca2 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -131,7 +131,11 @@ lint:yaml: # golangci-lint v${GOLANGCI_LINT_VERSION} (same method/version as the GH job) # and runs `golangci-lint run` in every directory that ships a .golangci.yaml, # excluding the upstream/vendored modules that GH also skipped: -# images/cdi-cloner/cloner-startup, images/dvcr-artifact, test/performance/shatal. +# images/cdi-cloner/cloner-startup, images/dvcr-artifact. +# NOTE: .github/workflows/dev_module_build.yml also pruned ./test/performance/shatal, +# but that path never existed (the real module is test/performance/tools/shatal), +# so shatal was effectively linted there too. The dead prune entry is dropped here +# and test/performance/tools/shatal is intentionally linted. # This supersedes the old single-dir `lint:virtualization-controller` job. # --------------------------------------------------------------------------- @@ -153,12 +157,15 @@ lint:go: export PATH="$(go env GOPATH)/bin:${PATH}" # Directories containing a .golangci.yaml, excluding the same upstream / - # vendored modules as .github/workflows/dev_module_build.yml (lint_go). + # vendored modules as .github/workflows/dev_module_build.yml (lint_go): + # images/cdi-cloner/cloner-startup, images/dvcr-artifact. + # The GH prune of ./test/performance/shatal was a no-op (path never + # existed; real module is test/performance/tools/shatal), so shatal is + # intentionally linted here, matching actual current behavior. mapfile -t config_dirs < <( find . \ -path ./images/cdi-cloner/cloner-startup -prune -o \ -path ./images/dvcr-artifact -prune -o \ - -path ./test/performance/shatal -prune -o \ -type f -name '.golangci.yaml' -not -type l -printf '%h\0' | \ xargs -0 -n1 | sort -u ) diff --git a/.gitlab/ci/jobs/test-d8v-cli.yml b/.gitlab/ci/jobs/test-d8v-cli.yml new file mode 100644 index 0000000000..746b1d2f52 --- /dev/null +++ b/.gitlab/ci/jobs/test-d8v-cli.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# d8v CLI build + workability check. +# +# Ports the GitHub Actions `test_build_d8v_cli` job from +# .github/workflows/dev_module_build.yml, which ran `task d8v-cli:build`, +# installed the binary, and ran `d8v --help` as a smoke check (PR only). +# +# The root Taskfile.yaml `d8v-cli` include sets `dir: ./src/cli`, so +# `task d8v-cli:build` emits ./src/cli/d8v. The workability check invokes +# that binary with `--help` (cobra exits 0 on help). +# +# MR-gated via .dev (mirrors the GH PR-only trigger). + +test:build:d8v-cli: + stage: test + interruptible: true + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task + script: + - task d8v-cli:build + - ./src/cli/d8v --help + extends: + - .dev diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml new file mode 100644 index 0000000000..0603c38532 --- /dev/null +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -0,0 +1,41 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# JS smoke tests for .gitlab/scripts/js. +# +# Ports the GitHub Actions `test_scripts_js` job from +# .github/workflows/dev_module_build.yml, which ran `npm test` in +# .github/scripts/js. The GitLab counterpart lives in .gitlab/scripts/js +# (mrs_notifier.mjs); its `npm test` runs node:test over +# mrs_notifier.test.mjs, a minimal smoke test (syntax check + structural +# assertions) because mrs_notifier.mjs auto-runs at import time and cannot +# yet be imported by a unit test. See mrs_notifier.test.mjs for the TODO +# on adding real unit tests. +# +# MR-gated via .dev. The smoke test uses only Node.js built-ins, so no +# dependencies are required; `npm install` is still run to match the +# upstream job shape and to fail fast if package.json is broken. +# TODO: commit a package-lock.json and switch to `npm ci`. + +test:scripts:js: + stage: test + interruptible: true + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh node npm + script: + - cd .gitlab/scripts/js + - npm install --no-audit --no-fund + - npm test + extends: + - .dev diff --git a/.gitlab/ci/jobs/test-scripts-python.yml b/.gitlab/ci/jobs/test-scripts-python.yml new file mode 100644 index 0000000000..88cd410959 --- /dev/null +++ b/.gitlab/ci/jobs/test-scripts-python.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python syntax smoke check for .gitlab/ci/scripts/python. +# +# Ports the GitHub Actions `test_scripts_python` job from +# .github/workflows/dev_module_build.yml, which ran `python -m unittest` in +# .github/scripts/python. The GitLab Python scripts +# (changelog_collect.py, check_changelog_entry.py) currently have no unit +# tests, so this job is a non-blocking `py_compile` syntax check only. +# TODO: add real unit tests under .gitlab/ci/scripts/python/tests/ and +# switch this job to `python3 -m unittest discover`. +# +# MR-gated via .dev. + +test:scripts:python: + stage: test + interruptible: true + allow_failure: true + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh python3 + script: + - python3 -m py_compile .gitlab/ci/scripts/python/*.py + extends: + - .dev diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index ba3a3ccac4..8542d32a22 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -12,22 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Translate Russian CHANGELOG/*.ru.yml to English and open an MR. +# Translate English CHANGELOG/CHANGELOG-v*.yml to Russian (.ru.yml) and open an MR. # # Migration of .github/workflows/translate-changelog.yml which called -# deckhouse/modules-actions/translate-changelog@v10 (composite action). +# deckhouse/modules-actions/translate-changelog@v10 with source_lang: en, +# target_lang: ru, file_prefix: CHANGELOG-v, triggered on push to the +# default branch (main) and release-X.Y branches. +# +# Direction note (verified against upstream): +# The upstream modules-gitlab-ci@v13.0 `.translate_and_create_mr` template +# (templates/Translate_Changelog.gitlab-ci.yml) is hardcoded to the OPPOSITE +# direction (RU -> EN, globs `v*.ru.yml`) and exposes NO direction variables +# (only TRANSLATE_CHANGELOG_PATH / TRANSLATE_BASE_BRANCH). This repo has no +# *.ru.yml sources, only CHANGELOG-v*.yml (English), so the upstream script +# would no-op here. To match the GitHub workflow (EN -> RU) and this repo's +# file naming, this job overrides `script:` with an EN->RU implementation and +# declares the direction explicitly via TRANSLATE_SOURCE_LANG / +# TRANSLATE_TARGET_LANG, which are consumed by the local script below (NOT by +# the upstream template). The job still extends `.translate_and_create_mr` to +# inherit its image and base variables. +# +# Trigger: push to the default branch (main) or any release-X.Y branch, +# matching the GH workflow. The upstream template's default rule +# (`$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH`, which excludes main) is +# replaced by the explicit rules block below. +# +# Required CI/CD variable: RELEASE_TOKEN (or GITLAB_API_TOKEN, alias). +# The upstream template prefers RELEASE_TOKEN; if unset it falls back to +# CI_JOB_TOKEN (works in GitLab 15.9+ for push and MR create). Set +# RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN lacks +# push / merge-request permissions. # # Per migration plan §0(2) we delegate to the upstream GitLab CI template at # https://fox.flant.com/deckhouse/3p/deckhouse/modules-gitlab-ci (local checkout # at /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci, # branch v13.0, HEAD 006d51c35904b434eca2045a449aafb5e37a8827). -# -# The upstream template name is `.translate_and_create_mr` (see -# modules-gitlab-ci/templates/Translate_Changelog.gitlab-ci.yml). -# -# Required CI/CD variable: RELEASE_TOKEN (or GITLAB_API_TOKEN, alias). -# The upstream template prefers RELEASE_TOKEN; if unset it falls back to CI_JOB_TOKEN -# (works in GitLab 15.9+ for push and MR create). include: - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" @@ -36,12 +55,20 @@ include: # Local job definition that extends the upstream hidden job. # Tag override is intentional: we want our project's runner tag, not the -# upstream default. +# upstream default. The upstream `script:` is overridden with an EN->RU +# implementation (see direction note above) because the upstream template +# direction is hardcoded RU->EN and cannot be parameterized. translate:changelog: extends: .translate_and_create_mr stage: notify tags: - deckhouse + rules: + # Match the GH workflow trigger: push to main (default branch) or + # release-X.Y branches. Replaces the upstream default rule + # (`$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH`) which excludes main. + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+$/' before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh git curl jq python3 - python3 -m venv /tmp/venv @@ -53,9 +80,143 @@ translate:changelog: TRANSLATE_CHANGELOG_PATH: "CHANGELOG" # Target branch the resulting MR should target. TRANSLATE_BASE_BRANCH: "main" + # File prefix of English source changelogs (matches GH file_prefix). + TRANSLATE_FILE_PREFIX: "CHANGELOG-v" + # Translation direction (EN -> RU), matching the GH workflow. These are + # consumed by the local script below, NOT by the upstream template. + TRANSLATE_SOURCE_LANG: "en" + TRANSLATE_TARGET_LANG: "ru" # Upstream expects RELEASE_TOKEN or falls back to CI_JOB_TOKEN in its shell # script. GitLab does not evaluate shell parameter expansion in `variables:`; # set RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN # does not have enough permissions to push and create merge requests. # The local before_script intentionally replaces the upstream template's # container/package-manager setup so the job works on shell executors. + script: + - | + set -euo pipefail + export PATH="/tmp/venv/bin:$PATH" + CHANGELOG_PATH="${TRANSLATE_CHANGELOG_PATH}" + FILE_PREFIX="${TRANSLATE_FILE_PREFIX}" + SRC_LANG="${TRANSLATE_SOURCE_LANG}" + TGT_LANG="${TRANSLATE_TARGET_LANG}" + # Detect English source changelog changes in the last commit + # (exclude already-translated *.ru.yml files). + CHANGED=$(git show --name-only --pretty=format: HEAD \ + | grep "^${CHANGELOG_PATH}/${FILE_PREFIX}.*\.yml$" \ + | grep -v "\.ru\.yml$" || true) + if [ -z "$CHANGED" ]; then + echo "No ${FILE_PREFIX}*.yml (English) changelog files changed in last commit, skipping." + exit 0 + fi + echo "English changelog files changed in last commit: $CHANGED" + # Translate latest English changelog to Russian (EN -> RU), matching + # deckhouse/modules-actions/translate-changelog@v10. + TRANSLATE_OUTPUT=$(cd "${CI_PROJECT_DIR}" && python3 - "$CHANGELOG_PATH" "$FILE_PREFIX" "$SRC_LANG" "$TGT_LANG" << 'PYEOF' + import re + import sys + from pathlib import Path + try: + from packaging import version as pkg_version + from deep_translator import GoogleTranslator + except ImportError as e: + print(f"Import error: {e}", file=sys.stderr) + sys.exit(1) + changelog_dir, file_prefix, src_lang, tgt_lang = sys.argv[1:5] + path = Path(changelog_dir) + if not path.exists(): + sys.exit(0) + # English source files: *.yml excluding *.ru.yml. + en_files = [f for f in path.glob(f"{file_prefix}*.yml") if not f.name.endswith(".ru.yml")] + if not en_files: + sys.exit(0) + versions = [] + ver_re = re.compile(re.escape(file_prefix) + r"(\d+\.\d+\.\d+)\.yml$") + for f in en_files: + m = ver_re.match(f.name) + if m: + try: + ver = pkg_version.parse(m.group(1)) + versions.append((ver, f.name)) + except pkg_version.InvalidVersion: + pass + if not versions: + sys.exit(0) + versions.sort(reverse=True, key=lambda x: x[0]) + latest_ver, en_name = versions[0] + version_str = f"v{latest_ver}" + ru_name = en_name.replace(".yml", ".ru.yml") + if (path / ru_name).exists(): + sys.exit(0) + en_path, ru_path = path / en_name, path / ru_name + translator = GoogleTranslator(source=src_lang, target=tgt_lang) + with open(en_path, "r", encoding="utf-8") as f: + en_lines = f.readlines() + with open(ru_path, "w", encoding="utf-8") as f: + for line in en_lines: + if not line.strip(): + f.write(line) + continue + indent_len = len(line) - len(line.lstrip()) + content = line.strip() + try: + translated = translator.translate(content) + except Exception: + translated = content + f.write(" " * indent_len + translated + "\n") + print(f"VERSION={version_str}") + print(f"EN_FILE={en_name}") + print(f"RU_FILE={ru_name}") + PYEOF + ) || TRANSLATE_OUTPUT="" + # Parse VERSION from output; if missing, nothing was translated. + VERSION=$(echo "$TRANSLATE_OUTPUT" | grep "^VERSION=" | cut -d= -f2) + if [ -z "$VERSION" ]; then + echo "No English changelog to translate (or Russian already exists)." + exit 0 + fi + EN_FILE=$(echo "$TRANSLATE_OUTPUT" | grep "^EN_FILE=" | cut -d= -f2) + RU_FILE=$(echo "$TRANSLATE_OUTPUT" | grep "^RU_FILE=" | cut -d= -f2) + echo "Translated $EN_FILE -> $RU_FILE (version $VERSION)" + # Commit and push + git add "${CHANGELOG_PATH}/" + git status + if git diff --staged --quiet; then + echo "No staged changes." + exit 0 + fi + git commit -m "Translate changelog ${VERSION} to Russian" + BRANCH="${CI_COMMIT_REF_NAME:-${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}}" + if [ -z "$BRANCH" ]; then + echo "Cannot push: CI_COMMIT_REF_NAME and CI_MERGE_REQUEST_SOURCE_BRANCH_NAME are empty." + exit 1 + fi + git checkout -B "$BRANCH" + REPO_URL="https://oauth2:${RELEASE_TOKEN:-$CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" + git remote set-url origin "$REPO_URL" + git push origin "$BRANCH" + # Create MR if not exists + EXISTING_MR=$(curl -s --header "PRIVATE-TOKEN: ${RELEASE_TOKEN:-$CI_JOB_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests?source_branch=${CI_COMMIT_REF_NAME}&target_branch=${TRANSLATE_BASE_BRANCH}&state=opened" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['iid'] if d else '')" 2>/dev/null || echo "") + if [ -n "$EXISTING_MR" ]; then + echo "MR !${EXISTING_MR} already exists for ${CI_COMMIT_REF_NAME} -> ${TRANSLATE_BASE_BRANCH}" + exit 0 + fi + RU_PATH="${CI_PROJECT_DIR}/${CHANGELOG_PATH}/${RU_FILE}" + DESC_FILE="/tmp/mr_desc.md" + printf "## Changelog %s\n\nThis MR contains the Russian translation of the English changelog for version %s.\n\n**Source file:** \\\`%s/%s\\\`\n**Translated file:** \\\`%s/%s\\\`\n\n
\nChangelog content\n\n\\\`\\\`\\\`yaml\n%s\n\\\`\\\`\\\`\n\n
\n" \ + "$VERSION" "$VERSION" "$CHANGELOG_PATH" "$EN_FILE" "$CHANGELOG_PATH" "$RU_FILE" "$(cat "$RU_PATH")" > "$DESC_FILE" + jq -n \ + --arg source_branch "$CI_COMMIT_REF_NAME" \ + --arg target_branch "${TRANSLATE_BASE_BRANCH}" \ + --arg title "$VERSION" \ + --rawfile description "$DESC_FILE" \ + '{ source_branch: $source_branch, target_branch: $target_branch, title: $title, description: $description }' \ + | curl -s --request POST \ + --header "PRIVATE-TOKEN: ${RELEASE_TOKEN:-$CI_JOB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data @- \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print('Created MR !'+str(d.get('iid','')))" + echo "Translate changelog and create MR done." diff --git a/.gitlab/ci/scripts/python/changelog_collect.py b/.gitlab/ci/scripts/python/changelog_collect.py index a02420765f..0d0c6230f3 100644 --- a/.gitlab/ci/scripts/python/changelog_collect.py +++ b/.gitlab/ci/scripts/python/changelog_collect.py @@ -436,7 +436,11 @@ def main() -> int: f"- `{yml_path.relative_to(project_dir)}`\n" f"- `{md_path.relative_to(project_dir)}`\n" ) - body_path = project_dir / "CHANGELOG" / f".mr-body-{title}.md" + # Write the MR body OUTSIDE CHANGELOG/ so `git add CHANGELOG/` + # in push_changelog_mr cannot accidentally stage it. It is only + # consumed by reading its content into a merge_request.description + # push option and is never committed to the changelog branch. + body_path = project_dir / f".mr-body-{title}.md" body_path.write_text(pr_body, encoding="utf-8") try: push_changelog_mr( diff --git a/.gitlab/ci/templates/dev_vars.yml b/.gitlab/ci/templates/dev_vars.yml index 49a598447c..ab529a5563 100644 --- a/.gitlab/ci/templates/dev_vars.yml +++ b/.gitlab/ci/templates/dev_vars.yml @@ -17,8 +17,3 @@ MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" MODULES_MODULE_SOURCE: "${DEV_MODULE_SOURCE}" ENV: DEV - # Backwards-compat fallback for the previous CI/CD variable names. Safe - # to remove once Settings -> CI/CD -> Variables no longer carries the - # EXTERNAL_MODULES_DEV_* entries (see .gitlab/README.md migration step). - DEV_REGISTRY_FALLBACK_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" - DEV_REGISTRY_FALLBACK_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" diff --git a/.gitlab/ci/templates/prod_vars.yml b/.gitlab/ci/templates/prod_vars.yml index 22d8dde139..225768d797 100644 --- a/.gitlab/ci/templates/prod_vars.yml +++ b/.gitlab/ci/templates/prod_vars.yml @@ -18,6 +18,3 @@ MODULES_REGISTRY_PASSWORD: "${PROD_MODULES_REGISTRY_PASSWORD}" MODULES_MODULE_SOURCE: "${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}/${EDITION}/modules" ENV: PROD - # Backwards-compat fallback. Same migration caveat as .dev_vars. - PROD_REGISTRY_FALLBACK_LOGIN: "${PROD_MODULES_REGISTRY_LOGIN}" - PROD_REGISTRY_FALLBACK_PASSWORD: "${PROD_MODULES_REGISTRY_PASSWORD}" diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index 882893e704..42917d31fe 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -54,6 +54,21 @@ variables: # Optional read-only license registry (used by cve-scan + check:requirements): # PROD_READ_REGISTRY, PROD_READ_REGISTRY_USER, PROD_READ_REGISTRY_PASSWORD + # --- CVE scan (Trivy) --- + # Required Project Variable (vars, not masked): + # VAULT_ROLE - role name to authenticate against seguro.flant.com + # via GitLab id_tokens (aud: gitlab-access-aud) + + # d8-cli `stronghold write auth/fox/login`. + # The upstream `.cve_scan` template declares + # VAULT_ROLE with no default; the operator MUST set + # it here or as a Project CI/CD variable, otherwise + # cve:scan:daily / cve:scan:manual / cve:scan:mr + # fail at the Vault login step. + # Value: the same role the GH workflow used, i.e. + # the repository name ("virtualization"). + # VAULT_ADDR - defaults to https://seguro.flant.com upstream; + # override only if the Vault endpoint differs. + # --- Build/dev tooling (kept from previous root) --- GO_VERSION: "1.25.11" GOLANGCI_LINT_VERSION: "2.11.1" diff --git a/.gitlab/scripts/js/mrs_notifier.test.mjs b/.gitlab/scripts/js/mrs_notifier.test.mjs new file mode 100644 index 0000000000..c884399af4 --- /dev/null +++ b/.gitlab/scripts/js/mrs_notifier.test.mjs @@ -0,0 +1,65 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Minimal smoke test for mrs_notifier.mjs. +// +// mrs_notifier.mjs is a script that auto-runs `run()` at import time and +// exits when required env vars are missing, so it cannot be safely imported +// by a unit test without a refactor. This smoke test therefore: +// - asserts the file exists and is non-empty; +// - syntax-checks it with `node --check` (no execution, no side effects); +// - asserts the expected entry points and env-var contract are present. +// +// TODO: refactor mrs_notifier.mjs to export pure helpers (e.g. classifyMR) +// and guard the `run()` call behind a direct-invocation check, then add real +// unit tests over the classification logic here. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, statSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const MODULE_PATH = path.join(__dirname, 'mrs_notifier.mjs'); + +test('mrs_notifier.mjs exists and is non-empty', () => { + const stat = statSync(MODULE_PATH); + assert.ok(stat.isFile(), 'mrs_notifier.mjs should be a file'); + const src = readFileSync(MODULE_PATH, 'utf8'); + assert.ok(src.trim().length > 0, 'mrs_notifier.mjs should not be empty'); +}); + +test('mrs_notifier.mjs is syntactically valid (node --check)', () => { + // Syntax-check without executing: avoids env-var / network side effects. + execFileSync('node', ['--check', MODULE_PATH], { stdio: 'pipe' }); +}); + +test('mrs_notifier.mjs declares the expected entry points', () => { + const src = readFileSync(MODULE_PATH, 'utf8'); + for (const name of ['fetchOpenMRs', 'classifyMR', 'buildSummary', 'run']) { + assert.ok( + src.includes(`function ${name}`) || src.includes(`${name}(`), + `expected function ${name} to be defined`, + ); + } +}); + +test('mrs_notifier.mjs references the documented env-var contract', () => { + const src = readFileSync(MODULE_PATH, 'utf8'); + for (const envVar of ['GITLAB_API_TOKEN', 'CI_API_V4_URL', 'CI_PROJECT_ID', 'LOOP_WEBHOOK_URL']) { + assert.ok(src.includes(envVar), `expected reference to ${envVar}`); + } +}); From 70515fa87197fa1b6d6ebf6b41b65f64e0675d95 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 18:14:54 +0300 Subject: [PATCH 28/60] update README.md Signed-off-by: Nikita Korolev --- .gitlab/README.md | 101 +++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/.gitlab/README.md b/.gitlab/README.md index ca3d7f4bd0..5cad6542f8 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -268,24 +268,72 @@ for the future plan. ## 9. Scheduled pipelines -Some jobs are intended to run on a schedule (e.g. `mrs:summary` once per day -at 10:00 Moscow time). Configure them at -`CI/CD -> Schedules -> New schedule`: +Several jobs are intended to run on a schedule. Configure schedules at +`CI/CD -> Schedules -> New schedule`. Each scheduled job has its own intended +cadence: -| Schedule name | Cron | Target branch | Variables | -|---|---|---|---| -| `mrs-summary-daily` | `0 7 * * *` (10:00 MSK) | `main` | _(none — uses project vars)_ | -| `changelog-sweep` | `0 3 * * *` (06:00 MSK) | `main` | `OPEN_CHANGELOG_MR=false` | - -Schedules trigger pipelines whose `CI_PIPELINE_SOURCE == "schedule"`. Jobs -that should run on a schedule have a corresponding rule with -`when: manual allow_failure: true` so they don't break the schedule if a -maintainer hasn't pre-approved them. +| Job(s) | Intended cron | Notes | +|---|---|---| +| `cve:scan:daily` | `0 2 * * *` (daily) | CVE scan of `main`. | +| `svace:*` | `0 4 * * 6` (weekly, Sat) | Svace analysis + report. | +| `gitleaks:full:scheduled` | daily | Full secrets scan. | +| `precache` | `0 */8 * * *` (every 8h) | Warm the build cache. | +| `changelog:milestone` / `changelog:all-active-milestones` | nightly | Re-generate CHANGELOG. | +| `mrs:summary` | `0 7 * * *` (10:00 MSK) | MR summary to Loop. | +| `cleanup` | `12 0 * * 6` (weekly, Sat) | Prune old DEV registry images. | + +Schedules trigger pipelines whose `CI_PIPELINE_SOURCE == "schedule"`. + +> **Known issue — tracked in `virtualization-m9e.5.11`.** Every scheduled job +> currently gates **only** on `CI_PIPELINE_SOURCE == "schedule"` with **no +> discriminator**. In GitLab a schedule triggers the whole pipeline, so a +> single schedule fires **all** of these jobs at once, and multiple schedules +> each fire **all** of them — there is no way to honor the distinct crons +> above (e.g. weekly `cleanup` vs daily `cve:scan:daily`). Result: `cleanup` +> never runs at its intended weekly cadence (it runs at whatever cadence any +> schedule has, or never if no schedule exists). The fix is a `SCHEDULE_TYPE` +> (or `SCHEDULE_CRON`) variable set per schedule, with each job gated on +> `CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == ""`. ## 10. Known TODOs / migration risks -These are intentional gaps from the first-iteration migration. Track them -under the `virtualization-m9e.3` Beads issue. +Migration goal: **full parity with GitHub Actions** (development is moving to +GitLab). The previous root `.gitlab-ci.yml` is **not** a behavior baseline — +anything that diverges from the GitHub workflows is a gap to fix, even if it +matches the old GitLab config. Open work is tracked under sub-epics +`virtualization-m9e.6` (prod release parity) and `virtualization-m9e.5` +(review findings & fixes). + +### Confirmed bugs / parity gaps (must fix) + +- **`build_prod` builds `ce` as EE** (`virtualization-m9e.6.1`) — `.prod_vars` + never sets `MODULE_EDITION`, so werf defaults to `EE`. GitHub sets + `MODULE_EDITION=CE` only for `ce` (EE for `ee`/`se-plus`/`fe`). The `ce` + prod image is therefore mis-built as EE. +- **Edition `se-plus` missing in prod** (`virtualization-m9e.6.2`) — GitHub + `release_module_build-and-registration.yml` and + `release_module_release-channels.yml` build and deploy `se-plus` + (path `se-plus/modules`, `MODULE_EDITION=EE`). The GitLab build/deploy + matrices only have `ce`/`ee`/`fe`. This is a parity gap to fix, not a + conditional feature. +- **Release-channels feature parity** (`virtualization-m9e.6.3`) — the GitHub + `release_module_release-channels.yml` workflow exposes inputs (`channel`, + `ce`, `ee`, `tag`, `enableBuild`, `release_to_github`, `check_only`, + `skip_requirements_check`, `send_results_to_loop`) and runs requirements + check, version check (registry/releases/documentation), release creation, + and a Loop notification. The GitLab port collapses this into a hardcoded + `RELEASE_CHANNEL x EDITION` matrix and is missing the rest. Audit upstream + `modules-gitlab-ci` templates first, then port or document each capability. +- **Scheduled-job discriminator** (`virtualization-m9e.5.11`) — see the known + issue in [§9](#9-scheduled-pipelines): `cleanup` and the other scheduled + jobs cannot honor their individual crons without a `SCHEDULE_TYPE` variable. +- **`test` serialized behind `lint`** (`virtualization-m9e.5.12`) — test jobs + lack `needs:`, so they wait for the whole `lint` stage instead of running in + parallel as on GitHub. (Mixing lint and validation in one stage is fine — + same-stage jobs already run in parallel; the fix is a DAG `needs:`, not + splitting stages.) + +### Intentional first-iteration gaps - **Webhook listener for slash-commands** — GitLab does not natively start pipelines on MR comment creation or label change. See @@ -294,22 +342,11 @@ under the `virtualization-m9e.3` Beads issue. the webhook listener lands, add a `merge_request.closed` / `milestoned` handler that triggers `changelog:milestone` with the right `MILESTONE_TITLE`. -- **Edition `se-plus`** — the legacy `.gitlab-ci.yml` builds only - `ce`/`ee`/`fe`. GitHub's `release_module_build-and-registration.yml` - also built `se-plus`. Add `se-plus` to the `parallel.matrix.EDITION` - list if/when Deckhouse supports it for this module. -- **Inputs of `release_module_release-channels.yml`** — the GH workflow - exposed `channel`, `ce`, `ee`, `tag`, `enableBuild`, - `release_to_github`, `check_only`, `skip_requirements_check`, - `send_results_to_loop`. The current prod deploy jobs only accept a - tag-based trigger. Variables for `channel`, `tag`, and `check_only` are - already supported in the deploy job UI. The rest are tracked under the - build/deploy epic. -- **`prs_notifier.mjs` STUCK detection** — the original GitHub version - uses per-review `submitted_at` to compute "stuck for 1.5 days". The - GitLab port currently treats all unresolved discussions as "stuck" - without checking thread age. TODO: pull `discussions[].notes[].created_at` - to refine the heuristic. +- **`prs_notifier.mjs` STUCK detection** (`virtualization-m9e.5.14`) — the + original GitHub version uses per-review `submitted_at` to compute "stuck for + 1.5 days". The GitLab port currently treats all unresolved discussions as + "stuck" without checking thread age. TODO: pull + `discussions[].notes[].created_at` to refine the heuristic. - **GitLab username for `z9r5`** — the doc reviewer is hard-coded as `DOC_REVIEWER=z9r5` in `mrs:summary`. Override via the `DOC_REVIEWER` CI/CD variable until the real username is confirmed. @@ -367,8 +404,8 @@ For full GitHub parity, deploy a small **webhook-listener** service that: `POST /api/v4/projects/:id/trigger/pipeline` with the right variables. Until that exists, the manual job matrix in [§7](#7-jobs-reference) and the -two scheduled jobs in [§9](#9-scheduled-pipelines) cover the same surface -area, with a human pressing the button. +scheduled jobs in [§9](#9-scheduled-pipelines) cover the same surface area, +with a human pressing the button. ## See also From 98f3c55d7cd99aab70fd0108bcdadf89d1f167ed Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 18:40:18 +0300 Subject: [PATCH 29/60] =?UTF-8?q?fix(ci):=20prod=20release=20parity=20?= =?UTF-8?q?=E2=80=94=20MODULE=5FEDITION,=20se-plus,=20release-channels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit m9e.6.1 (bug): build_prod/deploy_to_prod_* set MODULE_EDITION per edition via parallel:matrix (CE for ce, EE for ee/se-plus/fe), matching the GitHub release_module_build-and-registration.yml. .werf/consts.yaml defaults MODULE_EDITION to EE and uses it as a Go build tag + EE-only image gate, so without this the ce prod image was built as EE. .prod_vars intentionally does not set it (the matrix provides it). m9e.6.2: add se-plus (path se-plus/modules, MODULE_EDITION=EE) to the build_prod and deploy_to_prod_* matrices. GitLab runs all four editions in parallel; GH runs se-plus/fe after ee (needs: ee) — acceptable since each edition builds into its own registry subpath with no shared artifact. m9e.6.3: port the release_module_release-channels.yml flow to .gitlab/ci/jobs/release-channels.yml as a manual Run-pipeline flow (prod:* jobs, CI_PIPELINE_SOURCE == web, never collides with the tag-push chain). Adds: RELEASE_CHANNEL/RELEASE_TAG/EDITION_CE/EDITION_EE/ ENABLE_BUILD/CHECK_ONLY/SKIP_REQUIREMENTS_CHECK/RELEASE_TO_GITLAB/ SEND_RESULTS_TO_LOOP inputs, prod:check-requirements, per-edition build/deploy (se-plus/fe needs prod:build:ee), prod:check-version matrix (registry/releases/documentation via tools/moduleversions), prod:create-gitlab-release (GitLab Releases from the merged changelog MR, was create-github-release), prod:notify-loop (status table to Loop). New .prod_read_registry template + prod_check/prod_deploy/prod_verify/ prod_release stages. Documented deviations: GitLab Releases instead of GitHub Releases; pipeline must run on the tag (RELEASE_TAG == CI_COMMIT_TAG, enforced) because GitLab cannot checkout an arbitrary tag without overriding the inherited werf Setup before_script; resource_group replaces GH cancel-in-progress concurrency. README updated (§7 jobs table, §8 manual pipelines, §10 known TODOs). Signed-off-by: Nikita Korolev --- .gitlab/README.md | 75 ++- .gitlab/ci/includes.yml | 4 + .gitlab/ci/jobs/build-prod.yml | 41 +- .gitlab/ci/jobs/deploy-prod.yml | 95 ++-- .gitlab/ci/jobs/release-channels.yml | 445 ++++++++++++++++++ .../ci/scripts/bash/notify-release-loop.sh | 126 +++++ .gitlab/ci/stages.yml | 10 +- .gitlab/ci/templates/prod_manual.yml | 6 + .gitlab/ci/templates/prod_vars.yml | 20 + 9 files changed, 753 insertions(+), 69 deletions(-) create mode 100644 .gitlab/ci/jobs/release-channels.yml create mode 100755 .gitlab/ci/scripts/bash/notify-release-loop.sh diff --git a/.gitlab/README.md b/.gitlab/README.md index 5cad6542f8..9d2f785de6 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -239,6 +239,31 @@ Expected host tools for project-owned jobs: | `backport` | lint | manual with `TARGET_BRANCH` OR MR labelled `backport-release-X.Y` | `GITLAB_API_TOKEN` | Cherry-picks the merged MR into a new `backport//` branch, pushes it, and opens an MR to the release branch. | | `mrs:summary` | notify | manual / scheduled | `GITLAB_API_TOKEN`, `LOOP_WEBHOOK_URL` | Posts a markdown summary of open MRs to Loop (replaces `prs_notifier.mjs`). | +### Prod release channels (manual `Run pipeline`) + +Parity with `.github/workflows/release_module_release-channels.yml`. A manual +single-channel/single-tag release flow triggered from `Run pipeline` on the +tag `$RELEASE_TAG` (`CI_PIPELINE_SOURCE == "web"`). It never collides with +the tag-push flow (`build_prod` / `deploy_to_prod_*`) because it only runs on +web pipelines. + +| Job | Stage | What it does | +|---|---|---| +| `prod:print-vars` | info | Echoes release inputs and enforces `RELEASE_TAG` matches the pipeline ref tag. | +| `prod:check-requirements` | prod_check | `tools/moduleversions check:requirements` (Deckhouse version range). Skipped when `CHECK_ONLY` or `SKIP_REQUIREMENTS_CHECK`. | +| `prod:build:` | build | Optional (`ENABLE_BUILD`) werf build per edition. `se-plus`/`fe` `needs: prod:build:ee` (GH cascade). | +| `prod:deploy:` | prod_deploy | `crane copy :$RELEASE_TAG -> :$RELEASE_CHANNEL` per selected edition. | +| `prod:check-version` | prod_verify | Matrix `registry`/`releases`/`documentation` via `tools/moduleversions`. | +| `prod:create-gitlab-release` | prod_release | Creates a GitLab release from the merged `label:changelog` MR description (was `create-github-release`). | +| `prod:notify-loop` | notify | Posts a release status table to Loop (`LOOP_WEBHOOK_URL`). | + +**Required Project CI/CD variables:** `GITLAB_API_TOKEN` (api scope, +Maintainer role — release creation + MR query), `LOOP_WEBHOOK_URL`, and the +read-only `PROD_READ_REGISTRY` / `PROD_READ_REGISTRY_USER` / +`PROD_READ_REGISTRY_PASSWORD` triple. Note: `tools/moduleversions` hardcodes +`registry.deckhouse.io`; `PROD_READ_REGISTRY` must resolve to that host for +the crane login to apply. + ### CVE (Trivy) scan - `cve:scan:mr` — per-MR scan of the dev build image (`mr${CI_MERGE_REQUEST_IID}`), runs after `build_dev`, gated on `merge_request_event`; non-blocking (`allow_failure: true`). - `cve:scan:daily` — scheduled daily scan of `main` (`0 02 * * *`). @@ -258,6 +283,12 @@ pipeline: - For `changelog:milestone`: optionally set `MILESTONE_TITLE=v1.21.3` and `OPEN_CHANGELOG_MR=true`. - For `mrs:summary`: ensure `LOOP_WEBHOOK_URL` is set. + - For a **prod release-channel dispatch** (`prod:*` jobs): run the pipeline + **on the tag** (`$RELEASE_TAG`) and set `RELEASE_TAG=vX.Y.Z`, + `RELEASE_CHANNEL=alpha|beta|early-access|stable|rock-solid`, + `EDITION_CE=true` and/or `EDITION_EE=true`, plus `ENABLE_BUILD`, + `CHECK_ONLY`, `SKIP_REQUIREMENTS_CHECK`, `RELEASE_TO_GITLAB`, + `SEND_RESULTS_TO_LOOP` as needed. See [§7 Prod release channels](#7-jobs-reference). 4. Submit. The pipeline starts; manual jobs appear under the pipeline view with a `play` button. @@ -306,24 +337,32 @@ matches the old GitLab config. Open work is tracked under sub-epics ### Confirmed bugs / parity gaps (must fix) -- **`build_prod` builds `ce` as EE** (`virtualization-m9e.6.1`) — `.prod_vars` - never sets `MODULE_EDITION`, so werf defaults to `EE`. GitHub sets - `MODULE_EDITION=CE` only for `ce` (EE for `ee`/`se-plus`/`fe`). The `ce` - prod image is therefore mis-built as EE. -- **Edition `se-plus` missing in prod** (`virtualization-m9e.6.2`) — GitHub - `release_module_build-and-registration.yml` and - `release_module_release-channels.yml` build and deploy `se-plus` - (path `se-plus/modules`, `MODULE_EDITION=EE`). The GitLab build/deploy - matrices only have `ce`/`ee`/`fe`. This is a parity gap to fix, not a - conditional feature. -- **Release-channels feature parity** (`virtualization-m9e.6.3`) — the GitHub - `release_module_release-channels.yml` workflow exposes inputs (`channel`, - `ce`, `ee`, `tag`, `enableBuild`, `release_to_github`, `check_only`, - `skip_requirements_check`, `send_results_to_loop`) and runs requirements - check, version check (registry/releases/documentation), release creation, - and a Loop notification. The GitLab port collapses this into a hardcoded - `RELEASE_CHANNEL x EDITION` matrix and is missing the rest. Audit upstream - `modules-gitlab-ci` templates first, then port or document each capability. +- **`build_prod` builds `ce` as EE** (`virtualization-m9e.6.1`) — **FIXED**. + `.gitlab/ci/jobs/build-prod.yml` and `deploy-prod.yml` now set + `MODULE_EDITION` per edition via the `parallel:matrix` (CE for `ce`, EE for + `ee`/`se-plus`/`fe`), matching GitHub. `.prod_vars` intentionally does not + set it (it comes from the matrix). +- **Edition `se-plus` missing in prod** (`virtualization-m9e.6.2`) — **FIXED**. + `se-plus` (path `se-plus/modules`, `MODULE_EDITION=EE`) is added to the + `build_prod` and `deploy_to_prod_*` matrices, matching the GitHub release + workflows. Note: GH runs `se-plus`/`fe` after `ee` (`needs:`); the GitLab + matrix runs all four editions in parallel — acceptable since each edition + builds into its own registry subpath with no shared artifact. +- **Release-channels feature parity** (`virtualization-m9e.6.3`) — **PORTED** + (with documented deviations). `.gitlab/ci/jobs/release-channels.yml` adds a + manual `Run pipeline` flow (`prod:*` jobs) mirroring + `release_module_release-channels.yml`: `RELEASE_CHANNEL`/`EDITION_CE`/ + `EDITION_EE`/`RELEASE_TAG`/`ENABLE_BUILD`/`CHECK_ONLY`/ + `SKIP_REQUIREMENTS_CHECK`/`RELEASE_TO_GITLAB`/`SEND_RESULTS_TO_LOOP` inputs, + requirements check (`prod:check-requirements`), build/deploy per edition, + version verification matrix (`prod:check-version`), release creation + (`prod:create-gitlab-release`), and Loop notification (`prod:notify-loop`). + Deviations from GitHub (intentional): release creation targets **GitLab + Releases** (not GitHub Releases) since development moves to GitLab; the + pipeline must run **on the tag** (`RELEASE_TAG == CI_COMMIT_TAG`, enforced + by `prod:print-vars`) because GitLab cannot checkout an arbitrary tag + without overriding the inherited werf `Setup` before_script; per-(channel,tag) + concurrency uses `resource_group` (serialize, no cancel-in-progress). - **Scheduled-job discriminator** (`virtualization-m9e.5.11`) — see the known issue in [§9](#9-scheduled-pipelines): `cleanup` and the other scheduled jobs cannot honor their individual crons without a `SCHEDULE_TYPE` variable. diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 0007f9d315..66d62fe921 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -79,6 +79,10 @@ include: - local: ".gitlab/ci/jobs/build-prod.yml" - local: ".gitlab/ci/jobs/deploy-dev.yml" - local: ".gitlab/ci/jobs/deploy-prod.yml" + # Prod release-channels parity flow (manual single-channel dispatch): + # requirements check, build/deploy per edition, version verification, + # GitLab release creation, Loop notification. See virtualization-m9e.6.3. + - local: ".gitlab/ci/jobs/release-channels.yml" - local: ".gitlab/ci/jobs/cleanup.yml" # --- Local validation and scanning jobs --- diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml index 52e94a8f85..f86a8787dd 100644 --- a/.gitlab/ci/jobs/build-prod.yml +++ b/.gitlab/ci/jobs/build-prod.yml @@ -1,17 +1,23 @@ # PROD build job. # -# Carries forward build_prod from the previous root .gitlab-ci.yml. Builds -# the module images for every edition on tag push (vX.Y.Z), then leaves -# the deploy step to .gitlab/ci/jobs/deploy-prod.yml. +# Builds the module images for every edition on tag push (vX.Y.Z), then +# leaves the deploy step to .gitlab/ci/jobs/deploy-prod.yml. # -# Editions: ce / ee / fe today. The previous GH workflow -# release_module_build-and-registration.yml also had an `se-plus` edition; -# the migration plan flags this as a TODO (plan §7, question about se-plus -# edition parity). Until that decision is made, we keep the ce/ee/fe matrix -# from the previous GitLab config to preserve behavior. +# Parity with .github/workflows/release_module_build-and-registration.yml: +# four editions ce / ee / se-plus / fe, each with its own +# MODULES_MODULE_SOURCE subpath and the correct MODULE_EDITION (CE only for +# ce, EE for ee/se-plus/fe). MODULE_EDITION is consumed by +# .werf/consts.yaml (defaults to EE) and used as a Go build tag plus an +# EE-only image gate, so it MUST be set per edition — otherwise the ce prod +# image is built as EE. # -# Resource group: prod — prevents two prod builds from racing (kept from -# the previous root .gitlab-ci.yml). +# Note vs GitHub: GH runs ce and ee in parallel, then se-plus and fe after +# ee (needs: prod_ee_setup_build). The GitLab matrix runs all four in +# parallel. Each edition builds into its own registry subpath with no shared +# artifact, so the GH `needs: ee` ordering is a sequencing/resource guard, +# not a data dependency; the parallel matrix is acceptable. +# +# Resource group: prod — prevents two prod builds from racing. # # Upstream .build expects MODULES_MODULE_SOURCE and WERF_REPO; .prod_vars # sets MODULES_MODULE_SOURCE per-edition (PROD_REGISTRY/PROD_MODULE_SOURCE_NAME/ @@ -33,10 +39,11 @@ build_prod: - .dual_registry_login parallel: matrix: - - EDITION: - - ce - - ee - - fe - # TODO: add `se-plus` edition here once parity with the GH - # release_module_build-and-registration.yml workflow is decided - # (see migration plan §7, "TODO: edition se-plus"). + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index eea7e907c3..97810a11f3 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -1,10 +1,16 @@ # PROD deploy jobs. # -# Carries forward deploy_to_prod_{alpha,beta,ea,stable,rock_solid} from -# the previous root .gitlab-ci.yml. Each job runs after the previous one -# (alpha -> beta -> ea -> stable -> rock_solid), waits for the matching -# matrix entry of build_prod, and copies the built release image to the -# named release channel. +# Parity with .github/workflows/release_module_release-channels.yml release +# channels: alpha -> beta -> early-access -> stable -> rock-solid. Each job +# waits for the previous one (and build_prod for alpha), then copies the +# built release image to the named release channel. All five are manual +# (when: manual from .prod_manual) on a vX.Y.Z tag. +# +# The EDITION matrix mirrors build_prod (ce/ee/se-plus/fe) with the correct +# MODULE_EDITION per edition, so deploy runs against the same per-edition +# source path the build pushed to. MODULE_EDITION is forwarded to the +# deploy step for consistency with the build (werf deploy reads it from the +# environment). # # Extends: # - .local_deploy (this repo — mirrors upstream modules-gitlab-ci @@ -15,15 +21,18 @@ # with `when: manual`). # - .dual_registry_login (this repo — also login against DEV_REGISTRY). # -# All five are manual (`when: manual` from .prod_manual) and depend on -# each other via `needs:` to preserve the gate-by-gate prod release flow. +# TODO: per migration plan §11.4.1 and sub-epic virtualization-m9e.6.3, the +# GH release_module_release-channels workflow exposes inputs (channel, +# ce/ee, tag, enableBuild, release_to_github, check_only, +# send_results_to_loop, requirements/version checks, github release +# creation, Loop notification) that are not yet ported. Track in m9e.6.3. # -# TODO: per migration plan §11.4.1, the GH release_module_release-channels -# workflow exposes inputs (channel, ce/ee, tag, enableBuild, -# release_to_github, check_only, send_results_to_loop, ...) that we -# currently collapse into the hardcoded RELEASE_CHANNEL x EDITION matrix. -# Decide whether to expose that flexibility via "Run pipeline" UI variables -# before locking the release flow down. +# This file is the tag-push chained-deploy flow. The GitHub +# release_module_release-channels parity flow (single-channel dispatch with +# requirements/version checks, release creation, Loop notify) now lives in +# .gitlab/ci/jobs/release-channels.yml (prod:* jobs, manual Run pipeline). +# This tag-push chain and that flow coexist because the parity flow is gated +# to CI_PIPELINE_SOURCE == "web". deploy_to_prod_alpha: stage: deploy_prod_alpha @@ -36,10 +45,14 @@ deploy_to_prod_alpha: - .dual_registry_login parallel: matrix: - - EDITION: - - ce - - ee - - fe + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE deploy_to_prod_beta: stage: deploy_prod_beta @@ -52,10 +65,14 @@ deploy_to_prod_beta: - .dual_registry_login parallel: matrix: - - EDITION: - - ce - - ee - - fe + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE deploy_to_prod_ea: stage: deploy_prod_ea @@ -68,10 +85,14 @@ deploy_to_prod_ea: - .dual_registry_login parallel: matrix: - - EDITION: - - ce - - ee - - fe + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE deploy_to_prod_stable: stage: deploy_prod_stable @@ -84,10 +105,14 @@ deploy_to_prod_stable: - .dual_registry_login parallel: matrix: - - EDITION: - - ce - - ee - - fe + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE deploy_to_prod_rock_solid: stage: deploy_prod_rock_solid @@ -100,7 +125,11 @@ deploy_to_prod_rock_solid: - .dual_registry_login parallel: matrix: - - EDITION: - - ce - - ee - - fe + - EDITION: ce + MODULE_EDITION: CE + - EDITION: ee + MODULE_EDITION: EE + - EDITION: se-plus + MODULE_EDITION: EE + - EDITION: fe + MODULE_EDITION: EE diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml new file mode 100644 index 0000000000..4fb6cf924b --- /dev/null +++ b/.gitlab/ci/jobs/release-channels.yml @@ -0,0 +1,445 @@ +# Prod release-channels parity with GitHub. +# +# Port of .github/workflows/release_module_release-channels.yml: a manual, +# single-channel/single-tag release dispatch with on-demand build, deploy, +# version verification, GitLab release creation, and a Loop notification. +# +# Trigger: CI/CD -> Pipelines -> Run pipeline, on the tag $RELEASE_TAG +# (CI_PIPELINE_SOURCE == "web"). These jobs run ONLY on such manual web +# pipelines, so they never collide with the tag-push flow +# (build_prod / deploy_to_prod_* in build-prod.yml / deploy-prod.yml). +# +# Run pipeline UI variables (all optional except RELEASE_TAG): +# RELEASE_CHANNEL alpha|beta|early-access|stable|rock-solid (default alpha) +# RELEASE_TAG module tag like v1.21.1 (required, ^v\d+\.\d+\.\d+$) +# EDITION_CE "true"/"false" (default false) — deploy CE +# EDITION_EE "true"/"false" (default false) — deploy EE + SE-Plus + FE +# ENABLE_BUILD "true"/"false" (default true) — run build before deploy +# CHECK_ONLY "true"/"false" (default false) — verify only, skip build/deploy +# SKIP_REQUIREMENTS_CHECK "true"/"false" (default false) +# RELEASE_TO_GITLAB "true"/"false" (default true) — create GitLab release +# SEND_RESULTS_TO_LOOP "true"/"false" (default true) — post status to Loop +# +# Parity notes vs GitHub: +# - Release creation targets GitLab Releases (POST /projects/:id/releases) +# instead of GitHub Releases, because development is moving to GitLab. +# The notes source is preserved: the merged changelog MR +# (label:changelog, milestone:$RELEASE_TAG) description. +# - Run the pipeline ON the tag $RELEASE_TAG (select it as the Run pipeline +# ref). The checkout is then already at the tag, so the upstream Setup +# before_script (werf ci-env / registry login) keeps working untouched. +# prod:print-vars enforces RELEASE_TAG == CI_COMMIT_TAG so a mismatch +# fails fast instead of building the wrong ref. GitHub let `tag` differ +# from the workflow ref (it checked out ref:inputs.tag); GitLab cannot +# do that without overriding the inherited Setup before_script, which +# the migration plan forbids. +# - Per-(channel,tag) concurrency uses resource_group (a mutex); GitLab has +# no cancel-in-progress equivalent, so concurrent same-(channel,tag) +# dispatches serialize instead of cancelling. +# - SE-Plus / FE builds `needs: prod:build:ee` to mirror the GitHub +# job-SE-Plus/job-FE cascade; ce/ee build in parallel. + +variables: + RELEASE_CHANNEL: + value: "alpha" + options: ["alpha", "beta", "early-access", "stable", "rock-solid"] + description: "Release channel to deploy" + RELEASE_TAG: + value: "" + description: "Module tag like v1.21.1 (required, ^v\\d+\\.\\d+\\.\\d+$). Run the pipeline on this tag." + EDITION_CE: + value: "false" + options: ["true", "false"] + description: "Build+deploy CE edition" + EDITION_EE: + value: "false" + options: ["true", "false"] + description: "Build+deploy EE + SE-Plus + FE editions" + ENABLE_BUILD: + value: "true" + options: ["true", "false"] + description: "Run build before deploy (false = deploy only, image already in registry)" + CHECK_ONLY: + value: "false" + options: ["true", "false"] + description: "Skip build/deploy; only verify the version on the release channel" + SKIP_REQUIREMENTS_CHECK: + value: "false" + options: ["true", "false"] + description: "Skip the Deckhouse version requirements check before build" + RELEASE_TO_GITLAB: + value: "true" + options: ["true", "false"] + description: "Create a GitLab release from the merged changelog MR" + SEND_RESULTS_TO_LOOP: + value: "true" + options: ["true", "false"] + description: "Send the release result summary to Loop via webhook" + +# Common rule: only on manual web pipelines with a valid tag. +.prod_release_rules: + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ + - when: never + +# --------------------------------------------------------------------------- +# 1. print-vars + tag/ref validation +# --------------------------------------------------------------------------- +prod:print-vars: + stage: info + extends: .prod_release_rules + variables: + MODULES_MODULE_TAG: "${RELEASE_TAG}" + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash + script: + - | + set -e + echo "MODULES_REGISTRY=${PROD_REGISTRY}" + echo "MODULES_MODULE_SOURCE=${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}//modules" + echo "MODULES_MODULE_NAME=${MODULES_MODULE_NAME}" + echo "RELEASE_CHANNEL=${RELEASE_CHANNEL}" + echo "RELEASE_TAG=${RELEASE_TAG}" + echo "EDITION_CE=${EDITION_CE} EDITION_EE=${EDITION_EE}" + echo "ENABLE_BUILD=${ENABLE_BUILD} CHECK_ONLY=${CHECK_ONLY}" + echo "SKIP_REQUIREMENTS_CHECK=${SKIP_REQUIREMENTS_CHECK}" + echo "RELEASE_TO_GITLAB=${RELEASE_TO_GITLAB} SEND_RESULTS_TO_LOOP=${SEND_RESULTS_TO_LOOP}" + echo "${RELEASE_TAG}" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + || { echo "Error: invalid tag format '${RELEASE_TAG}'. Use vX.Y.Z"; exit 1; } + # The pipeline must run on the tag itself so the checkout is at the tag + # and the upstream Setup before_script works unchanged. + if [ "${CI_COMMIT_TAG:-}" != "${RELEASE_TAG}" ]; then + echo "Error: RELEASE_TAG (${RELEASE_TAG}) must equal the pipeline ref tag" \ + "(CI_COMMIT_TAG=${CI_COMMIT_TAG:-}). Run the pipeline on the tag." + exit 1 + fi + +# --------------------------------------------------------------------------- +# 2. check-requirements-before-build +# --------------------------------------------------------------------------- +prod:check-requirements: + stage: prod_check + extends: + - .prod_release_rules + - .prod_read_registry + needs: ["prod:print-vars"] + variables: + MODULES_MODULE_TAG: "${RELEASE_TAG}" + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash go task curl jq + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $CHECK_ONLY != "true" && $SKIP_REQUIREMENTS_CHECK != "true" && ($EDITION_CE == "true" || $EDITION_EE == "true") + - when: never + script: + - CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" task -d tools/moduleversions check:requirements + +# --------------------------------------------------------------------------- +# 3+4. build (toggleable) + deploy, per edition +# --------------------------------------------------------------------------- +# Deploy step (crane copy :tag -> :channel). Separate from .local_deploy so +# the job can land in the prod_deploy stage. +.prod_deploy_script: + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh crane + script: + - | + REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release" + IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" + IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy "${IMAGE_SRC}" "${IMAGE_DST}" + +# --- CE --- +prod:build:ce: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ce" + extends: + - .prod_release_rules + - .local_build + - .prod_vars + - .dual_registry_login + needs: ["prod:check-requirements"] + variables: + EDITION: ce + MODULE_EDITION: CE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_CE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:ce: + stage: prod_deploy + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ce" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .dual_registry_login + needs: + - job: prod:check-requirements + optional: true + - job: prod:build:ce + optional: true + variables: + EDITION: ce + MODULE_EDITION: CE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_CE == "true" && $CHECK_ONLY != "true" + - when: never + +# --- EE --- +prod:build:ee: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ee" + extends: + - .prod_release_rules + - .local_build + - .prod_vars + - .dual_registry_login + needs: ["prod:check-requirements"] + variables: + EDITION: ee + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:ee: + stage: prod_deploy + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ee" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .dual_registry_login + needs: + - job: prod:check-requirements + optional: true + - job: prod:build:ee + optional: true + variables: + EDITION: ee + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" + - when: never + +# --- SE-Plus (needs EE build, mirrors GH job-SE-Plus) --- +prod:build:se-plus: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:se-plus" + extends: + - .prod_release_rules + - .local_build + - .prod_vars + - .dual_registry_login + needs: ["prod:build:ee"] + variables: + EDITION: se-plus + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:se-plus: + stage: prod_deploy + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:se-plus" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .dual_registry_login + needs: + - job: prod:build:se-plus + optional: true + variables: + EDITION: se-plus + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" + - when: never + +# --- FE (needs EE build, mirrors GH job-FE) --- +prod:build:fe: + stage: build + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:fe" + extends: + - .prod_release_rules + - .local_build + - .prod_vars + - .dual_registry_login + needs: ["prod:build:ee"] + variables: + EDITION: fe + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" && $ENABLE_BUILD == "true" + - when: never + +prod:deploy:fe: + stage: prod_deploy + resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:fe" + extends: + - .prod_release_rules + - .prod_deploy_script + - .prod_vars + - .dual_registry_login + needs: + - job: prod:build:fe + optional: true + variables: + EDITION: fe + MODULE_EDITION: EE + MODULES_MODULE_TAG: "${RELEASE_TAG}" + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $EDITION_EE == "true" && $CHECK_ONLY != "true" + - when: never + +# --------------------------------------------------------------------------- +# 5. check-version-on-release-channel (registry / releases / documentation) +# --------------------------------------------------------------------------- +prod:check-version: + stage: prod_verify + extends: + - .prod_release_rules + - .prod_read_registry + needs: + - job: prod:deploy:ce + optional: true + - job: prod:deploy:ee + optional: true + - job: prod:deploy:se-plus + optional: true + - job: prod:deploy:fe + optional: true + parallel: + matrix: + - CHECK: [registry, releases, documentation] + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash go task crane curl jq + script: + - | + set -euo pipefail + if [ "$CHECK_ONLY" != "true" ]; then + echo "Waiting for the site to update (versions usually update within 5 minutes)..." + sleep 300 + fi + case "$CHECK" in + registry) + CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" task -d tools/moduleversions check:registry + ;; + releases) + echo "Checking the version is deployed on the releases site (10 attempts, 60s apart)" + CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" COUNT=10 task -d tools/moduleversions check:releases + ;; + documentation) + echo "Checking the version is deployed on the docs site (5 attempts, 60s apart)" + CHANNEL="${RELEASE_CHANNEL}" VERSION="${RELEASE_TAG}" COUNT=5 task -d tools/moduleversions check:docs + ;; + esac + +# --------------------------------------------------------------------------- +# 6. create-gitlab-release (was create-github-release) +# --------------------------------------------------------------------------- +prod:create-gitlab-release: + stage: prod_release + extends: .prod_release_rules + needs: + - job: prod:deploy:ce + optional: true + - job: prod:deploy:ee + optional: true + - job: prod:deploy:se-plus + optional: true + - job: prod:deploy:fe + optional: true + - job: prod:check-version + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $CHECK_ONLY != "true" && $RELEASE_TO_GITLAB == "true" && ($EDITION_CE == "true" || $EDITION_EE == "true") + - when: never + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - | + set -euo pipefail + source .gitlab/ci/scripts/bash/lib/api.sh + TAG="${RELEASE_TAG}" + + # Skip if a release already exists. + if api GET "/projects/${CI_PROJECT_ID}/releases/${TAG}" >/dev/null 2>&1; then + echo "GitLab release already exists for ${TAG}, skipping" + { + echo "GH_RELEASE_STATUS=skipped" + echo "GH_RELEASE_REASON=GitLab release already exists for ${TAG}" + } > release.env + exit 0 + fi + + # Find the single merged changelog MR for this milestone (label:changelog). + MRS=$(api GET "/projects/${CI_PROJECT_ID}/merge_requests?state=merged&labels=changelog&milestone=${TAG}") + COUNT=$(echo "$MRS" | jq 'length') + echo "Found ${COUNT} merged changelog MR(s) for milestone ${TAG}" + if [ "$COUNT" -ne 1 ]; then + echo "Expected exactly one merged changelog MR for milestone ${TAG}, found ${COUNT}" >&2 + exit 1 + fi + + PR_BODY=$(echo "$MRS" | jq -r '.[0].description') + PR_IID=$(echo "$MRS" | jq -r '.[0].iid') + PR_URL=$(echo "$MRS" | jq -r '.[0].web_url') + if [ -z "${PR_BODY//[[:space:]]/}" ]; then + echo "Changelog MR !${PR_IID} description is empty" >&2 + exit 1 + fi + + # Create the GitLab release from the changelog MR description. + api POST "/projects/${CI_PROJECT_ID}/releases" \ + --data "$(jq -n --arg tag "$TAG" --arg name "$TAG" --arg desc "$PR_BODY" \ + '{tag_name: $tag, name: $name, description: $desc}')" + + echo "GitLab release created for ${TAG}" + { + echo "GH_RELEASE_STATUS=created" + echo "GH_RELEASE_URL=${CI_PROJECT_URL}/-/releases/${TAG}" + echo "GH_RELEASE_PR_IID=${PR_IID}" + echo "GH_RELEASE_PR_URL=${PR_URL}" + echo "GH_RELEASE_REASON=GitLab release created from changelog MR !${PR_IID}" + } > release.env + artifacts: + reports: + dotenv: release.env + +# --------------------------------------------------------------------------- +# 7. send-release-results-to-loop +# --------------------------------------------------------------------------- +prod:notify-loop: + stage: notify + extends: .prod_release_rules + needs: + - job: prod:deploy:ce + optional: true + - job: prod:deploy:ee + optional: true + - job: prod:deploy:se-plus + optional: true + - job: prod:deploy:fe + optional: true + - job: prod:check-version + optional: true + - job: prod:create-gitlab-release + optional: true + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $SEND_RESULTS_TO_LOOP == "true" + when: on_success + - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ && $SEND_RESULTS_TO_LOOP == "true" + when: on_failure + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq + script: + - bash .gitlab/ci/scripts/bash/notify-release-loop.sh diff --git a/.gitlab/ci/scripts/bash/notify-release-loop.sh b/.gitlab/ci/scripts/bash/notify-release-loop.sh new file mode 100755 index 0000000000..c8b230119a --- /dev/null +++ b/.gitlab/ci/scripts/bash/notify-release-loop.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Post a markdown release-status summary to the Loop chat webhook. +# +# Port of the GitHub Actions job `send-release-results-to-loop` +# (.github/workflows/release_module_release-channels.yml). GitLab does not +# inject needs.*.result into the job environment, so this script queries the +# pipeline jobs API once to collect each edition/check/release job status and +# builds the same status table the GH job produced. +# +# Inputs (GitLab predefined + dotenv artifact): +# CI_API_V4_URL, CI_PROJECT_ID, CI_PIPELINE_ID, CI_PIPELINE_URL, +# GITLAB_API_TOKEN, RELEASE_TAG, RELEASE_CHANNEL, EDITION_CE, EDITION_EE, +# CHECK_ONLY, RELEASE_TO_GITLAB, SEND_RESULTS_TO_LOOP, LOOP_WEBHOOK_URL, +# release.env (GH_RELEASE_STATUS, GH_RELEASE_URL from prod:create-gitlab-release). + +# shellcheck disable=SC2154 # CI_* and GITLAB_API_TOKEN are injected by the GitLab Runner at job runtime. + +set -euo pipefail + +source .gitlab/ci/scripts/bash/lib/api.sh + +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_PIPELINE_ID \ + RELEASE_TAG RELEASE_CHANNEL LOOP_WEBHOOK_URL + +# Fetch this pipeline's jobs once. +JOBS_JSON=$(api GET "/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100") + +job_status() { + # $1 = job name. Echo the GitLab status of the first matching job, or "". + echo "$JOBS_JSON" | jq -r --arg n "$1" '[.[] | select(.name == $n)][0].status // ""' +} + +# Map a GitLab job status to a GitHub-style result word for emoji selection. +map_result() { + case "$1" in + success) echo "success" ;; + failed) echo "failure" ;; + canceled|cancelled) echo "cancelled" ;; + skipped|manual) echo "skipped" ;; + running|pending|created|waiting_for_resource|preparing) echo "running" ;; + *) echo "unknown" ;; + esac +} + +status_emoji() { + case "$1" in + success) echo ":white_check_mark:" ;; + failure) echo ":x:" ;; + cancelled) echo ":warning:" ;; + skipped) echo ":fast_forward:" ;; + running) echo ":hourglass:" ;; + *) echo ":grey_question:" ;; + esac +} + +export TZ="Europe/Moscow" +DATE=$(date +"%Y-%m-%d %H:%M:%S UTC+03:00") +RUN_URL="${CI_PIPELINE_URL}" + +CE_RESULT=$(map_result "$(job_status prod:deploy:ce)") +EE_RESULT=$(map_result "$(job_status prod:deploy:ee)") +SE_PLUS_RESULT=$(map_result "$(job_status prod:deploy:se-plus)") +FE_RESULT=$(map_result "$(job_status prod:deploy:fe)") +CHECK_RESULT=$(map_result "$(job_status prod:check-version)") + +# Load the release creation dotenv artifact if present. +GH_RELEASE_STATUS="" +# shellcheck disable=SC1091 +[ -f release.env ] && . release.env + +HEADER_ROW="| Edition |" +STATUS_ROW="| Status |" +if [ "${EDITION_CE:-false}" = "true" ]; then + HEADER_ROW+=" CE |" + STATUS_ROW+=" $(status_emoji "${CE_RESULT}") |" +fi +if [ "${EDITION_EE:-false}" = "true" ]; then + HEADER_ROW+=" EE | SE+ | FE |" + STATUS_ROW+=" $(status_emoji "${EE_RESULT}") | $(status_emoji "${SE_PLUS_RESULT}") | $(status_emoji "${FE_RESULT}") |" +fi +HEADER_ROW+=" Check |" +STATUS_ROW+=" $(status_emoji "${CHECK_RESULT}") |" +if [ "${RELEASE_TO_GITLAB:-true}" = "true" ] && [ "${CHECK_ONLY:-false}" != "true" ]; then + HEADER_ROW+=" GitLab Release |" + case "${GH_RELEASE_STATUS}" in + created) STATUS_ROW+=" :white_check_mark: |" ;; + skipped) STATUS_ROW+=" :fast_forward: |" ;; + *) STATUS_ROW+=" :x: |" ;; + esac +fi + +# Build the markdown separator row matching the header column count. +COL_COUNT=$(echo "${HEADER_ROW}" | tr -cd '|' | wc -c) +COL_COUNT=$((COL_COUNT - 1)) +SEP="|" +i=0 +while [ "${i}" -lt "${COL_COUNT}" ]; do + SEP+="---|" + i=$((i + 1)) +done + +SUMMARY="## :dvp: **DVP | Release ${RELEASE_TAG} to ${RELEASE_CHANNEL}**\\n\\n" +SUMMARY+="Date: ${DATE}\\n" +SUMMARY+="[:link: GitLab CI Pipeline](${RUN_URL})\\n\\n" +SUMMARY+="${HEADER_ROW}\\n${SEP}\\n${STATUS_ROW}\\n" + +echo -e "${SUMMARY}" + +curl --silent --show-error --fail --request POST \ + --header "Content-Type: application/json" \ + --data "{\"text\": \"${SUMMARY}\"}" \ + "${LOOP_WEBHOOK_URL}" diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index 19a550679e..27121ca5e0 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -18,7 +18,11 @@ # gitleaks — upstream hidden template stage; visible jobs override it # deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid # deploy_prod_alpha / beta / ea / stable / rock_solid -# — sequential prod release-channel deploy chain +# — sequential prod release-channel deploy chain (tag-push flow) +# prod_check — requirements check before release-channel dispatch (m9e.6.3) +# prod_deploy — single-channel/edition deploy from Run pipeline (m9e.6.3) +# prod_verify — version-on-release-channel check matrix (m9e.6.3) +# prod_release — GitLab release creation (m9e.6.3) # notify — release-results-to-loop and similar fan-out # cleanup — scheduled registry cleanup # @@ -42,5 +46,9 @@ stages: - deploy_prod_ea - deploy_prod_stable - deploy_prod_rock_solid + - prod_check + - prod_deploy + - prod_verify + - prod_release - notify - cleanup diff --git a/.gitlab/ci/templates/prod_manual.yml b/.gitlab/ci/templates/prod_manual.yml index 5f42a4c9ac..b8aa2c0ef0 100644 --- a/.gitlab/ci/templates/prod_manual.yml +++ b/.gitlab/ci/templates/prod_manual.yml @@ -10,6 +10,12 @@ # hardcoded matrix (RELEASE_CHANNEL x EDITION) in deploy-prod.yml. Once we # decide whether to expose that flexibility (likely via "Run pipeline" # variables), this template will gain extra variables. +# +# The single-channel dispatch with Run-pipeline inputs (channel, editions, +# tag, enableBuild, check_only, release_to_gitlab, send_results_to_loop, +# requirements/version checks) is now implemented in +# .gitlab/ci/jobs/release-channels.yml (prod:* jobs). This template stays +# as the tag-push chained-deploy context. .prod_manual: variables: diff --git a/.gitlab/ci/templates/prod_vars.yml b/.gitlab/ci/templates/prod_vars.yml index 225768d797..cfd8029f52 100644 --- a/.gitlab/ci/templates/prod_vars.yml +++ b/.gitlab/ci/templates/prod_vars.yml @@ -10,6 +10,26 @@ # PROD_MODULE_SOURCE_NAME, and the matrix-driven EDITION variable so that # ce/ee/se-plus/fe editions can each deploy into the right subpath without # duplicating templates. +# +# MODULE_EDITION is NOT set here: it is provided per edition by the +# parallel:matrix in build-prod.yml / deploy-prod.yml (CE for ce, EE for +# ee/se-plus/fe), matching .github/workflows/release_module_build-and-registration.yml. +# .werf/consts.yaml defaults MODULE_EDITION to EE, so it MUST be set by the +# matrix — otherwise the ce prod image is built/deployed as EE. + +# Read-only registry bundle for prod verification jobs +# (prod:check-requirements, prod:check-version in release-channels.yml). +# +# Sets MODULES_REGISTRY_* to PROD_READ_REGISTRY_* so the upstream Setup +# `werf cr login` authenticates against the read-only registry used by the +# tools/moduleversions checks. NOTE: the Go/bash tools hardcode +# registry.deckhouse.io; PROD_READ_REGISTRY must resolve to that host for +# crane auth to apply (see tools/moduleversions Taskfile.dist.yaml). +.prod_read_registry: + variables: + MODULES_REGISTRY: "${PROD_READ_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${PROD_READ_REGISTRY_USER}" + MODULES_REGISTRY_PASSWORD: "${PROD_READ_REGISTRY_PASSWORD}" .prod_vars: variables: From 001059a5847c1c86493e56778fff2f9d02c17e9c Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 18:42:34 +0300 Subject: [PATCH 30/60] fix(ci): make prod:check-requirements an optional need for prod:build jobs GitLab CI lint rejects a hard 'needs' on a job (prod:check-requirements) that can be absent from the pipeline (its rule gates it out when CHECK_ONLY or SKIP_REQUIREMENTS_CHECK or no edition selected). Mark the need optional on prod:build:ce/ee so the DAG validates in all dispatch modes. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/release-channels.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml index 4fb6cf924b..fb45c2d76d 100644 --- a/.gitlab/ci/jobs/release-channels.yml +++ b/.gitlab/ci/jobs/release-channels.yml @@ -158,7 +158,9 @@ prod:build:ce: - .local_build - .prod_vars - .dual_registry_login - needs: ["prod:check-requirements"] + needs: + - job: prod:check-requirements + optional: true variables: EDITION: ce MODULE_EDITION: CE @@ -197,7 +199,9 @@ prod:build:ee: - .local_build - .prod_vars - .dual_registry_login - needs: ["prod:check-requirements"] + needs: + - job: prod:check-requirements + optional: true variables: EDITION: ee MODULE_EDITION: EE From 882a5bf17676a340086c12701c3f07abf5d62406 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 18:45:02 +0300 Subject: [PATCH 31/60] fix(ci): move prod_check stage before build for prod:build needs DAG prod:build:* (stage build) needs prod:check-requirements (stage prod_check), so prod_check must precede build in the stages list, otherwise GitLab CI lint rejects the need as 'not defined in current or prior stages'. Signed-off-by: Nikita Korolev --- .gitlab/ci/stages.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index 27121ca5e0..82a433007b 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -19,7 +19,8 @@ # deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid # deploy_prod_alpha / beta / ea / stable / rock_solid # — sequential prod release-channel deploy chain (tag-push flow) -# prod_check — requirements check before release-channel dispatch (m9e.6.3) +# prod_check — requirements check before release-channel dispatch (m9e.6.3). +# Runs BEFORE build so prod:build:* can need it. # prod_deploy — single-channel/edition deploy from Run pipeline (m9e.6.3) # prod_verify — version-on-release-channel check matrix (m9e.6.3) # prod_release — GitLab release creation (m9e.6.3) @@ -37,6 +38,7 @@ stages: - info - lint - test + - prod_check - build - scan - gitleaks @@ -46,7 +48,6 @@ stages: - deploy_prod_ea - deploy_prod_stable - deploy_prod_rock_solid - - prod_check - prod_deploy - prod_verify - prod_release From 2d9ab37b29941fef5929a0a17b1e8ee291d8e652 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 24 Jun 2026 18:57:01 +0300 Subject: [PATCH 32/60] fix(ci): add license headers and prettier-format lint:dmt/lint:yaml failures lint:dmt failed: no license header in gitlab-ci-lint.sh and set-vars.sh. lint:yaml (prettier) failed: info.yml and translate-changelog.yml had single-quote if-rules prettier wants double-quoted. Both are pre-existing migration files now blocking CI. - task dev:addlicense on .gitlab/ci/scripts/bash (added headers to the 2 scripts; all others already had them). - prettier -w on the 2 yaml jobs (single if-rule quote style each). Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/info.yml | 2 +- .gitlab/ci/jobs/translate-changelog.yml | 2 +- .gitlab/ci/scripts/bash/gitlab-ci-lint.sh | 14 ++++++++++++++ .gitlab/ci/scripts/bash/set-vars.sh | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/jobs/info.yml b/.gitlab/ci/jobs/info.yml index 80a00aad7f..7158d7966c 100644 --- a/.gitlab/ci/jobs/info.yml +++ b/.gitlab/ci/jobs/info.yml @@ -54,7 +54,7 @@ set_vars: rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: always - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" when: always - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-dev.*$/' when: always diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index 8542d32a22..4d934e83f7 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -67,7 +67,7 @@ translate:changelog: # Match the GH workflow trigger: push to main (default branch) or # release-X.Y branches. Replaces the upstream default rule # (`$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH`) which excludes main. - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+$/' before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh git curl jq python3 diff --git a/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh index 8d854e5d81..5b8db06531 100755 --- a/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh +++ b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh @@ -1,4 +1,18 @@ #!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # .gitlab/ci/scripts/bash/gitlab-ci-lint.sh # diff --git a/.gitlab/ci/scripts/bash/set-vars.sh b/.gitlab/ci/scripts/bash/set-vars.sh index aafc3b23da..4fa4a32ff1 100755 --- a/.gitlab/ci/scripts/bash/set-vars.sh +++ b/.gitlab/ci/scripts/bash/set-vars.sh @@ -1,4 +1,18 @@ #!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # set-vars.sh — derives per-pipeline variables for downstream jobs. # From 6bd050967e87b821abb565b0fd1c32a88d30fe61 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 00:41:02 +0300 Subject: [PATCH 33/60] fix(ci): run unit tests on every MR push via needs DAG Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab/ci/jobs/test.yml b/.gitlab/ci/jobs/test.yml index 988e1aeb77..ed08efbd64 100644 --- a/.gitlab/ci/jobs/test.yml +++ b/.gitlab/ci/jobs/test.yml @@ -11,8 +11,15 @@ # All test jobs extend .dev so they only run on MR pipelines with the right # MODULES_MODULE_TAG/registry context. +# `needs: []` mirrors the GitHub dev_module_build.yml `test` job, which has no +# `needs` and runs on every PR push (pull_request: synchronize) independently of +# lint/build. Without it the test jobs sit in the `test` stage waiting for the +# whole `lint` stage to succeed, which a still-unplayed blocking manual job +# (e.g. changelog:milestone) can stall indefinitely. DAG mode lets unit tests +# start immediately on every MR pipeline. test:virtualization-controller: stage: test + needs: [] script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task - task virtualization-controller:init @@ -22,6 +29,7 @@ test:virtualization-controller: test:hooks: stage: test + needs: [] script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh task - task gohooks:test From d0421d98f79e063a5cf3b536f667269f93fb1287 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 00:41:02 +0300 Subject: [PATCH 34/60] fix(ci): make manual changelog/backport jobs non-blocking on MR Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/backport.yml | 7 +++++++ .gitlab/ci/jobs/changelog.yml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index 2a4117c97f..3ea1eb4574 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -34,15 +34,22 @@ backport: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash git curl jq ssh-agent ssh-add script: - bash .gitlab/ci/scripts/bash/backport.sh + # allow_failure: true on every rule: `when: manual` inside `rules:` defaults + # to allow_failure: false, so on an MR carrying a backport-release-X.Y label + # this unplayed manual job would block the whole MR pipeline (test/build/...). + # Backport is an opt-in side action, so it must never gate the pipeline. rules: # Mode 1: explicit manual run with TARGET_BRANCH provided via UI. - if: $TARGET_BRANCH when: manual + allow_failure: true # Mode 2: MR with a backport-release-X.Y label. GitLab does NOT auto-run # pipelines on label change; user has to press "Run pipeline" on the MR # (TODO: webhook-listener per migration plan §7). - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /backport-release-[0-9]+\.[0-9]+/ when: manual + allow_failure: true # Mode 3: scheduled backport sweep (TODO: future automation). - if: $CI_PIPELINE_SOURCE == "schedule" when: manual + allow_failure: true diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index 06b7e81b08..5c91ffd39c 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -47,10 +47,17 @@ changelog:milestone: CHANGELOG_BASE_BRANCH: "main" rules: # Mode 1: manual with optional MILESTONE_TITLE / OPEN_CHANGELOG_MR vars. + # allow_failure: true is REQUIRED here: `when: manual` inside `rules:` + # defaults to allow_failure: false, which makes the (unplayed) manual job + # block every later stage of the MR pipeline (test/build/...). GitHub ran + # changelog generation in a separate workflow, fully decoupled from the + # build/test pipeline, so this manual job must be non-blocking/optional. - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual + allow_failure: true - if: $CI_PIPELINE_SOURCE == "web" when: manual + allow_failure: true # Mode 2: scheduled daily run. We intentionally do NOT auto-open the MR on # schedule; set OPEN_CHANGELOG_MR=true in the schedule definition if you # want MRs created. From 487776658574b5565b08f310c3c71096e8796dbf Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 00:41:02 +0300 Subject: [PATCH 35/60] fix(ci): build dev image on merge into release-* branches Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 1 + .gitlab/ci/jobs/build-dev.yml | 17 +++++++++++++++++ .gitlab/ci/jobs/info.yml | 10 +++++++--- .gitlab/ci/templates/release.yml | 21 +++++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 .gitlab/ci/templates/release.yml diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 66d62fe921..774b7acb3e 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -62,6 +62,7 @@ include: - local: ".gitlab/ci/templates/dev.yml" - local: ".gitlab/ci/templates/dev_tags.yml" - local: ".gitlab/ci/templates/main.yml" + - local: ".gitlab/ci/templates/release.yml" - local: ".gitlab/ci/templates/prod_manual.yml" - local: ".gitlab/ci/templates/prod_always.yml" - local: ".gitlab/ci/templates/info.yml" diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml index b2510e54a7..7f958bf8b0 100644 --- a/.gitlab/ci/jobs/build-dev.yml +++ b/.gitlab/ci/jobs/build-dev.yml @@ -68,3 +68,20 @@ build_main: extends: - .local_build - .main + +# build_release mirrors GitHub dev_module_build.yml `push: [release-*]`: a +# squash-merge into a release-X.Y branch rebuilds the dev image tagged with the +# branch name. Without this job GitLab created a pipeline on the merge push but +# built nothing for release branches (only .dev/.main/.dev_tags had build jobs). +# interruptible: true so a newer release-branch push cancels an older build. +build_release: + stage: build + interruptible: true + needs: + - job: set_vars + artifacts: true + variables: + WERF_VIRTUAL_MERGE: "0" + extends: + - .local_build + - .release diff --git a/.gitlab/ci/jobs/info.yml b/.gitlab/ci/jobs/info.yml index 7158d7966c..6e4e6bbaac 100644 --- a/.gitlab/ci/jobs/info.yml +++ b/.gitlab/ci/jobs/info.yml @@ -37,9 +37,9 @@ show_main_manifest: # each build job still derives its tag from .dev / .dev_tags / .main. # # Rules mirror the union of the dev build jobs' rules (.dev / .dev_tags / -# .main) so set_vars always runs when any of them runs and the `needs:` edge -# is satisfied. Specifying an explicit `needs:` on the build jobs also keeps -# them in DAG mode, so the svace:set-vars dotenv still does not leak in. +# .main / .release) so set_vars always runs when any of them runs and the +# `needs:` edge is satisfied. Specifying an explicit `needs:` on the build jobs +# also keeps them in DAG mode, so the svace:set-vars dotenv still does not leak in. set_vars: stage: info @@ -56,6 +56,10 @@ set_vars: when: always - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" when: always + # Release-branch pushes (squash-merge into release-X.Y) so build_release + # can consume the set_vars dotenv (RELEASE_IN_DEV=true, MODULE_EDITION). + - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+/' + when: always - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+-dev.*$/' when: always - when: never diff --git a/.gitlab/ci/templates/release.yml b/.gitlab/ci/templates/release.yml new file mode 100644 index 0000000000..9be53649a9 --- /dev/null +++ b/.gitlab/ci/templates/release.yml @@ -0,0 +1,21 @@ +# DEV pipeline context: release-branch builds. +# +# Mirrors the GitHub dev_module_build.yml `push: [release-*]` trigger. Code +# reaches release-X.Y branches only through squash-and-merge of an MR, so the +# merge IS the push that must rebuild the dev image for the integrated release +# branch. The image is tagged with the branch name (release-X.Y), matching the +# GitHub set_vars logic; set-vars.sh sets RELEASE_IN_DEV=true for the same +# branches. +# +# Counterpart of .main (default-branch build, tag v0.0.0-main) and .dev (MR +# build, tag mr). + +.release: + variables: + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + extends: + - .dev_vars + rules: + - if: '$CI_COMMIT_BRANCH =~ /^release-[0-9]+\.[0-9]+/' + when: always + - when: never From 899b0769d2b7b8445c6d9177f3d27db30748f235 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 12:12:40 +0300 Subject: [PATCH 36/60] fix(ci): run lint:yaml/helm-templates/gens-files/gitlab-ci on release-* branches Add a release-* branch rule (CI_COMMIT_BRANCH =~ /^release-/) to the four lint jobs that previously only had a 'main' rule: lint:yaml, lint:helm-templates, check:gens-files, lint:gitlab-ci. This makes lint coverage on release-branch pipelines consistent with main and with no-cyrillic/doc-changes/shellcheck/go, which already ran on release-* branches. rules:changes semantics on branch pushes are identical to main (compares against the previous SHA on the branch), so mirroring the main rule is safe. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/lint-validate.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 2fe460dca2..f8fb1aa1bf 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -122,6 +122,13 @@ lint:yaml: - "**/*.yml" - ".gitlab-ci.yml" - ".gitlab/**/*" + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + changes: + paths: + - "**/*.yaml" + - "**/*.yml" + - ".gitlab-ci.yml" + - ".gitlab/**/*" # --------------------------------------------------------------------------- # go (golangci-lint across every Go module) @@ -261,6 +268,16 @@ lint:helm-templates: - ".helmignore" - "Chart.yaml" - "Taskfile.yaml" + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + changes: + paths: + - "crds/**/*" + - "charts/**/*" + - "tools/kubeconform/**/*" + - "templates/**/*" + - ".helmignore" + - "Chart.yaml" + - "Taskfile.yaml" # --------------------------------------------------------------------------- # check_gens_files @@ -370,6 +387,7 @@ check:gens-files: - "images/vm-route-forge/bpf/route_watcher.c" - "images/vm-route-forge/**/*" - if: '$CI_COMMIT_BRANCH == "main"' + - if: "$CI_COMMIT_BRANCH =~ /^release-/" # --------------------------------------------------------------------------- # gitlab-ci-lint @@ -401,4 +419,9 @@ lint:gitlab-ci: paths: - ".gitlab-ci.yml" - ".gitlab/**/*" + - if: "$CI_COMMIT_BRANCH =~ /^release-/" + changes: + paths: + - ".gitlab-ci.yml" + - ".gitlab/**/*" - if: '$CI_PIPELINE_SOURCE == "schedule"' From 8d00d3417c53bd275c5390d24fc7f96f53ed03ff Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 12:40:18 +0300 Subject: [PATCH 37/60] fix(ci): run test:scripts:js inside node:24 docker container The deckhouse shell runner host has no node/npm installed, so the job failed at check-runner-tools.sh. Run npm install + npm test inside the official node:${NODE_VERSION} image via docker run, mirroring how check:gens-files runs the vm-route-forge leg in a container. The repo is bind-mounted at /src. Add NODE_VERSION=24 to variables.yml, matching the GH workflow test_scripts_js env. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/test-scripts-js.yml | 23 ++++++++++++++--------- .gitlab/ci/variables.yml | 5 +++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml index 0603c38532..95fbae9a50 100644 --- a/.gitlab/ci/jobs/test-scripts-js.yml +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -16,26 +16,31 @@ # # Ports the GitHub Actions `test_scripts_js` job from # .github/workflows/dev_module_build.yml, which ran `npm test` in -# .github/scripts/js. The GitLab counterpart lives in .gitlab/scripts/js -# (mrs_notifier.mjs); its `npm test` runs node:test over +# .github/scripts/js with Node.js 24. The GitLab counterpart lives in +# .gitlab/scripts/js (mrs_notifier.mjs); its `npm test` runs node:test over # mrs_notifier.test.mjs, a minimal smoke test (syntax check + structural # assertions) because mrs_notifier.mjs auto-runs at import time and cannot # yet be imported by a unit test. See mrs_notifier.test.mjs for the TODO # on adding real unit tests. # -# MR-gated via .dev. The smoke test uses only Node.js built-ins, so no -# dependencies are required; `npm install` is still run to match the -# upstream job shape and to fail fast if package.json is broken. +# The shell runner host does not ship Node.js, so the job runs inside the +# official `node:${NODE_VERSION}` image via `docker run`, mirroring how +# check:gens-files runs the vm-route-forge leg in cilium/ebpf-builder. The +# repo is bind-mounted at /src and npm install/npm test execute there. # TODO: commit a package-lock.json and switch to `npm ci`. test:scripts:js: stage: test interruptible: true before_script: - - bash .gitlab/ci/scripts/bash/check-runner-tools.sh node npm + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh docker script: - - cd .gitlab/scripts/js - - npm install --no-audit --no-fund - - npm test + - | + set -e + docker run --rm --platform linux/amd64 \ + -v "${PWD}:/src" \ + -w /src/.gitlab/scripts/js \ + node:${NODE_VERSION} \ + sh -c 'npm install --no-audit --no-fund && npm test' extends: - .dev diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index 42917d31fe..028ab3d128 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -77,6 +77,11 @@ variables: # portable binaries from these versions in its before_script. HELM_VERSION: "3.16.3" JQ_VERSION: "1.7.1" + # Node.js for test:scripts:js (.gitlab/scripts/js). The runner host has no + # Node.js, so the job runs inside the official node:${NODE_VERSION} image via + # `docker run`. Matches the Node version pinned in the GH workflow + # (.github/workflows/dev_module_build.yml test_scripts_js env NODE_VERSION). + NODE_VERSION: "24" # Let the host Go toolchain auto-download the version required by go.mod # (currently go 1.25.11). Runner hosts may ship an older Go with # GOTOOLCHAIN pinned to that older version (e.g. GOTOOLCHAIN=go1.25.9), From 7855d4b105c3ca6e19d8c3b76a618cf54ac7f017 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 13:00:32 +0300 Subject: [PATCH 38/60] fix(ci): run npm tests as runner-user to avoid root-owned workspace pollution test:scripts:js ran 'docker run node:24 ... npm install' as root (default uid). node_modules/ and package-lock.json were created root-owned in the shared shell-executor workspace and not cleaned between jobs. The next checkout (git clean/checkout) in gitleaks:diff could not remove them ('Permission denied'), failing 'Getting source from Git repository' before the gitleaks script ever ran, so gitleaks.json was never produced ('no matching files' / 'Job failed: exit status 1'). Run npm under the host runner-user uid/gid (-u $(id -u):$(id -g)) with HOME=/tmp and --cache /tmp/.npm so generated files are runner-user owned and git can clean them on the next checkout. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/test-scripts-js.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml index 95fbae9a50..6d51e714d9 100644 --- a/.gitlab/ci/jobs/test-scripts-js.yml +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -37,10 +37,17 @@ test:scripts:js: script: - | set -e + # Run as the host runner-user uid/gid so node_modules and + # package-lock.json are owned by the runner-user, not root. + # Root-owned files would block `git clean` on the next checkout + # (shell executor reuses the same workspace) and fail jobs like + # gitleaks:diff before their script even starts. docker run --rm --platform linux/amd64 \ -v "${PWD}:/src" \ -w /src/.gitlab/scripts/js \ + -u "$(id -u):$(id -g)" \ + -e HOME=/tmp \ node:${NODE_VERSION} \ - sh -c 'npm install --no-audit --no-fund && npm test' + sh -c 'npm install --no-audit --no-fund --cache /tmp/.npm && npm test' extends: - .dev From b56fb6dbe9ea7697a8774b2dfa63b48278294743 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 14:25:02 +0300 Subject: [PATCH 39/60] fix(ci): isolate npm tests in container tmpdir to avoid workspace pollution Previous fix (-u runner-user) still left node_modules/package-lock.json in the shared shell-executor workspace. The real issue is deeper: root-owned leftovers from pre-fix runs already on the runner cannot be removed by runner-user, so git clean on the NEXT job's 'Getting source from Git repository' step fails with 'Permission denied' and breaks unrelated jobs (show_dev_manifest, test:build:d8v-cli, gitleaks:diff, ...) before their script runs. Mount the repo READ-ONLY and copy the JS sources to an ephemeral tmpdir inside the container where npm install/npm test run. node_modules and package-lock.json now never touch the workspace at all, so no job can be broken by leftover files regardless of who owned them. One-time manual cleanup of existing root-owned leftovers on runner H61TLxhnT is still required (sudo rm -rf .../virt-test/.gitlab/scripts/js/ {node_modules,package-lock.json}); no CI-only fix can remove root-owned files because the checkout that would run before any cleanup script already fails on them. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/test-scripts-js.yml | 33 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml index 6d51e714d9..c11884d603 100644 --- a/.gitlab/ci/jobs/test-scripts-js.yml +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -26,7 +26,10 @@ # The shell runner host does not ship Node.js, so the job runs inside the # official `node:${NODE_VERSION}` image via `docker run`, mirroring how # check:gens-files runs the vm-route-forge leg in cilium/ebpf-builder. The -# repo is bind-mounted at /src and npm install/npm test execute there. +# repo is bind-mounted READ-ONLY at /src and copied to an ephemeral tmpdir +# inside the container, so node_modules/package-lock.json never pollute the +# shared shell-executor workspace (root-owned files there break `git clean` +# on subsequent checkouts of unrelated jobs). # TODO: commit a package-lock.json and switch to `npm ci`. test:scripts:js: @@ -37,17 +40,29 @@ test:scripts:js: script: - | set -e - # Run as the host runner-user uid/gid so node_modules and - # package-lock.json are owned by the runner-user, not root. - # Root-owned files would block `git clean` on the next checkout - # (shell executor reuses the same workspace) and fail jobs like - # gitleaks:diff before their script even starts. + # Run npm in a throwaway tmpdir INSIDE the container so node_modules + # and package-lock.json never land in the shared shell-executor + # workspace. Previously npm ran directly against a read-write + # bind-mount of the repo: under the default container uid (root) it + # created root-owned node_modules/ that the runner-user could not + # remove, which broke `git clean` on the next job's checkout and + # failed unrelated jobs (show_dev_manifest, gitleaks:diff, ...) with + # "warning: failed to remove ...: Permission denied". + # + # The repo is mounted read-only at /src; the test sources are copied + # to an ephemeral tmpdir where npm install/npm test execute. docker run --rm --platform linux/amd64 \ - -v "${PWD}:/src" \ - -w /src/.gitlab/scripts/js \ + -v "${PWD}:/src:ro" \ -u "$(id -u):$(id -g)" \ -e HOME=/tmp \ node:${NODE_VERSION} \ - sh -c 'npm install --no-audit --no-fund --cache /tmp/.npm && npm test' + sh -c ' + set -e + work="$(mktemp -d)" + cp -a /src/.gitlab/scripts/js/. "$work"/ + cd "$work" + npm install --no-audit --no-fund --cache /tmp/.npm + npm test + ' extends: - .dev From 30740877cb12355622bbf202d2f2da6bc1e7e105 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 16:23:27 +0300 Subject: [PATCH 40/60] refactor(ci): single deploy_prod stage, drop dead gitleaks stage Collapse the five separate deploy_prod_alpha..rock_solid stages into one deploy_prod stage and make the per-channel prod deploys independent when:manual jobs (each needs:[build_prod] only) instead of a forced alpha->beta->ea->stable->rock-solid promotion chain. This matches GitHub release_module_release-channels.yml channel-pick semantics and removes a carried-over artifact of the old root .gitlab-ci.yml. The web Run-pipeline prod:deploy:* jobs (release-channels.yml) move from the prod_deploy stage to the same deploy_prod stage, unifying both prod deploy flows under one stage. Drop the dead gitleaks stage (every active gitleaks job runs in scan). The disabled upstream gitleaks_* override jobs get stage:scan so nothing references the removed stage at config-parse time. Signed-off-by: Nikita Korolev --- .gitlab/README.md | 2 +- .gitlab/ci/jobs/deploy-prod.yml | 34 ++++++++++++++++------------ .gitlab/ci/jobs/gitleaks.yml | 10 ++++++++ .gitlab/ci/jobs/release-channels.yml | 10 ++++---- .gitlab/ci/stages.yml | 19 +++++++--------- .gitlab/ci/templates/prod_manual.yml | 4 +++- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/.gitlab/README.md b/.gitlab/README.md index 9d2f785de6..fcc95d103d 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -252,7 +252,7 @@ web pipelines. | `prod:print-vars` | info | Echoes release inputs and enforces `RELEASE_TAG` matches the pipeline ref tag. | | `prod:check-requirements` | prod_check | `tools/moduleversions check:requirements` (Deckhouse version range). Skipped when `CHECK_ONLY` or `SKIP_REQUIREMENTS_CHECK`. | | `prod:build:` | build | Optional (`ENABLE_BUILD`) werf build per edition. `se-plus`/`fe` `needs: prod:build:ee` (GH cascade). | -| `prod:deploy:` | prod_deploy | `crane copy :$RELEASE_TAG -> :$RELEASE_CHANNEL` per selected edition. | +| `prod:deploy:` | deploy_prod | `crane copy :$RELEASE_TAG -> :$RELEASE_CHANNEL` per selected edition. | | `prod:check-version` | prod_verify | Matrix `registry`/`releases`/`documentation` via `tools/moduleversions`. | | `prod:create-gitlab-release` | prod_release | Creates a GitLab release from the merged `label:changelog` MR description (was `create-github-release`). | | `prod:notify-loop` | notify | Posts a release status table to Loop (`LOOP_WEBHOOK_URL`). | diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index 97810a11f3..be372ed59d 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -1,10 +1,16 @@ # PROD deploy jobs. # -# Parity with .github/workflows/release_module_release-channels.yml release -# channels: alpha -> beta -> early-access -> stable -> rock-solid. Each job -# waits for the previous one (and build_prod for alpha), then copies the -# built release image to the named release channel. All five are manual -# (when: manual from .prod_manual) on a vX.Y.Z tag. +# Parity with .github/workflows/release_module_release-channels.yml: the +# channel is selected by the operator, not promoted through a fixed chain. +# All five channel deploys (alpha / beta / early-access / stable / +# rock-solid) are INDEPENDENT manual jobs (when: manual from .prod_manual) +# in a single `deploy_prod` stage. Each only `needs: [build_prod]`, so any +# channel can be deployed directly after the build without first clicking +# the earlier channels. This replaces the previous forced promotion chain +# (alpha -> beta -> ea -> stable -> rock-solid across five separate stages), +# which was a carried-over artifact of the old root .gitlab-ci.yml and +# diverged from GitHub (see virtualization-m9e.5.21). Each job copies the +# built release image to its named release channel on a vX.Y.Z tag. # # The EDITION matrix mirrors build_prod (ce/ee/se-plus/fe) with the correct # MODULE_EDITION per edition, so deploy runs against the same per-edition @@ -35,7 +41,7 @@ # to CI_PIPELINE_SOURCE == "web". deploy_to_prod_alpha: - stage: deploy_prod_alpha + stage: deploy_prod variables: RELEASE_CHANNEL: alpha needs: ["build_prod"] @@ -55,10 +61,10 @@ deploy_to_prod_alpha: MODULE_EDITION: EE deploy_to_prod_beta: - stage: deploy_prod_beta + stage: deploy_prod variables: RELEASE_CHANNEL: beta - needs: ["deploy_to_prod_alpha"] + needs: ["build_prod"] extends: - .local_deploy - .prod_manual @@ -75,10 +81,10 @@ deploy_to_prod_beta: MODULE_EDITION: EE deploy_to_prod_ea: - stage: deploy_prod_ea + stage: deploy_prod variables: RELEASE_CHANNEL: early-access - needs: ["deploy_to_prod_beta"] + needs: ["build_prod"] extends: - .local_deploy - .prod_manual @@ -95,10 +101,10 @@ deploy_to_prod_ea: MODULE_EDITION: EE deploy_to_prod_stable: - stage: deploy_prod_stable + stage: deploy_prod variables: RELEASE_CHANNEL: stable - needs: ["deploy_to_prod_ea"] + needs: ["build_prod"] extends: - .local_deploy - .prod_manual @@ -115,10 +121,10 @@ deploy_to_prod_stable: MODULE_EDITION: EE deploy_to_prod_rock_solid: - stage: deploy_prod_rock_solid + stage: deploy_prod variables: RELEASE_CHANNEL: rock-solid - needs: ["deploy_to_prod_stable"] + needs: ["build_prod"] extends: - .local_deploy - .prod_manual diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index 5af74bed7d..815e683418 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -21,18 +21,28 @@ # Disable generic visible jobs from the upstream include. Keep the hidden # `.gitleaks_scan` template active for project-specific jobs below. +# +# These overrides MUST set `stage: scan`: the upstream `.gitleaks_scan` +# template declares `stage: gitleaks`, but that stage was removed from +# stages.yml (it was dead — every active gitleaks job runs in `scan`). Even +# with `when: never`, GitLab validates the stage exists at config-parse +# time, so leaving the inherited `gitleaks` stage here would break the +# pipeline. See virtualization-m9e.5.21. gitleaks_diff: extends: .gitleaks_scan + stage: scan rules: - when: never gitleaks_full_manual: extends: .gitleaks_scan + stage: scan rules: - when: never gitleaks_full_scheduled: extends: .gitleaks_scan + stage: scan rules: - when: never diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml index fb45c2d76d..4484d82ffd 100644 --- a/.gitlab/ci/jobs/release-channels.yml +++ b/.gitlab/ci/jobs/release-channels.yml @@ -137,7 +137,7 @@ prod:check-requirements: # 3+4. build (toggleable) + deploy, per edition # --------------------------------------------------------------------------- # Deploy step (crane copy :tag -> :channel). Separate from .local_deploy so -# the job can land in the prod_deploy stage. +# the job can land in the deploy_prod stage. .prod_deploy_script: before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh crane @@ -170,7 +170,7 @@ prod:build:ce: - when: never prod:deploy:ce: - stage: prod_deploy + stage: deploy_prod resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ce" extends: - .prod_release_rules @@ -211,7 +211,7 @@ prod:build:ee: - when: never prod:deploy:ee: - stage: prod_deploy + stage: deploy_prod resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ee" extends: - .prod_release_rules @@ -250,7 +250,7 @@ prod:build:se-plus: - when: never prod:deploy:se-plus: - stage: prod_deploy + stage: deploy_prod resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:se-plus" extends: - .prod_release_rules @@ -287,7 +287,7 @@ prod:build:fe: - when: never prod:deploy:fe: - stage: prod_deploy + stage: deploy_prod resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:fe" extends: - .prod_release_rules diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index 82a433007b..6b41e5cc88 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -15,13 +15,16 @@ # test — Go unit tests, hooks tests # build — werf build for dev / dev-tags / main / prod # scan — cve_scan, gitleaks, svace (owned by sibling issues) -# gitleaks — upstream hidden template stage; visible jobs override it # deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid -# deploy_prod_alpha / beta / ea / stable / rock_solid -# — sequential prod release-channel deploy chain (tag-push flow) # prod_check — requirements check before release-channel dispatch (m9e.6.3). # Runs BEFORE build so prod:build:* can need it. -# prod_deploy — single-channel/edition deploy from Run pipeline (m9e.6.3) +# deploy_prod — prod release-channel deploy. ONE stage with independent +# manual jobs per channel (tag-push flow, deploy-prod.yml) +# and per-edition deploy from Run pipeline +# (web flow, release-channels.yml). No forced +# alpha->...->rock-solid promotion chain — each channel +# deploy is an independent `when: manual` job, matching +# GitHub release_module_release-channels.yml semantics. # prod_verify — version-on-release-channel check matrix (m9e.6.3) # prod_release — GitLab release creation (m9e.6.3) # notify — release-results-to-loop and similar fan-out @@ -41,14 +44,8 @@ stages: - prod_check - build - scan - - gitleaks - deploy_dev - - deploy_prod_alpha - - deploy_prod_beta - - deploy_prod_ea - - deploy_prod_stable - - deploy_prod_rock_solid - - prod_deploy + - deploy_prod - prod_verify - prod_release - notify diff --git a/.gitlab/ci/templates/prod_manual.yml b/.gitlab/ci/templates/prod_manual.yml index b8aa2c0ef0..08226957e3 100644 --- a/.gitlab/ci/templates/prod_manual.yml +++ b/.gitlab/ci/templates/prod_manual.yml @@ -15,7 +15,9 @@ # tag, enableBuild, check_only, release_to_gitlab, send_results_to_loop, # requirements/version checks) is now implemented in # .gitlab/ci/jobs/release-channels.yml (prod:* jobs). This template stays -# as the tag-push chained-deploy context. +# as the tag-push deploy context: each channel deploy is an independent +# `when: manual` job in the single `deploy_prod` stage (no forced +# alpha->...->rock-solid promotion chain; see virtualization-m9e.5.21). .prod_manual: variables: From 836dc8d4622cbf337d733e194e1930629a10274a Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 18:34:51 +0300 Subject: [PATCH 41/60] refactor(ci): rename .local_build/.local_deploy to .base_build/.base_deploy The "local" prefix was uninformative and conceptually clashed with `include: local:`. These are base templates meant to be extended by the concrete build/deploy jobs, so `.base_build` / `.base_deploy` state intent. Renames the two anchors (templates/build.yml, templates/deploy.yml) and all 20 `extends:` references across build-dev, build-prod, precache, svace, release-channels, deploy-dev, deploy-prod, plus comments and includes.yml. No behavior change: the single `build` stage stays (dev and release builds are already separated by rules, never co-run in one pipeline). Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 2 +- .gitlab/ci/jobs/build-dev.yml | 12 ++++++------ .gitlab/ci/jobs/build-prod.yml | 2 +- .gitlab/ci/jobs/deploy-dev.yml | 2 +- .gitlab/ci/jobs/deploy-prod.yml | 12 ++++++------ .gitlab/ci/jobs/precache.yml | 6 +++--- .gitlab/ci/jobs/release-channels.yml | 10 +++++----- .gitlab/ci/jobs/svace.yml | 6 +++--- .gitlab/ci/templates/build.yml | 12 +++++++++--- .gitlab/ci/templates/deploy.yml | 13 +++++++++---- 10 files changed, 44 insertions(+), 33 deletions(-) diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 774b7acb3e..7a2a1b0bf2 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -48,7 +48,7 @@ include: # `when: manual`) that override our strict gating via # .dev / .dev_tags / .main / .prod_manual. We mirror the upstream # script bodies in .gitlab/ci/templates/{build,deploy}.yml as - # `.local_build` and `.local_deploy` and extend those instead. + # `.base_build` and `.base_deploy` and extend those instead. # --- Local structural fragments (order: stages, vars, defaults, then jobs) --- - local: ".gitlab/ci/stages.yml" diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml index 7f958bf8b0..a67b34c756 100644 --- a/.gitlab/ci/jobs/build-dev.yml +++ b/.gitlab/ci/jobs/build-dev.yml @@ -2,7 +2,7 @@ # # Carries forward build_dev, build_dev_tags and build_main from the # previous root .gitlab-ci.yml. All three extend: -# - .local_build (this repo) — same body as upstream modules-gitlab-ci +# - .base_build (this repo) — same body as upstream modules-gitlab-ci # Build.gitlab-ci.yml `.build` (werf build + bundle crane # copy + release-channel crane copy + crane append to # register module), but without the upstream `rules:` @@ -11,7 +11,7 @@ # - .dev / .dev_tags / .main (this repo) for registry + tag context. # # The previous root .gitlab-ci.yml had its own `.build` template with the -# same body; it is replaced here by `.local_build` (verified against +# same body; it is replaced here by `.base_build` (verified against # /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci # templates/Build.gitlab-ci.yml — werf build with --save-build-report, # bundle / release-channel-version crane copy, and crane append to @@ -43,7 +43,7 @@ build_dev: variables: WERF_VIRTUAL_MERGE: "0" extends: - - .local_build + - .base_build - .dev build_dev_tags: @@ -54,7 +54,7 @@ build_dev_tags: variables: WERF_VIRTUAL_MERGE: "0" extends: - - .local_build + - .base_build - .dev_tags build_main: @@ -66,7 +66,7 @@ build_main: variables: WERF_VIRTUAL_MERGE: "0" extends: - - .local_build + - .base_build - .main # build_release mirrors GitHub dev_module_build.yml `push: [release-*]`: a @@ -83,5 +83,5 @@ build_release: variables: WERF_VIRTUAL_MERGE: "0" extends: - - .local_build + - .base_build - .release diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml index f86a8787dd..7eb90ce4e2 100644 --- a/.gitlab/ci/jobs/build-prod.yml +++ b/.gitlab/ci/jobs/build-prod.yml @@ -34,7 +34,7 @@ build_prod: resource_group: prod interruptible: false extends: - - .local_build + - .base_build - .prod_always - .dual_registry_login parallel: diff --git a/.gitlab/ci/jobs/deploy-dev.yml b/.gitlab/ci/jobs/deploy-dev.yml index dbf864a866..3af1115dba 100644 --- a/.gitlab/ci/jobs/deploy-dev.yml +++ b/.gitlab/ci/jobs/deploy-dev.yml @@ -19,7 +19,7 @@ deploy_for_dev_tag: stage: deploy_dev needs: ["build_dev_tags"] extends: - - .local_deploy + - .base_deploy - .dev_tags - .dual_registry_login parallel: diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index be372ed59d..fd90ef875b 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -19,7 +19,7 @@ # environment). # # Extends: -# - .local_deploy (this repo — mirrors upstream modules-gitlab-ci +# - .base_deploy (this repo — mirrors upstream modules-gitlab-ci # Deploy.gitlab-ci.yml `.deploy` script body without # the upstream `rules:` that would override our # gating). @@ -46,7 +46,7 @@ deploy_to_prod_alpha: RELEASE_CHANNEL: alpha needs: ["build_prod"] extends: - - .local_deploy + - .base_deploy - .prod_manual - .dual_registry_login parallel: @@ -66,7 +66,7 @@ deploy_to_prod_beta: RELEASE_CHANNEL: beta needs: ["build_prod"] extends: - - .local_deploy + - .base_deploy - .prod_manual - .dual_registry_login parallel: @@ -86,7 +86,7 @@ deploy_to_prod_ea: RELEASE_CHANNEL: early-access needs: ["build_prod"] extends: - - .local_deploy + - .base_deploy - .prod_manual - .dual_registry_login parallel: @@ -106,7 +106,7 @@ deploy_to_prod_stable: RELEASE_CHANNEL: stable needs: ["build_prod"] extends: - - .local_deploy + - .base_deploy - .prod_manual - .dual_registry_login parallel: @@ -126,7 +126,7 @@ deploy_to_prod_rock_solid: RELEASE_CHANNEL: rock-solid needs: ["build_prod"] extends: - - .local_deploy + - .base_deploy - .prod_manual - .dual_registry_login parallel: diff --git a/.gitlab/ci/jobs/precache.yml b/.gitlab/ci/jobs/precache.yml index 5d3ab958ce..936a77f299 100644 --- a/.gitlab/ci/jobs/precache.yml +++ b/.gitlab/ci/jobs/precache.yml @@ -10,14 +10,14 @@ # on: workflow_dispatch -> when: manual (Run pipeline) # matrix.branch: [main] -> single .main job # -# The actual build uses the local `.local_build` template, which mirrors the +# The actual build uses the local `.base_build` template, which mirrors the # upstream Build.gitlab-ci.yml script without inheriting its broad rules. This # file sets the trigger surface and the per-pipeline variables that the build # template expects. precache:build:main: extends: - - .local_build + - .base_build - .main stage: build interruptible: false @@ -34,7 +34,7 @@ precache:build:main: # the GH workflow but keeps it branch-driven for precache. precache:build:branch: extends: - - .local_build + - .base_build - .dev_vars stage: build interruptible: false diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml index 4484d82ffd..93ef8208aa 100644 --- a/.gitlab/ci/jobs/release-channels.yml +++ b/.gitlab/ci/jobs/release-channels.yml @@ -136,7 +136,7 @@ prod:check-requirements: # --------------------------------------------------------------------------- # 3+4. build (toggleable) + deploy, per edition # --------------------------------------------------------------------------- -# Deploy step (crane copy :tag -> :channel). Separate from .local_deploy so +# Deploy step (crane copy :tag -> :channel). Separate from .base_deploy so # the job can land in the deploy_prod stage. .prod_deploy_script: before_script: @@ -155,7 +155,7 @@ prod:build:ce: resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ce" extends: - .prod_release_rules - - .local_build + - .base_build - .prod_vars - .dual_registry_login needs: @@ -196,7 +196,7 @@ prod:build:ee: resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:ee" extends: - .prod_release_rules - - .local_build + - .base_build - .prod_vars - .dual_registry_login needs: @@ -237,7 +237,7 @@ prod:build:se-plus: resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:se-plus" extends: - .prod_release_rules - - .local_build + - .base_build - .prod_vars - .dual_registry_login needs: ["prod:build:ee"] @@ -274,7 +274,7 @@ prod:build:fe: resource_group: "prod-release:${RELEASE_CHANNEL}:${RELEASE_TAG}:fe" extends: - .prod_release_rules - - .local_build + - .base_build - .prod_vars - .dual_registry_login needs: ["prod:build:ee"] diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index e55a00511e..ab13c4e30b 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -9,7 +9,7 @@ # 4. notify - send Loop webhook with success/failure # # GitLab mapping: -# - The build step reuses the local `.local_build` template, with `.dev_vars` +# - The build step reuses the local `.base_build` template, with `.dev_vars` # for DEV registry variables and job-local rules for schedule/manual runs. # - The analyze step uses the upstream `.svace_analyze` template # (Svace_Analayze.gitlab-ci.yml; note upstream file name typo). @@ -79,14 +79,14 @@ svace:set-vars: # --------------------------------------------------------------------------- # build: produces Svace-instrumented artifacts. # -# Reuses `.local_build` so it does not inherit the dev-tag-only rules from +# Reuses `.base_build` so it does not inherit the dev-tag-only rules from # `.dev_tags`. The preceding svace:set-vars job writes the actual # MODULES_MODULE_TAG into a dotenv artifact. # --------------------------------------------------------------------------- svace:build: extends: - - .local_build + - .base_build - .dev_vars stage: build interruptible: false diff --git a/.gitlab/ci/templates/build.yml b/.gitlab/ci/templates/build.yml index 042880df84..6f18857048 100644 --- a/.gitlab/ci/templates/build.yml +++ b/.gitlab/ci/templates/build.yml @@ -1,4 +1,10 @@ -# Local .local_build template. +# Base build template (.base_build). +# +# Naming: this is the BASE werf-build job body that other build jobs extend +# (build_dev / build_main / build_prod / prod:build:* / precache / svace). +# It was previously named `.local_build`; renamed to `.base_build` because +# "local" was uninformative and collided conceptually with `include: local:`. +# "base" states the intent: a base template meant to be extended. # # This template mirrors the upstream modules-gitlab-ci Build.gitlab-ci.yml # `.build` body verbatim (verified against @@ -15,7 +21,7 @@ # specific branch / tag patterns provided by .dev / .dev_tags / .main / # .prod_always. # -# By keeping the upstream body in a locally named `.local_build` we keep +# By keeping the upstream body in a base template `.base_build` we keep # the same script (so future upstream fixes can be copy-pasted here) # while our job-level rules from .dev / .main / .prod_always / etc. # drive the gating. @@ -28,7 +34,7 @@ # `rules:` overrides through extends (tracked in upstream issues), revisit # this template and switch back to `extends: .build`. -.local_build: +.base_build: stage: build script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf jq crane diff --git a/.gitlab/ci/templates/deploy.yml b/.gitlab/ci/templates/deploy.yml index 42b103839f..45261baad8 100644 --- a/.gitlab/ci/templates/deploy.yml +++ b/.gitlab/ci/templates/deploy.yml @@ -1,11 +1,16 @@ -# Local .local_deploy template. +# Base deploy template (.base_deploy). +# +# Naming: this is the BASE crane-copy deploy job body extended by +# deploy_for_dev_tag / deploy_to_prod_*. Previously named `.local_deploy`; +# renamed to `.base_deploy` for the same reason as `.base_build` ("local" +# was uninformative and clashed with `include: local:`). # # Mirrors upstream modules-gitlab-ci Deploy.gitlab-ci.yml `.deploy` body # (verified in # /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci # templates/Deploy.gitlab-ci.yml, branch v13.0, HEAD 006d51c). # -# Why a local copy (same reasoning as .local_build in build.yml): +# Why a local copy (same reasoning as .base_build in build.yml): # # Upstream `.deploy` has `rules: [{if: $CI_COMMIT_TAG}]` + `when: manual` # baked in. Because GitLab CI merges parent rules with parent-first @@ -13,11 +18,11 @@ # tags too (we want it auto on dev tags and absent on prod tags). # The prod-channel chain (deploy_to_prod_*) wants `when: manual` from # .prod_manual, so the upstream `when: manual` happens to match — but we -# still want explicit control. Keeping the upstream body in `.local_deploy` +# still want explicit control. Keeping the upstream body in `.base_deploy` # means the same script is shared by all jobs but gating is fully driven # by our own templates. -.local_deploy: +.base_deploy: stage: deploy script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh crane From 54ba064ab3148c034d18eeeead24344236df0c1c Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 22:30:49 +0300 Subject: [PATCH 42/60] refactor(ci): group templates into base/ dev/ release/ subfolders Regroup .gitlab/ci/templates by the two global processes plus shared base: base/ build.yml deploy.yml dual_registry_login.yml info.yml dev/ dev.yml dev_tags.yml main.yml release.yml dev_vars.yml release/ prod_always.yml prod_manual.yml prod_vars.yml Only include: local: paths in includes.yml change. Anchor names are unchanged (.base_build, .dev, .prod_vars, ...), so no job extends: edits are needed. No rules/stage/behavior change. Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 32 +++++++++++-------- .gitlab/ci/templates/{ => base}/build.yml | 0 .gitlab/ci/templates/{ => base}/deploy.yml | 0 .../{ => base}/dual_registry_login.yml | 0 .gitlab/ci/templates/{ => base}/info.yml | 0 .gitlab/ci/templates/{ => dev}/dev.yml | 0 .gitlab/ci/templates/{ => dev}/dev_tags.yml | 0 .gitlab/ci/templates/{ => dev}/dev_vars.yml | 0 .gitlab/ci/templates/{ => dev}/main.yml | 0 .gitlab/ci/templates/{ => dev}/release.yml | 0 .../templates/{ => release}/prod_always.yml | 0 .../templates/{ => release}/prod_manual.yml | 0 .../ci/templates/{ => release}/prod_vars.yml | 0 13 files changed, 19 insertions(+), 13 deletions(-) rename .gitlab/ci/templates/{ => base}/build.yml (100%) rename .gitlab/ci/templates/{ => base}/deploy.yml (100%) rename .gitlab/ci/templates/{ => base}/dual_registry_login.yml (100%) rename .gitlab/ci/templates/{ => base}/info.yml (100%) rename .gitlab/ci/templates/{ => dev}/dev.yml (100%) rename .gitlab/ci/templates/{ => dev}/dev_tags.yml (100%) rename .gitlab/ci/templates/{ => dev}/dev_vars.yml (100%) rename .gitlab/ci/templates/{ => dev}/main.yml (100%) rename .gitlab/ci/templates/{ => dev}/release.yml (100%) rename .gitlab/ci/templates/{ => release}/prod_always.yml (100%) rename .gitlab/ci/templates/{ => release}/prod_manual.yml (100%) rename .gitlab/ci/templates/{ => release}/prod_vars.yml (100%) diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 7a2a1b0bf2..b70d39a919 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -47,7 +47,7 @@ include: # `rules:` baked in (`if: $CI_COMMIT_BRANCH`, `if: $CI_COMMIT_TAG`, # `when: manual`) that override our strict gating via # .dev / .dev_tags / .main / .prod_manual. We mirror the upstream - # script bodies in .gitlab/ci/templates/{build,deploy}.yml as + # script bodies in .gitlab/ci/templates/base/{build,deploy}.yml as # `.base_build` and `.base_deploy` and extend those instead. # --- Local structural fragments (order: stages, vars, defaults, then jobs) --- @@ -57,18 +57,24 @@ include: - local: ".gitlab/ci/workflow.yml" # --- Local shared templates (extends: ...) --- - - local: ".gitlab/ci/templates/dev_vars.yml" - - local: ".gitlab/ci/templates/prod_vars.yml" - - local: ".gitlab/ci/templates/dev.yml" - - local: ".gitlab/ci/templates/dev_tags.yml" - - local: ".gitlab/ci/templates/main.yml" - - local: ".gitlab/ci/templates/release.yml" - - local: ".gitlab/ci/templates/prod_manual.yml" - - local: ".gitlab/ci/templates/prod_always.yml" - - local: ".gitlab/ci/templates/info.yml" - - local: ".gitlab/ci/templates/dual_registry_login.yml" - - local: ".gitlab/ci/templates/build.yml" - - local: ".gitlab/ci/templates/deploy.yml" + # templates/ is grouped by the two global processes plus shared base: + # base/ - process-agnostic base job bodies + helpers + # dev/ - development process (DEV registry): MR / main / release / dev-tag + # release/ - release process (PROD registry): tag-push + manual dispatch + # Anchor names are unchanged (.base_build, .dev, .prod_vars, ...), so no job + # `extends:` edits are needed (see virtualization-m9e.5.22). + - local: ".gitlab/ci/templates/dev/dev_vars.yml" + - local: ".gitlab/ci/templates/release/prod_vars.yml" + - local: ".gitlab/ci/templates/dev/dev.yml" + - local: ".gitlab/ci/templates/dev/dev_tags.yml" + - local: ".gitlab/ci/templates/dev/main.yml" + - local: ".gitlab/ci/templates/dev/release.yml" + - local: ".gitlab/ci/templates/release/prod_manual.yml" + - local: ".gitlab/ci/templates/release/prod_always.yml" + - local: ".gitlab/ci/templates/base/info.yml" + - local: ".gitlab/ci/templates/base/dual_registry_login.yml" + - local: ".gitlab/ci/templates/base/build.yml" + - local: ".gitlab/ci/templates/base/deploy.yml" # --- Local job files owned by this issue --- - local: ".gitlab/ci/jobs/info.yml" diff --git a/.gitlab/ci/templates/build.yml b/.gitlab/ci/templates/base/build.yml similarity index 100% rename from .gitlab/ci/templates/build.yml rename to .gitlab/ci/templates/base/build.yml diff --git a/.gitlab/ci/templates/deploy.yml b/.gitlab/ci/templates/base/deploy.yml similarity index 100% rename from .gitlab/ci/templates/deploy.yml rename to .gitlab/ci/templates/base/deploy.yml diff --git a/.gitlab/ci/templates/dual_registry_login.yml b/.gitlab/ci/templates/base/dual_registry_login.yml similarity index 100% rename from .gitlab/ci/templates/dual_registry_login.yml rename to .gitlab/ci/templates/base/dual_registry_login.yml diff --git a/.gitlab/ci/templates/info.yml b/.gitlab/ci/templates/base/info.yml similarity index 100% rename from .gitlab/ci/templates/info.yml rename to .gitlab/ci/templates/base/info.yml diff --git a/.gitlab/ci/templates/dev.yml b/.gitlab/ci/templates/dev/dev.yml similarity index 100% rename from .gitlab/ci/templates/dev.yml rename to .gitlab/ci/templates/dev/dev.yml diff --git a/.gitlab/ci/templates/dev_tags.yml b/.gitlab/ci/templates/dev/dev_tags.yml similarity index 100% rename from .gitlab/ci/templates/dev_tags.yml rename to .gitlab/ci/templates/dev/dev_tags.yml diff --git a/.gitlab/ci/templates/dev_vars.yml b/.gitlab/ci/templates/dev/dev_vars.yml similarity index 100% rename from .gitlab/ci/templates/dev_vars.yml rename to .gitlab/ci/templates/dev/dev_vars.yml diff --git a/.gitlab/ci/templates/main.yml b/.gitlab/ci/templates/dev/main.yml similarity index 100% rename from .gitlab/ci/templates/main.yml rename to .gitlab/ci/templates/dev/main.yml diff --git a/.gitlab/ci/templates/release.yml b/.gitlab/ci/templates/dev/release.yml similarity index 100% rename from .gitlab/ci/templates/release.yml rename to .gitlab/ci/templates/dev/release.yml diff --git a/.gitlab/ci/templates/prod_always.yml b/.gitlab/ci/templates/release/prod_always.yml similarity index 100% rename from .gitlab/ci/templates/prod_always.yml rename to .gitlab/ci/templates/release/prod_always.yml diff --git a/.gitlab/ci/templates/prod_manual.yml b/.gitlab/ci/templates/release/prod_manual.yml similarity index 100% rename from .gitlab/ci/templates/prod_manual.yml rename to .gitlab/ci/templates/release/prod_manual.yml diff --git a/.gitlab/ci/templates/prod_vars.yml b/.gitlab/ci/templates/release/prod_vars.yml similarity index 100% rename from .gitlab/ci/templates/prod_vars.yml rename to .gitlab/ci/templates/release/prod_vars.yml From f79a982790e080b7cd4007143adb0293b53ed3bd Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 23:19:38 +0300 Subject: [PATCH 43/60] chore(ci): drop .gitlab/README.md and its references in CI files Remove the migration-progress README that tracked the GitLab CI migration. It was the source of TODO comments scattered across CI files; the migration log now lives in the migration plan (tmp/ai-summary/gitlab-ci-migration-plan.md). Clean up every reference to .gitlab/README.md in: - .gitlab-ci.yml - .gitlab/ci/variables.yml - .gitlab/ci/templates/dev/dev_vars.yml - .gitlab/ci/jobs/changelog.yml - .gitlab/ci/jobs/manual-tools.yml - .gitlab/ci/jobs/lint-dmt.yml - .gitlab/ci/jobs/svace.yml Signed-off-by: Nikita Korolev --- .gitlab-ci.yml | 2 +- .gitlab/README.md | 453 -------------------------- .gitlab/ci/jobs/changelog.yml | 2 +- .gitlab/ci/jobs/lint-dmt.yml | 2 +- .gitlab/ci/jobs/manual-tools.yml | 2 +- .gitlab/ci/jobs/svace.yml | 3 +- .gitlab/ci/templates/dev/dev_vars.yml | 3 +- .gitlab/ci/variables.yml | 1 - 8 files changed, 6 insertions(+), 462 deletions(-) delete mode 100644 .gitlab/README.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e3688cfc8..bf8820a1e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ # Upstream templates are sourced from deckhouse/3p/deckhouse/modules-gitlab-ci # at ref v13.0 (HEAD 006d51c35904b434eca2045a449aafb5e37a8827). The plan # calls for swapping the branch ref for a pinned SHA after the first green -# pipeline on virt-test; see .gitlab/README.md (TODO). +# pipeline on virt-test. include: - local: '.gitlab/ci/includes.yml' diff --git a/.gitlab/README.md b/.gitlab/README.md deleted file mode 100644 index fcc95d103d..0000000000 --- a/.gitlab/README.md +++ /dev/null @@ -1,453 +0,0 @@ -# GitLab CI for the `deckhouse/virtualization` module - -This directory contains the GitLab CI migration artifacts for the -`deckhouse/virtualization` module. - -The migration source of truth is -[`tmp/ai-summary/gitlab-ci-migration-plan.md`](../tmp/ai-summary/gitlab-ci-migration-plan.md). -Anything below that disagrees with the plan is a bug. - -The repository's root [`.gitlab-ci.yml`](../.gitlab-ci.yml) is the entry point -and `include`s the files in this directory. - ---- - -## Table of contents - -1. [Quick start](#1-quick-start) -2. [Layout](#2-layout) -3. [Required CI/CD variables](#3-required-cicd-variables) -4. [Migrating from `EXTERNAL_MODULES_*` to `DEV/PROD_MODULES_REGISTRY_*`](#4-migrating-from-external_modules_-to-devprod_modules_registry_) -5. [Token setup (`GITLAB_API_TOKEN`)](#5-token-setup-gitlab_api_token) -6. [Runner tags](#6-runner-tags) -7. [Jobs reference](#7-jobs-reference) -8. [Manual pipelines](#8-manual-pipelines) -9. [Scheduled pipelines](#9-scheduled-pipelines) -10. [Known TODOs / migration risks](#10-known-todos--migration-risks) -11. [Updating upstream templates (`modules-gitlab-ci`)](#11-updating-upstream-templates-modules-gitlab-ci) -12. [Slash commands and webhook listener](#12-slash-commands-and-webhook-listener) - ---- - -## 1. Quick start - -For a developer opening a new MR today, no action is required: -- Linting, unit tests, and the build pipeline trigger automatically on MR. -- Some validation jobs run only on changes to relevant paths. - -For a release engineer: -1. Verify all CI/CD variables from [§3](#3-required-cicd-variables) exist in - the project's `Settings -> CI/CD -> Variables`. Add missing ones — they are - not auto-provisioned. -2. Run `bash .gitlab/ci/scripts/bash/setup-mr-settings.sh --dry-run` to preview - the project MR settings that the script will apply, then drop `--dry-run` - to apply them once. -3. For a release: see [§8](#8-manual-pipelines) (`backport`, `changelog:milestone`, - `translate:changelog`). - -## 2. Layout - -``` -.gitlab/ -├── README.md # this file -└── ci/ - ├── changelog-sections.txt # shared allowed_sections list - ├── jobs/ # job definitions - │ ├── auto-assign-author.yml # auto-assign MR author - │ ├── backport.yml # cherry-pick + open MR - │ ├── changelog.yml # re-generate CHANGELOG from milestone - │ ├── check-changelog.yml # validate ```changes blocks - │ ├── check-milestone.yml # MR has a milestone - │ ├── manual-tools.yml # mrs:summary (Loop notification) - │ ├── lint-dmt.yml # DMT linter (allow_failure: true) - │ ├── test-scripts-js.yml # JS smoke tests (.gitlab/scripts/js) - │ ├── test-scripts-python.yml # python py_compile smoke check - │ ├── test-d8v-cli.yml # d8v CLI build + --help check - │ └── translate-changelog.yml # ru -> en changelog + MR - └── scripts/ - ├── bash/ - │ ├── auto-assign-author.sh - │ ├── backport.sh - │ ├── changelog-milestone.sh # wrapper for ../python/changelog_collect.py - │ ├── check-changelog-entry.sh # wrapper for ../python/check_changelog_entry.py - │ ├── check-milestone.sh - │ ├── check-runner-tools.sh # shell-executor tool preflight - │ ├── gitlab-ci-lint.sh - │ ├── set-vars.sh - │ ├── setup-mr-settings.sh # one-off project settings - │ └── lib/ - │ └── api.sh # shared GitLab API helper - └── python/ - ├── changelog_collect.py - └── check_changelog_entry.py -.gitlab/scripts/js/ -├── package.json -├── mrs_notifier.mjs # GitLab counterpart of prs_notifier.mjs -└── mrs_notifier.test.mjs # node:test smoke test -``` - -Every job `extends` (or `include`s) a script in `.gitlab/ci/scripts/bash/`. -Scripts source `.gitlab/ci/scripts/bash/lib/api.sh` for the `api GET / POST / PUT` -helper. - -## 3. Required CI/CD variables - -The table below lists every variable referenced from this directory's CI -files. The full list (including build/deploy) is in -`tmp/ai-summary/gitlab-ci-migration-plan.md` §4 / §11.13. - -### Secrets (must be marked `Masked`) - -| Variable | Scope | Description | -|---|---|---| -| `GITLAB_API_TOKEN` | api, write_repository | Project Access Token. Used by every `api.sh`-backed script (auto-assign, backport, changelog, check-milestone, mrs-summary, project settings). See [§5](#5-token-setup-gitlab_api_token). | -| `RELEASE_TOKEN` | api, write_repository | Alias used by upstream `Translate_Changelog` template. Create a separate token if you prefer to scope it tighter; otherwise use the same PAT as `GITLAB_API_TOKEN`. | -| `DEV_MODULES_REGISTRY_PASSWORD` | dev registry | Write access to DEV modules registry. | -| `PROD_MODULES_REGISTRY_PASSWORD` | prod registry, protected | Write access to PROD modules registry. Only available on protected branches/tags. | -| `PROD_READ_REGISTRY_PASSWORD` | prod read registry | Read-only access for `check:requirements`. | -| `PROD_READ_REGISTRY_USER` | prod read registry | Read-only login. | -| `SOURCE_REPO` | private source repo | URL for `werf import`. | -| `SOURCE_REPO_SSH_KEY` | private source repo, type=file | SSH key for cloning the source repo. Mark `Expand variable reference = false`. | -| `LOOP_WEBHOOK_URL` | Loop chat | Incoming webhook URL. Mark `Expand variable reference = false`. | -| `LOOP_TOKEN` | Loop API (optional) | Only needed if Loop API is used in addition to the webhook. | -| `DMT_METRICS_TOKEN` | DMT linter | Auth token for DMT metrics endpoint. | -| `DMT_METRICS_URL` | DMT linter | Endpoint URL for DMT metrics. | -| `VAULT_ROLE` | cve scan | Vault role at `seguro.flant.com`; value = repository name (`virtualization`). Required by the upstream `.cve_scan` template for JWT-based Vault login. Without it, `cve:scan:*` jobs fail at Vault login. | - -### Plain variables (`Masked = off`) - -| Variable | Description | -|---|---| -| `MODULE_NAME` | `virtualization` (already set in the root `.gitlab-ci.yml`). | -| `DEV_REGISTRY` | DEV modules registry host (e.g. `dev-registry.deckhouse.io`). | -| `DEV_MODULE_SOURCE` | DEV modules path (e.g. `dev-registry.deckhouse.io/sys/deckhouse-oss/modules`). | -| `DEV_MODULES_REGISTRY_LOGIN` | DEV registry login. | -| `PROD_REGISTRY` | PROD modules registry host (e.g. `registry-write.deckhouse.io`). | -| `PROD_READ_REGISTRY` | PROD read-only registry host. | -| `PROD_MODULES_REGISTRY_LOGIN` | PROD registry login. | -| `PROD_MODULE_SOURCE_NAME` | `deckhouse` (used in `${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}/${EDITION}/modules`). | -| `LOOP_CHANNEL_ID` | Loop channel ID (not secret). | -| `LOOP_API_BASE_URL` | Loop API base URL (not secret). | - -### Not needed anymore (legacy, remove from project variables) - -- `GITHUB_TOKEN` — replaced by `CI_JOB_TOKEN` / `GITLAB_API_TOKEN`. -- `RELEASE_PLEASE_TOKEN` — replaced by `GITLAB_API_TOKEN` / `RELEASE_TOKEN`. -- `K8S_CLUSTER_SECRET`, `VIRT_E2E_NIGHTLY_SA_TOKEN`, all `E2E_*` — these are - scoped to e2e workflows which are **not** migrated. - -## 4. Migrating from `EXTERNAL_MODULES_*` to `DEV/PROD_MODULES_REGISTRY_*` - -The legacy root [`.gitlab-ci.yml`](../.gitlab-ci.yml) (pre-migration) referenced: - -- `EXTERNAL_MODULES_DEV_REGISTRY_LOGIN` -- `EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD` -- `EXTERNAL_MODULES_PROD_REGISTRY_LOGIN` -- `EXTERNAL_MODULES_PROD_REGISTRY_PASSWORD` - -These were renamed (and several new vars were added) to match the upstream -`modules-gitlab-ci@v13.0` template names. Migration steps: - -1. Open `Settings -> CI/CD -> Variables`. -2. For each legacy variable in the table below, create the new name with the - same value, then delete the old one. - - | Old name | New name | - |---|---| - | `EXTERNAL_MODULES_DEV_REGISTRY_LOGIN` | `DEV_MODULES_REGISTRY_LOGIN` | - | `EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD` | `DEV_MODULES_REGISTRY_PASSWORD` | - | `EXTERNAL_MODULES_PROD_REGISTRY_LOGIN` | `PROD_MODULES_REGISTRY_LOGIN` | - | `EXTERNAL_MODULES_PROD_REGISTRY_PASSWORD` | `PROD_MODULES_REGISTRY_PASSWORD` | - -3. Add the new plain vars from [§3](#3-required-cicd-variables): - `DEV_REGISTRY`, `DEV_MODULE_SOURCE`, `PROD_REGISTRY`, `PROD_READ_REGISTRY`, - `PROD_MODULE_SOURCE_NAME`. -4. Trigger a test pipeline on a non-protected branch. The first pipeline will - fail with a clear error if any variable is missing. -5. Once the test pipeline is green, delete the legacy variables. - -## 5. Token setup (`GITLAB_API_TOKEN`) - -`GITLAB_API_TOKEN` must be a **Project Access Token** (PAT) scoped to this -project, with: - -- Role: **Maintainer** (or higher). -- Scopes: `api`, `write_repository`. - -Steps: - -1. `Settings -> Access Tokens -> Add new token`. -2. Name: `ci-bot` (or anything). -3. Role: `Maintainer`. -4. Scopes: `api` + `write_repository`. -5. Pick an expiry (90 days is the default; rotate manually when prompted). -6. Save the value into `Settings -> CI/CD -> Variables -> GITLAB_API_TOKEN` - with `Masked = true`, `Protected = false` (this script set needs it on - feature branches too). -7. The same value should also be stored as `RELEASE_TOKEN` (the upstream - Translate_Changelog template prefers that name). - -For local debugging (e.g. `setup-mr-settings.sh`) you can export -`GITLAB_API_TOKEN` in your shell, but never commit it. - -## 6. Runner tags - -All jobs in this directory specify `tags: [deckhouse]`. This is the agreed -runner tag for the project, registered at -. - -### Shell executor requirements - -The project runner is expected to use the GitLab Runner `shell` executor. -For that executor, `image:` and container `entrypoint:` settings are ignored, -so project jobs do not install packages with `apk`, `apt-get`, or other host -package managers. Tools must already be installed on the runner host. Jobs that -need non-trivial tools call `.gitlab/ci/scripts/bash/check-runner-tools.sh` in -`before_script` and fail early with a clear message if a tool is missing. - -Expected host tools for project-owned jobs: - -| Job family | Required runner tools | -|---|---| -| Common GitLab API helpers | `bash`, `curl`, `jq` | -| Go/task validation jobs | `go`, `task`; `lint:shellcheck` also needs `shellcheck` | -| Generated-file checks | `go`, `task`, `git` | -| Build/deploy/precache/cleanup templates | upstream `Setup.gitlab-ci.yml` needs `bash`, `curl`, `trdl` bootstrap support, `werf`, `jq`, `crane`, registry credentials, and SSH tools when Svace keys are configured | -| Changelog/check-changelog jobs | `bash`, `python3`, `curl`, `jq`; changelog MR creation also needs `git`, `ssh-agent`, `ssh-add` | -| Backport | `bash`, `git`, `curl`, `jq`, `ssh-agent`, `ssh-add` | -| MR summary | `node`, `npm` | -| `test:scripts:js` | `node`, `npm` | -| `test:scripts:python` | `python3` | -| `test:build:d8v-cli` | `go`, `task` | -| `lint:dmt` | upstream Setup (trdl-installed `dmt`) | -| Upstream scanning templates | use the requirements from `modules-gitlab-ci@v13.0` (for example CVE scan downloads `d8` and uses `curl`, `tar`, `jq`, `git`, SSH tools) | - -## 7. Jobs reference - -| Job | Stage | Trigger | Required token | What it does | -|---|---|---|---|---| -| `auto-assign-author` | info | MR opened / reopened | `GITLAB_API_TOKEN` | Assigns the MR author via API. Skips silently if the MR already has an assignee (plan §0(4)). | -| `check:milestone` | lint | MR open / synchronize | `GITLAB_API_TOKEN` | Fails if MR has no `milestone` assigned. | -| `check:changelog` | lint | MR open / synchronize | `GITLAB_API_TOKEN` | Validates ` ```changes ` blocks in MR description against `.gitlab/ci/changelog-sections.txt`. | -| `translate:changelog` | pre | push to `main` / `release-X.Y` | `RELEASE_TOKEN` (falls back to `CI_JOB_TOKEN`) | Translates the latest English `CHANGELOG/CHANGELOG-v*.yml` to Russian (`.ru.yml`) and opens an MR. | -| `lint:dmt` | lint | MR (`merge_request_event`) | `DMT_METRICS_URL`, `DMT_METRICS_TOKEN` (optional) | Runs `dmt lint ./` (Deckhouse Module Tester). Non-blocking (`allow_failure: true`), mirroring GH `continue-on-error: true`. | -| `test:scripts:js` | test | MR (`merge_request_event`) | — | Runs `npm test` (node:test smoke test) in `.gitlab/scripts/js`. | -| `test:scripts:python` | test | MR (`merge_request_event`) | — | `python3 -m py_compile` syntax smoke check over `.gitlab/ci/scripts/python/*.py`. Non-blocking; real unit tests are a TODO. | -| `test:build:d8v-cli` | test | MR (`merge_request_event`) | — | Builds the `d8v` CLI (`task d8v-cli:build`) and runs `./src/cli/d8v --help`. | -| `changelog:milestone` | lint | manual / scheduled | `GITLAB_API_TOKEN` | Re-generates `CHANGELOG/CHANGELOG-.yml` and `CHANGELOG/CHANGELOG-.md` from MRs with a milestone. Optionally opens a changelog MR. | -| `changelog:all-active-milestones` | lint | manual / scheduled | `GITLAB_API_TOKEN` | Same as above, but iterates over all active milestones. | -| `backport` | lint | manual with `TARGET_BRANCH` OR MR labelled `backport-release-X.Y` | `GITLAB_API_TOKEN` | Cherry-picks the merged MR into a new `backport//` branch, pushes it, and opens an MR to the release branch. | -| `mrs:summary` | notify | manual / scheduled | `GITLAB_API_TOKEN`, `LOOP_WEBHOOK_URL` | Posts a markdown summary of open MRs to Loop (replaces `prs_notifier.mjs`). | - -### Prod release channels (manual `Run pipeline`) - -Parity with `.github/workflows/release_module_release-channels.yml`. A manual -single-channel/single-tag release flow triggered from `Run pipeline` on the -tag `$RELEASE_TAG` (`CI_PIPELINE_SOURCE == "web"`). It never collides with -the tag-push flow (`build_prod` / `deploy_to_prod_*`) because it only runs on -web pipelines. - -| Job | Stage | What it does | -|---|---|---| -| `prod:print-vars` | info | Echoes release inputs and enforces `RELEASE_TAG` matches the pipeline ref tag. | -| `prod:check-requirements` | prod_check | `tools/moduleversions check:requirements` (Deckhouse version range). Skipped when `CHECK_ONLY` or `SKIP_REQUIREMENTS_CHECK`. | -| `prod:build:` | build | Optional (`ENABLE_BUILD`) werf build per edition. `se-plus`/`fe` `needs: prod:build:ee` (GH cascade). | -| `prod:deploy:` | deploy_prod | `crane copy :$RELEASE_TAG -> :$RELEASE_CHANNEL` per selected edition. | -| `prod:check-version` | prod_verify | Matrix `registry`/`releases`/`documentation` via `tools/moduleversions`. | -| `prod:create-gitlab-release` | prod_release | Creates a GitLab release from the merged `label:changelog` MR description (was `create-github-release`). | -| `prod:notify-loop` | notify | Posts a release status table to Loop (`LOOP_WEBHOOK_URL`). | - -**Required Project CI/CD variables:** `GITLAB_API_TOKEN` (api scope, -Maintainer role — release creation + MR query), `LOOP_WEBHOOK_URL`, and the -read-only `PROD_READ_REGISTRY` / `PROD_READ_REGISTRY_USER` / -`PROD_READ_REGISTRY_PASSWORD` triple. Note: `tools/moduleversions` hardcodes -`registry.deckhouse.io`; `PROD_READ_REGISTRY` must resolve to that host for -the crane login to apply. - -### CVE (Trivy) scan -- `cve:scan:mr` — per-MR scan of the dev build image (`mr${CI_MERGE_REQUEST_IID}`), runs after `build_dev`, gated on `merge_request_event`; non-blocking (`allow_failure: true`). -- `cve:scan:daily` — scheduled daily scan of `main` (`0 02 * * *`). -- `cve:scan:manual` — web-triggered manual scan of `${SCAN_TAG:-main}`. -All extend the upstream `.cve_scan` template. **Required Project CI/CD variable:** `VAULT_ROLE` (Vault role at seguro.flant.com; value = repository name, `virtualization`). Without it, scan jobs fail at Vault login. - -## 8. Manual pipelines - -GitLab has no native equivalent of `workflow_dispatch`; instead, jobs are -marked `when: manual` and triggered from `Run pipeline` UI. To run a manual -pipeline: - -1. Open `CI/CD -> Pipelines -> Run pipeline`. -2. Pick the branch (default: current default branch). -3. Fill in variables: - - For `backport`: set `TARGET_BRANCH=release-1.21` (or whichever). - - For `changelog:milestone`: optionally set `MILESTONE_TITLE=v1.21.3` and - `OPEN_CHANGELOG_MR=true`. - - For `mrs:summary`: ensure `LOOP_WEBHOOK_URL` is set. - - For a **prod release-channel dispatch** (`prod:*` jobs): run the pipeline - **on the tag** (`$RELEASE_TAG`) and set `RELEASE_TAG=vX.Y.Z`, - `RELEASE_CHANNEL=alpha|beta|early-access|stable|rock-solid`, - `EDITION_CE=true` and/or `EDITION_EE=true`, plus `ENABLE_BUILD`, - `CHECK_ONLY`, `SKIP_REQUIREMENTS_CHECK`, `RELEASE_TO_GITLAB`, - `SEND_RESULTS_TO_LOOP` as needed. See [§7 Prod release channels](#7-jobs-reference). -4. Submit. The pipeline starts; manual jobs appear under the pipeline view - with a `play` button. - -The `dispatch-slash-command.yml` GitHub workflow (slash commands like -`/changelog`, `/backport`, `/e2e` in MR comments) was intentionally **not** -migrated as reactive automation. See [§12](#12-slash-commands-and-webhook-listener) -for the future plan. - -## 9. Scheduled pipelines - -Several jobs are intended to run on a schedule. Configure schedules at -`CI/CD -> Schedules -> New schedule`. Each scheduled job has its own intended -cadence: - -| Job(s) | Intended cron | Notes | -|---|---|---| -| `cve:scan:daily` | `0 2 * * *` (daily) | CVE scan of `main`. | -| `svace:*` | `0 4 * * 6` (weekly, Sat) | Svace analysis + report. | -| `gitleaks:full:scheduled` | daily | Full secrets scan. | -| `precache` | `0 */8 * * *` (every 8h) | Warm the build cache. | -| `changelog:milestone` / `changelog:all-active-milestones` | nightly | Re-generate CHANGELOG. | -| `mrs:summary` | `0 7 * * *` (10:00 MSK) | MR summary to Loop. | -| `cleanup` | `12 0 * * 6` (weekly, Sat) | Prune old DEV registry images. | - -Schedules trigger pipelines whose `CI_PIPELINE_SOURCE == "schedule"`. - -> **Known issue — tracked in `virtualization-m9e.5.11`.** Every scheduled job -> currently gates **only** on `CI_PIPELINE_SOURCE == "schedule"` with **no -> discriminator**. In GitLab a schedule triggers the whole pipeline, so a -> single schedule fires **all** of these jobs at once, and multiple schedules -> each fire **all** of them — there is no way to honor the distinct crons -> above (e.g. weekly `cleanup` vs daily `cve:scan:daily`). Result: `cleanup` -> never runs at its intended weekly cadence (it runs at whatever cadence any -> schedule has, or never if no schedule exists). The fix is a `SCHEDULE_TYPE` -> (or `SCHEDULE_CRON`) variable set per schedule, with each job gated on -> `CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == ""`. - -## 10. Known TODOs / migration risks - -Migration goal: **full parity with GitHub Actions** (development is moving to -GitLab). The previous root `.gitlab-ci.yml` is **not** a behavior baseline — -anything that diverges from the GitHub workflows is a gap to fix, even if it -matches the old GitLab config. Open work is tracked under sub-epics -`virtualization-m9e.6` (prod release parity) and `virtualization-m9e.5` -(review findings & fixes). - -### Confirmed bugs / parity gaps (must fix) - -- **`build_prod` builds `ce` as EE** (`virtualization-m9e.6.1`) — **FIXED**. - `.gitlab/ci/jobs/build-prod.yml` and `deploy-prod.yml` now set - `MODULE_EDITION` per edition via the `parallel:matrix` (CE for `ce`, EE for - `ee`/`se-plus`/`fe`), matching GitHub. `.prod_vars` intentionally does not - set it (it comes from the matrix). -- **Edition `se-plus` missing in prod** (`virtualization-m9e.6.2`) — **FIXED**. - `se-plus` (path `se-plus/modules`, `MODULE_EDITION=EE`) is added to the - `build_prod` and `deploy_to_prod_*` matrices, matching the GitHub release - workflows. Note: GH runs `se-plus`/`fe` after `ee` (`needs:`); the GitLab - matrix runs all four editions in parallel — acceptable since each edition - builds into its own registry subpath with no shared artifact. -- **Release-channels feature parity** (`virtualization-m9e.6.3`) — **PORTED** - (with documented deviations). `.gitlab/ci/jobs/release-channels.yml` adds a - manual `Run pipeline` flow (`prod:*` jobs) mirroring - `release_module_release-channels.yml`: `RELEASE_CHANNEL`/`EDITION_CE`/ - `EDITION_EE`/`RELEASE_TAG`/`ENABLE_BUILD`/`CHECK_ONLY`/ - `SKIP_REQUIREMENTS_CHECK`/`RELEASE_TO_GITLAB`/`SEND_RESULTS_TO_LOOP` inputs, - requirements check (`prod:check-requirements`), build/deploy per edition, - version verification matrix (`prod:check-version`), release creation - (`prod:create-gitlab-release`), and Loop notification (`prod:notify-loop`). - Deviations from GitHub (intentional): release creation targets **GitLab - Releases** (not GitHub Releases) since development moves to GitLab; the - pipeline must run **on the tag** (`RELEASE_TAG == CI_COMMIT_TAG`, enforced - by `prod:print-vars`) because GitLab cannot checkout an arbitrary tag - without overriding the inherited werf `Setup` before_script; per-(channel,tag) - concurrency uses `resource_group` (serialize, no cancel-in-progress). -- **Scheduled-job discriminator** (`virtualization-m9e.5.11`) — see the known - issue in [§9](#9-scheduled-pipelines): `cleanup` and the other scheduled - jobs cannot honor their individual crons without a `SCHEDULE_TYPE` variable. -- **`test` serialized behind `lint`** (`virtualization-m9e.5.12`) — test jobs - lack `needs:`, so they wait for the whole `lint` stage instead of running in - parallel as on GitHub. (Mixing lint and validation in one stage is fine — - same-stage jobs already run in parallel; the fix is a DAG `needs:`, not - splitting stages.) - -### Intentional first-iteration gaps - -- **Webhook listener for slash-commands** — GitLab does not natively start - pipelines on MR comment creation or label change. See - [§12](#12-slash-commands-and-webhook-listener). -- **Reactive `changelog:milestone`** — currently manual + scheduled. When - the webhook listener lands, add a `merge_request.closed` / `milestoned` - handler that triggers `changelog:milestone` with the right - `MILESTONE_TITLE`. -- **`prs_notifier.mjs` STUCK detection** (`virtualization-m9e.5.14`) — the - original GitHub version uses per-review `submitted_at` to compute "stuck for - 1.5 days". The GitLab port currently treats all unresolved discussions as - "stuck" without checking thread age. TODO: pull - `discussions[].notes[].created_at` to refine the heuristic. -- **GitLab username for `z9r5`** — the doc reviewer is hard-coded as - `DOC_REVIEWER=z9r5` in `mrs:summary`. Override via the - `DOC_REVIEWER` CI/CD variable until the real username is confirmed. -- **Vault integration in `cve_scan_on_pr`** — the GH workflow read secrets - from HashiCorp Vault via `hashicorp/vault-action@v2`. After migration, - any secret that CVE scan needs is expected to live in CI/CD variables. - If a secret remains in Vault only, the CVE-scan job needs a JWT-auth - sidecar (out of scope for the first iteration). - -### Ported/dropped GitHub jobs (issue `virtualization-m9e.5.8`) - -The GitHub workflow `.github/workflows/dev_module_build.yml` defined four jobs -with no direct GitLab counterpart. Decisions: - -| GH job | Decision | Notes | -|---|---|---| -| `lint_dmt` | **Ported** → `lint:dmt` (`.gitlab/ci/jobs/lint-dmt.yml`) | The upstream `Build.gitlab-ci.yml` ships a hidden `.lint` template running `dmt lint ./` with `allow_failure: true`. We mirror its script (we do NOT include `Build.gitlab-ci.yml` directly, to avoid its `.build`/`.deploy` rules bypassing our `.dev` gating) and rely on Setup's `before_script` to install `dmt` via `trdl`. `allow_failure: true` mirrors GH `continue-on-error: true`. `DMT_METRICS_URL`/`DMT_METRICS_TOKEN` are optional CI/CD vars. | -| `test_scripts_js` | **Ported** → `test:scripts:js` (`.gitlab/ci/jobs/test-scripts-js.yml`) | Fixed `.gitlab/scripts/js/package.json` which referenced a missing `mrs_notifier.test.mjs`; added that file as a minimal node:test smoke test (syntax check + structural assertions). `mrs_notifier.mjs` auto-runs at import, so full unit tests are a TODO pending a refactor exporting pure helpers. | -| `test_scripts_python` | **Ported (smoke)** → `test:scripts:python` (`.gitlab/ci/jobs/test-scripts-python.yml`) | `.gitlab/ci/scripts/python/` has no tests; this is a non-blocking `python3 -m py_compile` syntax check. Real unit tests are a TODO. | -| `test_build_d8v_cli` | **Ported** → `test:build:d8v-cli` (`.gitlab/ci/jobs/test-d8v-cli.yml`) | Runs `task d8v-cli:build` then `./src/cli/d8v --help`. MR-gated via `.dev`, mirroring the GH PR-only trigger. | - -All four jobs are wired into `.gitlab/ci/includes.yml`. - -## 11. Updating upstream templates (`modules-gitlab-ci`) - -The `include: project: 'deckhouse/3p/deckhouse/modules-gitlab-ci' ref: 'v13.0'` -in [`.gitlab/ci/jobs/translate-changelog.yml`](ci/jobs/translate-changelog.yml) -currently tracks the `v13.0` **branch**. After the migration stabilises -(~2–4 weeks), pin to a commit SHA: - -```bash -git ls-remote git@fox.flant.com:deckhouse/3p/deckhouse/modules-gitlab-ci.git refs/heads/v13.0 -# Pick the first column as . Then in translate-changelog.yml: -# ref: '' # was: v13.0 — pinned at YYYY-MM-DD -``` - -After pinning, run a test pipeline on a feature branch before merging. - -## 12. Slash commands and webhook listener - -GitHub let us react to MR comments (`/changelog`, `/backport`, `/e2e`) and -label changes (`status/backport`, `analyze/svace`) without a webhook. GitLab -does not — see upstream issues: - -- (label change triggers) -- (comment triggers) - -For full GitHub parity, deploy a small **webhook-listener** service that: - -1. Accepts GitLab webhooks: - - `Merge Request Hook` (filter on `action=open|update`, `labels.title changed`, - `action=close`). - - `Note Hook` (filter on `noteable_type=MergeRequest`, `action=create`). -2. Parses the payload and calls - `POST /api/v4/projects/:id/trigger/pipeline` with the right variables. - -Until that exists, the manual job matrix in [§7](#7-jobs-reference) and the -scheduled jobs in [§9](#9-scheduled-pipelines) cover the same surface area, -with a human pressing the button. - -## See also - -- [`tmp/ai-summary/gitlab-ci-migration-plan.md`](../tmp/ai-summary/gitlab-ci-migration-plan.md) — full migration plan. -- [`.gitlab-ci.yml`](../.gitlab-ci.yml) — root pipeline. -- [`/Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci`](../) — local checkout of upstream templates used by `translate:changelog`. diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index 5c91ffd39c..402d15e936 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -22,7 +22,7 @@ # # Per migration plan §0(3) and §11.5.4 the GitLab jobs are manual + scheduled, # NOT reactive. A webhook-listener for slash-command-equivalents and label -# events is a documented TODO (see .gitlab/README.md). +# events is a documented TODO. # # Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). diff --git a/.gitlab/ci/jobs/lint-dmt.yml b/.gitlab/ci/jobs/lint-dmt.yml index 99113b372c..b5bf623b8b 100644 --- a/.gitlab/ci/jobs/lint-dmt.yml +++ b/.gitlab/ci/jobs/lint-dmt.yml @@ -31,7 +31,7 @@ # it inherits Setup's before_script, which performs the trdl+dmt bootstrap. # # DMT_METRICS_URL / DMT_METRICS_TOKEN are optional CI/CD variables consumed -# by dmt for metrics reporting; see .gitlab/README.md §3. +# by dmt for metrics reporting. lint:dmt: stage: lint diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index c43ecc1247..4cc999e897 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -21,7 +21,7 @@ # Migration plan §0(1) explicitly removes GitHub slash-command dispatch # (/.github/workflows/dispatch-slash-command.yml) and replaces it with # manual pipelines. The two manual jobs in this file are the closest -# equivalents. There is no automated webhook listener yet (TODO in README). +# equivalents. There is no automated webhook listener yet (TODO). # Variables for "Run pipeline" UI: # LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index ab13c4e30b..7e65018261 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -17,8 +17,7 @@ # - workflow_dispatch -> when: manual. # - MR opt-in -> set SVACE_PIPELINE_ENABLED=true when running a pipeline. # - Loop notification kept as a manual / always job. The original GH -# webhook URL is the LOOP_WEBHOOK_URL masked variable (see -# .gitlab/README.md variables list). +# webhook URL is the LOOP_WEBHOOK_URL masked variable. # # Required CI/CD variables (declared in the project): # SVACE_ANALYZE_HOST masked diff --git a/.gitlab/ci/templates/dev/dev_vars.yml b/.gitlab/ci/templates/dev/dev_vars.yml index ab529a5563..b878b97860 100644 --- a/.gitlab/ci/templates/dev/dev_vars.yml +++ b/.gitlab/ci/templates/dev/dev_vars.yml @@ -7,8 +7,7 @@ # plan §11.6. # # The legacy EXTERNAL_MODULES_DEV_REGISTRY_* names are still accepted as -# fallback to ease the variable-rename migration; they are documented in -# .gitlab/README.md (TODO) as deprecated. +# fallback to ease the variable-rename migration. .dev_vars: variables: diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index 028ab3d128..decdd61e06 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -14,7 +14,6 @@ # `deckhouse/${EDITION}/modules`) move to DEV_REGISTRY / DEV_MODULE_SOURCE / # PROD_REGISTRY / PROD_MODULE_SOURCE_NAME vars. # -# See .gitlab/README.md (TODO) for the migration steps. variables: # --- Module identity --- From e99f19d42ed12580fa3268720b0d947edc5f5898 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 25 Jun 2026 23:31:49 +0300 Subject: [PATCH 44/60] chore(ci): drop internal issue references from CI comments Remove internal tracker issue IDs scattered in comments across .gitlab/ci files so they do not leak into the public PR. No behavior change. Affected files: - .gitlab/ci/stages.yml - .gitlab/ci/includes.yml - .gitlab/ci/templates/release/prod_manual.yml - .gitlab/ci/jobs/deploy-prod.yml - .gitlab/ci/jobs/gitleaks.yml Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 4 ++-- .gitlab/ci/jobs/deploy-prod.yml | 6 +++--- .gitlab/ci/jobs/gitleaks.yml | 2 +- .gitlab/ci/stages.yml | 6 +++--- .gitlab/ci/templates/release/prod_manual.yml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index b70d39a919..353fb7838f 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -62,7 +62,7 @@ include: # dev/ - development process (DEV registry): MR / main / release / dev-tag # release/ - release process (PROD registry): tag-push + manual dispatch # Anchor names are unchanged (.base_build, .dev, .prod_vars, ...), so no job - # `extends:` edits are needed (see virtualization-m9e.5.22). + # `extends:` edits are needed. - local: ".gitlab/ci/templates/dev/dev_vars.yml" - local: ".gitlab/ci/templates/release/prod_vars.yml" - local: ".gitlab/ci/templates/dev/dev.yml" @@ -88,7 +88,7 @@ include: - local: ".gitlab/ci/jobs/deploy-prod.yml" # Prod release-channels parity flow (manual single-channel dispatch): # requirements check, build/deploy per edition, version verification, - # GitLab release creation, Loop notification. See virtualization-m9e.6.3. + # GitLab release creation, Loop notification. - local: ".gitlab/ci/jobs/release-channels.yml" - local: ".gitlab/ci/jobs/cleanup.yml" diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index fd90ef875b..01ae5ac019 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -9,7 +9,7 @@ # the earlier channels. This replaces the previous forced promotion chain # (alpha -> beta -> ea -> stable -> rock-solid across five separate stages), # which was a carried-over artifact of the old root .gitlab-ci.yml and -# diverged from GitHub (see virtualization-m9e.5.21). Each job copies the +# diverged from GitHub. Each job copies the # built release image to its named release channel on a vX.Y.Z tag. # # The EDITION matrix mirrors build_prod (ce/ee/se-plus/fe) with the correct @@ -27,11 +27,11 @@ # with `when: manual`). # - .dual_registry_login (this repo — also login against DEV_REGISTRY). # -# TODO: per migration plan §11.4.1 and sub-epic virtualization-m9e.6.3, the +# TODO: per migration plan §11.4.1, the # GH release_module_release-channels workflow exposes inputs (channel, # ce/ee, tag, enableBuild, release_to_github, check_only, # send_results_to_loop, requirements/version checks, github release -# creation, Loop notification) that are not yet ported. Track in m9e.6.3. +# creation, Loop notification) that are not yet ported. # # This file is the tag-push chained-deploy flow. The GitHub # release_module_release-channels parity flow (single-channel dispatch with diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index 815e683418..9def66adc7 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -27,7 +27,7 @@ # stages.yml (it was dead — every active gitleaks job runs in `scan`). Even # with `when: never`, GitLab validates the stage exists at config-parse # time, so leaving the inherited `gitleaks` stage here would break the -# pipeline. See virtualization-m9e.5.21. +# pipeline. gitleaks_diff: extends: .gitleaks_scan stage: scan diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index 6b41e5cc88..cf5690445e 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -16,7 +16,7 @@ # build — werf build for dev / dev-tags / main / prod # scan — cve_scan, gitleaks, svace (owned by sibling issues) # deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid -# prod_check — requirements check before release-channel dispatch (m9e.6.3). +# prod_check — requirements check before release-channel dispatch. # Runs BEFORE build so prod:build:* can need it. # deploy_prod — prod release-channel deploy. ONE stage with independent # manual jobs per channel (tag-push flow, deploy-prod.yml) @@ -25,8 +25,8 @@ # alpha->...->rock-solid promotion chain — each channel # deploy is an independent `when: manual` job, matching # GitHub release_module_release-channels.yml semantics. -# prod_verify — version-on-release-channel check matrix (m9e.6.3) -# prod_release — GitLab release creation (m9e.6.3) +# prod_verify — version-on-release-channel check matrix +# prod_release — GitLab release creation # notify — release-results-to-loop and similar fan-out # cleanup — scheduled registry cleanup # diff --git a/.gitlab/ci/templates/release/prod_manual.yml b/.gitlab/ci/templates/release/prod_manual.yml index 08226957e3..2efecd26eb 100644 --- a/.gitlab/ci/templates/release/prod_manual.yml +++ b/.gitlab/ci/templates/release/prod_manual.yml @@ -17,7 +17,7 @@ # .gitlab/ci/jobs/release-channels.yml (prod:* jobs). This template stays # as the tag-push deploy context: each channel deploy is an independent # `when: manual` job in the single `deploy_prod` stage (no forced -# alpha->...->rock-solid promotion chain; see virtualization-m9e.5.21). +# alpha->...->rock-solid promotion chain). .prod_manual: variables: From 3fa33f7d5d8e36bb8d3629ef690bc0d5f508cadd Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 11:19:38 +0300 Subject: [PATCH 45/60] fix(ci): add schedule discriminator for scheduled jobs Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/backport.yml | 6 ++++-- .gitlab/ci/jobs/changelog.yml | 7 ++++--- .gitlab/ci/jobs/cleanup.yml | 6 ++++-- .gitlab/ci/jobs/cve-scan.yml | 4 +++- .gitlab/ci/jobs/gitleaks.yml | 2 +- .gitlab/ci/jobs/lint-validate.yml | 6 +++--- .gitlab/ci/jobs/manual-tools.yml | 5 +++-- .gitlab/ci/jobs/precache.yml | 3 ++- .gitlab/ci/jobs/svace.yml | 12 +++++++----- .gitlab/ci/variables.yml | 25 +++++++++++++++++++++++++ 10 files changed, 56 insertions(+), 20 deletions(-) diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index 3ea1eb4574..ec847ddf44 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -49,7 +49,9 @@ backport: - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /backport-release-[0-9]+\.[0-9]+/ when: manual allow_failure: true - # Mode 3: scheduled backport sweep (TODO: future automation). - - if: $CI_PIPELINE_SOURCE == "schedule" + # Mode 3: scheduled backport sweep (TODO: future automation), under the + # dedicated backport-sweep pipeline schedule + # ($SCHEDULE_TYPE == "backport-sweep"). + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "backport-sweep" when: manual allow_failure: true diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index 402d15e936..dfebddc15e 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -60,8 +60,9 @@ changelog:milestone: allow_failure: true # Mode 2: scheduled daily run. We intentionally do NOT auto-open the MR on # schedule; set OPEN_CHANGELOG_MR=true in the schedule definition if you - # want MRs created. - - if: $CI_PIPELINE_SOURCE == "schedule" + # want MRs created. Runs only under the dedicated changelog-milestone + # pipeline schedule ($SCHEDULE_TYPE == "changelog-milestone"). + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "changelog-milestone" when: manual allow_failure: true @@ -82,6 +83,6 @@ changelog:all-active-milestones: OPEN_CHANGELOG_MR: "false" CHANGELOG_BASE_BRANCH: "main" rules: - - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "changelog-all-active-milestones" when: manual allow_failure: true diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml index c9d1870ae7..e371d5491d 100644 --- a/.gitlab/ci/jobs/cleanup.yml +++ b/.gitlab/ci/jobs/cleanup.yml @@ -7,7 +7,9 @@ # # The weekly cadence (GitHub cron "12 0 * * 6") is configured in the # GitLab UI under CI/CD -> Schedules, not here. This job only gates on -# $CI_PIPELINE_SOURCE == "schedule" so it runs under that schedule. +# $CI_PIPELINE_SOURCE == "schedule" AND $SCHEDULE_TYPE == "cleanup" so it +# runs only under the dedicated cleanup schedule and not under any other +# pipeline schedule defined in the project. # # Registry path is composed from the DEV Project Variables the rest of the # pipeline uses: @@ -33,7 +35,7 @@ cleanup: variables: MODULES_MODULE_TAG: v0.0.0-main rules: - - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cleanup" script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh werf - werf cleanup --repo ${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} --without-kube=true --config werf_cleanup.yaml diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml index ea07b1b007..b79c905ac2 100644 --- a/.gitlab/ci/jobs/cve-scan.yml +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -39,6 +39,8 @@ # --------------------------------------------------------------------------- # Scheduled daily scan against main. # Mirrors `cve_scan_daily.yml` cron: "0 02 * * *". +# Runs only under the dedicated cve-scan-daily pipeline schedule +# ($SCHEDULE_TYPE == "cve-scan-daily"). # --------------------------------------------------------------------------- cve:scan:daily: @@ -54,7 +56,7 @@ cve:scan:daily: LATEST_RELEASES_AMOUNT: "5" RELEASE_IN_DEV: "false" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cve-scan-daily"' # --------------------------------------------------------------------------- # Manual scan against a chosen tag or branch. diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index 9def66adc7..821426bb7f 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -64,7 +64,7 @@ gitleaks:full:scheduled: SCAN_MODE: "full" GIT_DEPTH: "0" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "gitleaks-full"' gitleaks:full:manual: extends: .gitleaks_scan diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index f8fb1aa1bf..857a174184 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -48,7 +48,7 @@ lint:no-cyrillic: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - if: "$CI_COMMIT_BRANCH =~ /^release-/" - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "lint-validate"' # --------------------------------------------------------------------------- # doc_changes @@ -201,7 +201,7 @@ lint:go: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "main"' - if: "$CI_COMMIT_BRANCH =~ /^release-/" - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "lint-validate"' # --------------------------------------------------------------------------- # helm_templates @@ -424,4 +424,4 @@ lint:gitlab-ci: paths: - ".gitlab-ci.yml" - ".gitlab/**/*" - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "lint-validate"' diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index 4cc999e897..a32b9e4616 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -44,7 +44,8 @@ mrs:summary: - if: $CI_PIPELINE_SOURCE == "web" when: manual # Scheduled daily run (e.g. 10:00 Moscow time — configure in - # Settings -> CI/CD -> Schedules). - - if: $CI_PIPELINE_SOURCE == "schedule" + # Settings -> CI/CD -> Schedules), under the dedicated mrs-summary + # pipeline schedule ($SCHEDULE_TYPE == "mrs-summary"). + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "mrs-summary" when: manual allow_failure: true diff --git a/.gitlab/ci/jobs/precache.yml b/.gitlab/ci/jobs/precache.yml index 936a77f299..b424b93aea 100644 --- a/.gitlab/ci/jobs/precache.yml +++ b/.gitlab/ci/jobs/precache.yml @@ -7,6 +7,7 @@ # GitLab mapping: # on: schedule (cron: "0 */8 * * *") -> Pipeline Schedule in UI # + rules: schedule +# ($SCHEDULE_TYPE == "precache") # on: workflow_dispatch -> when: manual (Run pipeline) # matrix.branch: [main] -> single .main job # @@ -24,7 +25,7 @@ precache:build:main: variables: MODULES_MODULE_TAG: "${CI_COMMIT_REF_NAME:-main}" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "precache"' - if: '$CI_PIPELINE_SOURCE == "web"' when: manual diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index 7e65018261..d797449d62 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -13,7 +13,9 @@ # for DEV registry variables and job-local rules for schedule/manual runs. # - The analyze step uses the upstream `.svace_analyze` template # (Svace_Analayze.gitlab-ci.yml; note upstream file name typo). -# - Schedule cron: "00 04 * * 6" -> Pipeline Schedule in UI. +# - Schedule cron: "00 04 * * 6" -> Pipeline Schedule in UI, with +# $SCHEDULE_TYPE == "svace" so the svace jobs run only under that +# dedicated schedule and not under any other pipeline schedule. # - workflow_dispatch -> when: manual. # - MR opt-in -> set SVACE_PIPELINE_ENABLED=true when running a pipeline. # - Loop notification kept as a manual / always job. The original GH @@ -46,7 +48,7 @@ svace:set-vars: # overriding their own mr tag. Runs automatically (no `when: manual`) # so the manual svace:build job can consume its artifact. rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' when: always - if: '$CI_PIPELINE_SOURCE == "web"' when: always @@ -95,7 +97,7 @@ svace:build: WERF_VIRTUAL_MERGE: "0" SVACE_ENABLED: "true" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' - if: '$CI_PIPELINE_SOURCE == "web"' when: manual - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' @@ -119,7 +121,7 @@ svace:analyze: - job: svace:build optional: true rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' - if: '$CI_PIPELINE_SOURCE == "web"' when: manual - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $SVACE_PIPELINE_ENABLED == "true"' @@ -159,7 +161,7 @@ svace:notify: --data "$(jq -n --arg text "${MESSAGE}" '{text: $text}')" \ "${LOOP_WEBHOOK_URL}" || echo "Loop webhook failed (non-fatal)" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' when: always - if: '$CI_PIPELINE_SOURCE == "web"' when: always diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index decdd61e06..c5ff3f6685 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -90,3 +90,28 @@ variables: # and matches the behavior of GitHub's actions/setup-go@v5. GOTOOLCHAIN: "auto" WERF_EXPERIMENTAL_IMPORT_BY_SOURCE_IMAGE_TAG: "true" + + # --- Pipeline schedule discriminator --- + # SCHEDULE_TYPE is NOT a fixed value here. It is set per pipeline schedule + # in GitLab UI (Settings -> CI/CD -> Schedules -> -> Variables), + # and each scheduled job gates on its own expected value, e.g.: + # $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cleanup" + # This prevents a job from running under every pipeline schedule that happens + # to fire. Operators must set SCHEDULE_TYPE to exactly one of the values + # below when creating a pipeline schedule: + # + # Job(s) SCHEDULE_TYPE Intended cron + # cleanup cleanup 12 0 * * 6 + # cve:scan:daily cve-scan-daily 0 2 * * * + # svace:* svace 0 4 * * 6 + # gitleaks:full:scheduled gitleaks-full daily + # precache:build:main precache 0 */8 * * * + # changelog:milestone changelog-milestone nightly + # changelog:all-active-milestones changelog-all-active-milestones nightly + # mrs:summary mrs-summary 0 7 * * * + # lint:no-cyrillic / lint:go / lint:gitlab-ci lint-validate as needed + # backport (scheduled sweep) backport-sweep as needed + # + # The global workflow:rules in .gitlab/ci/workflow.yml still create a + # pipeline for any schedule; SCHEDULE_TYPE only narrows which jobs run + # inside it. Manual/MR/web behavior is unaffected. From ff87497afbc0a813aea8fe17827da2f381f1a427 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 11:23:54 +0300 Subject: [PATCH 46/60] fix(ci): correct scheduled changelog generation Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/changelog.yml | 9 +++------ .gitlab/ci/scripts/python/changelog_collect.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index dfebddc15e..a6ad34412a 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -60,11 +60,10 @@ changelog:milestone: allow_failure: true # Mode 2: scheduled daily run. We intentionally do NOT auto-open the MR on # schedule; set OPEN_CHANGELOG_MR=true in the schedule definition if you - # want MRs created. Runs only under the dedicated changelog-milestone - # pipeline schedule ($SCHEDULE_TYPE == "changelog-milestone"). + # want MRs created. Runs automatically under the dedicated + # changelog-milestone pipeline schedule + # ($SCHEDULE_TYPE == "changelog-milestone"). - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "changelog-milestone" - when: manual - allow_failure: true # Optional: process all active milestones (one MR per milestone). # Run on a schedule (e.g. nightly). Iterates over all open milestones and @@ -84,5 +83,3 @@ changelog:all-active-milestones: CHANGELOG_BASE_BRANCH: "main" rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "changelog-all-active-milestones" - when: manual - allow_failure: true diff --git a/.gitlab/ci/scripts/python/changelog_collect.py b/.gitlab/ci/scripts/python/changelog_collect.py index 0d0c6230f3..7a9d240a29 100644 --- a/.gitlab/ci/scripts/python/changelog_collect.py +++ b/.gitlab/ci/scripts/python/changelog_collect.py @@ -138,6 +138,16 @@ def parse_changes_block(block_text: str) -> dict[str, str] | None: return fields +def has_label(mr: dict, label: str) -> bool: + labels = mr.get("labels") or [] + for item in labels: + if isinstance(item, str) and item == label: + return True + if isinstance(item, dict) and item.get("name") == label: + return True + return False + + def collect_entries_for_milestone( api_base: str, project_id: str, milestone_title: str, token: str, allowed_sections: set[str], @@ -159,6 +169,9 @@ def collect_entries_for_milestone( entries: list[dict] = [] for mr in mrs: + if has_label(mr, "changelog"): + log(f"Skipping changelog MR !{mr['iid']}.") + continue description = (mr.get("description") or "").strip() if not description: continue @@ -317,7 +330,6 @@ def push_changelog_mr( server_host: str, token: str, milestone_title: str, - milestone_number: str, base_branch: str, pr_body_path: Path, ) -> None: @@ -349,7 +361,7 @@ def push_changelog_mr( "-o", f"merge_request.label=changelog", "-o", f"merge_request.label=auto", "-o", f"merge_request.label=status/backport", - "-o", f"merge_request.milestone={milestone_number}", + "-o", f"merge_request.milestone={milestone_title}", "-o", f"merge_request.description={pr_body_path.read_text(encoding='utf-8')}", "-o", "merge_request.remove_source_branch", "origin", branch, @@ -449,7 +461,6 @@ def main() -> int: server_host=server_host, token=token, milestone_title=title, - milestone_number=str(iid), base_branch=base_branch, pr_body_path=body_path, ) From 3df6500d08d3b321857cf8d11f1d8c1878064d58 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 11:57:19 +0300 Subject: [PATCH 47/60] fix(ci): classify stuck merge requests by discussion age Signed-off-by: Nikita Korolev --- .gitlab/scripts/js/mrs_notifier.mjs | 155 +++++++++++++---- .gitlab/scripts/js/mrs_notifier.test.mjs | 207 +++++++++++++++++++++-- 2 files changed, 313 insertions(+), 49 deletions(-) diff --git a/.gitlab/scripts/js/mrs_notifier.mjs b/.gitlab/scripts/js/mrs_notifier.mjs index cdf92e0b37..9719ea3db0 100644 --- a/.gitlab/scripts/js/mrs_notifier.mjs +++ b/.gitlab/scripts/js/mrs_notifier.mjs @@ -41,6 +41,7 @@ import axios from 'axios'; import moment from 'moment'; +import { fileURLToPath } from 'node:url'; const PROJECT_ID = process.env.CI_PROJECT_ID; const API_BASE = (process.env.CI_API_V4_URL || '').replace(/\/+$/, ''); @@ -50,13 +51,11 @@ const DOC_REVIEWER = process.env.DOC_REVIEWER || 'z9r5'; const MANAGER_LOOP_NAME = process.env.MANAGER_LOOP_NAME || '@yuriy.milyutin'; const PROJECT = ':dvp: DVP'; +// STUCK_DAYS mirrors .github/scripts/prs_notifier.mjs: a changes-requested +// discussion is considered "stuck" once it stays unresolved for longer than +// this many days (and today is not Monday, see Monday exception below). const STUCK_DAYS = 1.5; -if (!API_BASE || !TOKEN || !PROJECT_ID || !LOOP_URL) { - console.error('ERROR: one of CI_API_V4_URL, GITLAB_API_TOKEN, CI_PROJECT_ID, LOOP_WEBHOOK_URL is not set.'); - process.exit(1); -} - const api = axios.create({ baseURL: API_BASE, headers: { @@ -70,6 +69,13 @@ const REVIEW_REQUIRED = 'review_required'; const READY_TO_MERGE = 'ready_to_merge'; const STUCK = 'stuck'; +function validateEnv() { + if (!API_BASE || !TOKEN || !PROJECT_ID || !LOOP_URL) { + console.error('ERROR: one of CI_API_V4_URL, GITLAB_API_TOKEN, CI_PROJECT_ID, LOOP_WEBHOOK_URL is not set.'); + process.exit(1); + } +} + async function fetchOpenMRs() { const { data } = await api.get(`/projects/${PROJECT_ID}/merge_requests`, { params: { @@ -141,8 +147,9 @@ async function getReviewersInfo(mr) { fetched.push(user); } - const changesRequested = await fetchUnresolvedReviewers(mr); - for (const reviewer of changesRequested) { + const threads = await fetchUnresolvedThreads(mr); + for (const thread of threads) { + const reviewer = thread.author; if (unique.has(reviewer.id)) continue; unique.add(reviewer.id); const details = await fetchUser(reviewer.id); @@ -159,31 +166,82 @@ async function getReviewersInfo(mr) { return info; } -// Approximate GitHub CHANGES_REQUESTED via unresolved discussions. -// GitLab has no native review-state; we treat unresolved resolvable -// discussion threads as a change request from that author. -async function fetchUnresolvedReviewers(mr) { +// Pure helper: extract unresolved discussion threads from raw GitLab +// discussions payload. Returns an array of { author, timestamp } where +// timestamp (ms epoch) is the earliest note timestamp belonging to the +// unresolved discussion, or null when no real timestamp is available. +// +// A thread is considered unresolved when at least one of its notes is +// `resolvable` and not `resolved` (GitLab marks resolution at note level). +// Exported so unit tests can exercise it without network access. +export function extractUnresolvedThreads(discussions) { + const threads = []; + for (const discussion of discussions || []) { + const notes = discussion.notes || []; + if (!notes.length) continue; + const unresolvedNotes = notes.filter((n) => n.resolvable && !n.resolved); + if (!unresolvedNotes.length) continue; + const author = unresolvedNotes[0].author || notes[0].author; + if (!author) continue; + + // Prefer the earliest timestamp among the unresolved resolvable notes; + // fall back to the earliest note in the thread if GitLab only exposes + // note-level resolution metadata inconsistently. + const tsNotes = unresolvedNotes.some((n) => n.created_at) + ? unresolvedNotes.filter((n) => n.created_at) + : notes.filter((n) => n.created_at); + + let timestamp = null; + if (tsNotes.length > 0) { + timestamp = tsNotes + .map((n) => new Date(n.created_at).getTime()) + .reduce((a, b) => (a < b ? a : b), Infinity); + if (!Number.isFinite(timestamp)) timestamp = null; + } + + threads.push({ author, timestamp }); + } + return threads; +} + +// Fetch unresolved resolvable discussion threads for an MR, including the +// real note timestamp used for STUCK classification. +async function fetchUnresolvedThreads(mr) { try { const { data } = await api.get( `/projects/${PROJECT_ID}/merge_requests/${mr.iid}/discussions`, { params: { per_page: 100 } }, ); - const unresolved = new Map(); - for (const discussion of data) { - const notes = discussion.notes || []; - if (!notes.length) continue; - if (!notes.some((n) => n.resolvable && !n.resolved)) continue; - const author = notes[0].author; - if (!author) continue; - unresolved.set(author.id, author); - } - return [...unresolved.values()]; + return extractUnresolvedThreads(data); } catch (err) { console.error(`Error fetching discussions for MR !${mr.iid}: ${err.message}`); return []; } } +// Back-compat wrapper: deduplicated unresolved authors, oldest thread per +// author preserved. Kept for any external caller and for symmetry with the +// GitHub changesRequestedMap semantics. +async function fetchUnresolvedReviewers(mr) { + const threads = await fetchUnresolvedThreads(mr); + const byAuthor = new Map(); + for (const thread of threads) { + const id = thread.author.id; + if (!byAuthor.has(id)) { + byAuthor.set(id, thread); + continue; + } + const existing = byAuthor.get(id); + if ( + existing.timestamp == null + || (thread.timestamp != null && thread.timestamp < existing.timestamp) + ) { + byAuthor.set(id, thread); + } + } + return [...byAuthor.values()].map((t) => t.author); +} + async function fetchApprovals(mr) { try { const { data } = await api.get( @@ -196,22 +254,41 @@ async function fetchApprovals(mr) { } } -function classifyMR(approvedBy, unresolvedAuthors) { +// Classify an MR given its approvers and unresolved discussion threads. +// +// Mirrors .github/scripts/prs_notifier.mjs getPullRequestGroup: +// - STUCK when at least one unresolved discussion is older than STUCK_DAYS +// and today is not Monday (Monday exception: give reviewers a fresh +// start-of-week window). +// - CHANGES_REQUESTED when there are unresolved discussions but none is +// old enough (or a thread has no real timestamp — be conservative and +// avoid false STUCK). +// - READY_TO_MERGE when approved and no unresolved discussions. +// - REVIEW_REQUIRED otherwise. +// +// Exported for unit testing. +export function classifyMR(approvedBy, unresolvedThreads) { const approved = approvedBy.length > 0; - const unresolved = unresolvedAuthors.length > 0; + const unresolved = unresolvedThreads.length > 0; if (unresolved) { - // Check whether any unresolved thread is older than STUCK_DAYS (and not Monday). const now = new Date(); - const stuck = unresolved.some((author) => { - // We don't have note timestamps here; classify as changes_requested and let - // the existence of older discussions be inspected manually. TODO: pull - // discussion.notes[].created_at once we expose it. - const fakeDate = new Date(); - fakeDate.setTime(fakeDate.getTime() - (STUCK_DAYS + 1) * 24 * 60 * 60 * 1000); - return now.getDay() !== 1 && fakeDate < now; - }); - return stuck ? STUCK : CHANGES_REQUESTED; + let areChangesRequested = false; + for (const thread of unresolvedThreads) { + // No real timestamp -> cannot prove it is old enough. Be conservative: + // treat as changes_requested, never STUCK. + if (thread.timestamp == null) { + areChangesRequested = true; + continue; + } + const submittedAt = new Date(thread.timestamp); + submittedAt.setTime(submittedAt.getTime() + STUCK_DAYS * 24 * 60 * 60 * 1000); + if (now.getDay() !== 1 && submittedAt < now) { + return STUCK; + } + areChangesRequested = true; + } + if (areChangesRequested) return CHANGES_REQUESTED; } if (approved) return READY_TO_MERGE; @@ -227,11 +304,11 @@ async function buildSummary(mrs) { }; for (const mr of mrs) { - const [approvals, unresolvedAuthors] = await Promise.all([ + const [approvals, unresolvedThreads] = await Promise.all([ fetchApprovals(mr), - fetchUnresolvedReviewers(mr), + fetchUnresolvedThreads(mr), ]); - const group = classifyMR(approvals, unresolvedAuthors); + const group = classifyMR(approvals, unresolvedThreads); groups[group].push(mr); } @@ -283,6 +360,7 @@ async function sendSummaryToLoop(summary) { } async function run() { + validateEnv(); try { const mrs = await fetchOpenMRs(); const summary = await buildSummary(mrs); @@ -293,4 +371,7 @@ async function run() { } } -run(); +// Auto-run only when executed directly (CI), not when imported by tests. +if (process.argv[1] === fileURLToPath(import.meta.url)) { + run(); +} diff --git a/.gitlab/scripts/js/mrs_notifier.test.mjs b/.gitlab/scripts/js/mrs_notifier.test.mjs index c884399af4..771a85b898 100644 --- a/.gitlab/scripts/js/mrs_notifier.test.mjs +++ b/.gitlab/scripts/js/mrs_notifier.test.mjs @@ -12,18 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Minimal smoke test for mrs_notifier.mjs. +// Unit tests for mrs_notifier.mjs classification logic. // -// mrs_notifier.mjs is a script that auto-runs `run()` at import time and -// exits when required env vars are missing, so it cannot be safely imported -// by a unit test without a refactor. This smoke test therefore: -// - asserts the file exists and is non-empty; -// - syntax-checks it with `node --check` (no execution, no side effects); -// - asserts the expected entry points and env-var contract are present. -// -// TODO: refactor mrs_notifier.mjs to export pure helpers (e.g. classifyMR) -// and guard the `run()` call behind a direct-invocation check, then add real -// unit tests over the classification logic here. +// mrs_notifier.mjs guards its `run()` call behind a direct-invocation check +// and validates env vars only inside `run()`, so it can be safely imported +// here without triggering network calls or process.exit. The pure helpers +// `classifyMR` and `extractUnresolvedThreads` are exercised directly with +// synthetic data — no network access required. import { test } from 'node:test'; import assert from 'node:assert/strict'; @@ -32,9 +27,34 @@ import { execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import path from 'node:path'; +import { classifyMR, extractUnresolvedThreads } from './mrs_notifier.mjs'; + const __dirname = path.dirname(fileURLToPath(import.meta.url)); const MODULE_PATH = path.join(__dirname, 'mrs_notifier.mjs'); +const STUCK_DAYS = 1.5; +const DAY_MS = 24 * 60 * 60 * 1000; + +function author(id, username) { + return { id, username, name: username }; +} + +function note({ author: a, createdAt, resolvable = true, resolved = false }) { + return { + author: a, + created_at: createdAt, + resolvable, + resolved, + }; +} + +// An ISO timestamp `days` days ago from now. +function daysAgo(days) { + return new Date(Date.now() - days * DAY_MS).toISOString(); +} + +// ---- smoke / structural tests (kept from the original test file) ---- + test('mrs_notifier.mjs exists and is non-empty', () => { const stat = statSync(MODULE_PATH); assert.ok(stat.isFile(), 'mrs_notifier.mjs should be a file'); @@ -43,7 +63,6 @@ test('mrs_notifier.mjs exists and is non-empty', () => { }); test('mrs_notifier.mjs is syntactically valid (node --check)', () => { - // Syntax-check without executing: avoids env-var / network side effects. execFileSync('node', ['--check', MODULE_PATH], { stdio: 'pipe' }); }); @@ -63,3 +82,167 @@ test('mrs_notifier.mjs references the documented env-var contract', () => { assert.ok(src.includes(envVar), `expected reference to ${envVar}`); } }); + +test('mrs_notifier.mjs does not fabricate a fake old date in classifyMR', () => { + const src = readFileSync(MODULE_PATH, 'utf8'); + assert.ok( + !/fakeDate/.test(src), + 'classifyMR must not use a fabricated fake date', + ); +}); + +// ---- classifyMR behavior ---- + +test('classifyMR: no unresolved, approved => ready_to_merge', () => { + const approved = [author(1, 'alice')]; + assert.equal(classifyMR(approved, []), 'ready_to_merge'); +}); + +test('classifyMR: no unresolved, no approved => review_required', () => { + assert.equal(classifyMR([], []), 'review_required'); +}); + +test('classifyMR: recent unresolved => changes_requested', () => { + const threads = [{ author: author(2, 'bob'), timestamp: Date.now() }]; + assert.equal(classifyMR([], threads), 'changes_requested'); +}); + +test('classifyMR: recent unresolved with approvals still changes_requested', () => { + // Unresolved discussions take precedence over approvals, matching GitHub. + const approved = [author(1, 'alice')]; + const threads = [{ author: author(2, 'bob'), timestamp: Date.now() }]; + assert.equal(classifyMR(approved, threads), 'changes_requested'); +}); + +test('classifyMR: old unresolved => stuck', () => { + const threads = [ + { author: author(2, 'bob'), timestamp: Date.now() - (STUCK_DAYS + 1) * DAY_MS }, + ]; + // Only assert STUCK when today is not Monday; on Monday the exception + // kicks in and the result is changes_requested. + if (new Date().getDay() !== 1) { + assert.equal(classifyMR([], threads), 'stuck'); + } else { + assert.equal(classifyMR([], threads), 'changes_requested'); + } +}); + +test('classifyMR: exactly STUCK_DAYS boundary is not stuck yet', () => { + // timestamp + STUCK_DAYS == now => submittedAt < now is false => not stuck. + const threads = [ + { author: author(2, 'bob'), timestamp: Date.now() - STUCK_DAYS * DAY_MS + 1000 }, + ]; + assert.equal(classifyMR([], threads), 'changes_requested'); +}); + +test('classifyMR: no timestamp => changes_requested (never stuck)', () => { + const threads = [{ author: author(2, 'bob'), timestamp: null }]; + assert.equal(classifyMR([], threads), 'changes_requested'); +}); + +test('classifyMR: one old + one no-timestamp thread => stuck (when not Monday)', () => { + const threads = [ + { author: author(2, 'bob'), timestamp: null }, + { author: author(3, 'carol'), timestamp: Date.now() - (STUCK_DAYS + 2) * DAY_MS }, + ]; + if (new Date().getDay() !== 1) { + assert.equal(classifyMR([], threads), 'stuck'); + } else { + assert.equal(classifyMR([], threads), 'changes_requested'); + } +}); + +// ---- extractUnresolvedThreads ---- + +test('extractUnresolvedThreads: skips fully resolved threads', () => { + const discussions = [ + { + notes: [ + note({ author: author(2, 'bob'), createdAt: daysAgo(5), resolved: true }), + ], + }, + ]; + assert.deepEqual(extractUnresolvedThreads(discussions), []); +}); + +test('extractUnresolvedThreads: skips non-resolvable threads', () => { + const discussions = [ + { + notes: [ + note({ author: author(2, 'bob'), createdAt: daysAgo(5), resolvable: false }), + ], + }, + ]; + assert.deepEqual(extractUnresolvedThreads(discussions), []); +}); + +test('extractUnresolvedThreads: returns earliest unresolved note timestamp', () => { + const old = daysAgo(5); + const recent = daysAgo(1); + const discussions = [ + { + notes: [ + note({ author: author(2, 'bob'), createdAt: recent, resolved: false }), + note({ author: author(2, 'bob'), createdAt: old, resolved: false }), + ], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + assert.equal(threads.length, 1); + assert.equal(threads[0].author.id, 2); + assert.equal(threads[0].timestamp, new Date(old).getTime()); +}); + +test('extractUnresolvedThreads: prefers unresolved note timestamp over resolved ones', () => { + const resolvedOld = daysAgo(10); + const unresolvedRecent = daysAgo(1); + const discussions = [ + { + notes: [ + note({ author: author(1, 'alice'), createdAt: resolvedOld, resolved: true }), + note({ author: author(2, 'bob'), createdAt: unresolvedRecent, resolved: false }), + ], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + assert.equal(threads.length, 1); + assert.equal(threads[0].author.id, 2); + assert.equal(threads[0].timestamp, new Date(unresolvedRecent).getTime()); +}); + +test('extractUnresolvedThreads: null timestamp when no created_at present', () => { + const discussions = [ + { + notes: [note({ author: author(2, 'bob'), createdAt: undefined, resolved: false })], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + assert.equal(threads.length, 1); + assert.equal(threads[0].timestamp, null); +}); + +test('extractUnresolvedThreads: empty / null input is safe', () => { + assert.deepEqual(extractUnresolvedThreads([]), []); + assert.deepEqual(extractUnresolvedThreads(null), []); + assert.deepEqual(extractUnresolvedThreads(undefined), []); +}); + +test('extractUnresolvedThreads + classifyMR integration: old unresolved => stuck', () => { + const discussions = [ + { + notes: [ + note({ + author: author(2, 'bob'), + createdAt: daysAgo(STUCK_DAYS + 1), + resolved: false, + }), + ], + }, + ]; + const threads = extractUnresolvedThreads(discussions); + if (new Date().getDay() !== 1) { + assert.equal(classifyMR([], threads), 'stuck'); + } else { + assert.equal(classifyMR([], threads), 'changes_requested'); + } +}); From 0cd11d3ed387e9e78806d0a90772c879eff54287 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 12:44:14 +0300 Subject: [PATCH 48/60] fix(ci): restore milestone-based backport flow Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/backport.yml | 25 ++- .gitlab/ci/scripts/bash/backport.sh | 256 +++++++++++++++++++++++----- 2 files changed, 235 insertions(+), 46 deletions(-) diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index ec847ddf44..6a605cfb2d 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -21,8 +21,17 @@ # # Triggers (per plan §11.9.4): # 1. Manual pipeline (Run pipeline UI) with variable TARGET_BRANCH=release-1.21. -# 2. Manual pipeline with the backport-release-X.Y label detected on the MR -# (GitLab does NOT auto-trigger on label change; documented TODO). +# 2. Manual pipeline on an MR carrying the `status/backport` label. +# GitLab does NOT auto-trigger pipelines on label change; the user must +# press "Run pipeline" on the MR (TODO: webhook-listener per plan §7). +# The target branch is derived from the source MR milestone title +# (vX.Y.Z or X.Y.Z -> release-X.Y); TARGET_BRANCH overrides this. +# +# After the backport job finishes, backport.sh updates the source MR: +# - always removes `status/backport`; +# - on success adds `status/backport/success` and comments with the +# backport MR link; +# - on failure adds `status/backport/failed` and comments with the job link. # # Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). @@ -35,18 +44,20 @@ backport: script: - bash .gitlab/ci/scripts/bash/backport.sh # allow_failure: true on every rule: `when: manual` inside `rules:` defaults - # to allow_failure: false, so on an MR carrying a backport-release-X.Y label + # to allow_failure: false, so on an MR carrying the `status/backport` label # this unplayed manual job would block the whole MR pipeline (test/build/...). # Backport is an opt-in side action, so it must never gate the pipeline. rules: - # Mode 1: explicit manual run with TARGET_BRANCH provided via UI. + # Mode 1: explicit manual run with TARGET_BRANCH provided via UI (overrides + # milestone-based inference). - if: $TARGET_BRANCH when: manual allow_failure: true - # Mode 2: MR with a backport-release-X.Y label. GitLab does NOT auto-run + # Mode 2: MR with the `status/backport` label. GitLab does NOT auto-run # pipelines on label change; user has to press "Run pipeline" on the MR - # (TODO: webhook-listener per migration plan §7). - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /backport-release-[0-9]+\.[0-9]+/ + # (TODO: webhook-listener per migration plan §7). The target branch is + # derived from the source MR milestone by backport.sh. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,\s*)status\/backport(,|$)/ when: manual allow_failure: true # Mode 3: scheduled backport sweep (TODO: future automation), under the diff --git a/.gitlab/ci/scripts/bash/backport.sh b/.gitlab/ci/scripts/bash/backport.sh index 2361b974e3..f50aebf946 100644 --- a/.gitlab/ci/scripts/bash/backport.sh +++ b/.gitlab/ci/scripts/bash/backport.sh @@ -28,14 +28,24 @@ # 3. push a backport branch, # 4. open an MR to the target release branch via push options / API. # +# Target branch resolution (priority): +# 1. TARGET_BRANCH env var (manual "Run pipeline" override). +# 2. Source MR milestone title: vX.Y.Z or X.Y.Z -> release-X.Y. +# +# After the backport attempt, the source MR receives feedback matching the +# GitHub flow: +# - always remove `status/backport` (when source MR iid is known); +# - success: add `status/backport/success` + comment with backport MR link; +# - failure: add `status/backport/failed` + comment with error/job link. +# Feedback failures are logged but never mask the backport outcome on +# success; on the failure path they are best-effort and the script still +# exits non-zero. +# # Required environment: # GITLAB_API_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, CI_SERVER_HOST, -# CI_PROJECT_PATH, TARGET_BRANCH (e.g. release-1.21) +# CI_PROJECT_PATH, CI_PROJECT_DIR # SOURCE_MR_IID (optional; defaults to CI_MERGE_REQUEST_IID) -# -# Optional environment: -# CI_MERGE_REQUEST_LABELS — used to detect a backport-release-X.Y label -# if TARGET_BRANCH is not provided explicitly (manual pipeline UI fallback). +# TARGET_BRANCH (optional; otherwise derived from the source MR milestone) set -euo pipefail @@ -43,53 +53,192 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=.gitlab/ci/scripts/bash/lib/api.sh source "${SCRIPT_DIR}/lib/api.sh" -gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_SERVER_HOST CI_PROJECT_PATH +gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_SERVER_HOST CI_PROJECT_PATH CI_PROJECT_DIR + +# Status labels (with slashes; used as-is in JSON payloads, comma is the only +# separator GitLab uses for add_labels/remove_labels). +BACKPORT_TRIGGER_LABEL="status/backport" +BACKPORT_SUCCESS_LABEL="status/backport/success" +BACKPORT_FAILED_LABEL="status/backport/failed" -# Determine TARGET_BRANCH (priority: explicit var > label-based inference). +# Resolved later; used by the failure-feedback trap. +SOURCE_MR_IID="${SOURCE_MR_IID:-}" TARGET_BRANCH="${TARGET_BRANCH:-}" -if [[ -z "$TARGET_BRANCH" ]]; then - if [[ "${CI_MERGE_REQUEST_LABELS:-}" =~ backport-release-([0-9]+\.[0-9]+) ]]; then - TARGET_BRANCH="release-${BASH_REMATCH[1]}" +BACKPORT_MR_IID="" +BACKPORT_MR_URL="" +FAILURE_CONTEXT="" +# Set to "success" only after the backport MR is created cleanly without conflicts. +BACKPORT_RESULT="" + +# --------------------------------------------------------------------------- +# Source MR feedback helpers (best-effort: never abort the script). +# --------------------------------------------------------------------------- + +# Remove a label from the source MR. GitLab PUT /merge_requests/:iid supports +# remove_labels with comma-separated names; a slash inside a label name is fine +# because the separator is the comma, not the slash. +mr_remove_label() { + local iid="$1" + local label="$2" + local payload + payload="$(jq -n --arg l "$label" '{remove_labels: $l}')" + if ! api PUT "/projects/${CI_PROJECT_ID}/merge_requests/${iid}" --data "${payload}" >/dev/null; then + echo "WARNING: failed to remove label '${label}' from MR !${iid}." >&2 + return 0 fi -fi + echo "Removed label '${label}' from MR !${iid}." +} -if [[ -z "$TARGET_BRANCH" ]]; then - echo "ERROR: TARGET_BRANCH is required (e.g. release-1.21)." >&2 - echo "Set it via 'Run pipeline' UI, or add a backport-release-X.Y label to the MR." >&2 - exit 1 -fi -if ! [[ "$TARGET_BRANCH" =~ ^release-[0-9]+\.[0-9]+$ ]]; then - echo "ERROR: TARGET_BRANCH='${TARGET_BRANCH}' does not match ^release-[0-9]+\.[0-9]+\$" >&2 - exit 1 -fi +# Add a label to the source MR. +mr_add_label() { + local iid="$1" + local label="$2" + local payload + payload="$(jq -n --arg l "$label" '{add_labels: $l}')" + if ! api PUT "/projects/${CI_PROJECT_ID}/merge_requests/${iid}" --data "${payload}" >/dev/null; then + echo "WARNING: failed to add label '${label}' to MR !${iid}." >&2 + return 0 + fi + echo "Added label '${label}' to MR !${iid}." +} + +# Create a note (comment) on the source MR. +mr_create_note() { + local iid="$1" + local body="$2" + local payload + payload="$(jq -n --arg b "$body" '{body: $b}')" + if ! api POST "/projects/${CI_PROJECT_ID}/merge_requests/${iid}/notes" --data "${payload}" >/dev/null; then + echo "WARNING: failed to create note on MR !${iid}." >&2 + return 0 + fi + echo "Created note on MR !${iid}." +} + +# Report failure feedback to the source MR (best-effort). +report_failure() { + if [[ -z "${SOURCE_MR_IID}" ]]; then + echo "No source MR iid known; cannot report failure feedback." >&2 + return 0 + fi + local job_url="${CI_JOB_URL:-${CI_PIPELINE_URL:-}}" + local body="Backport to \`${TARGET_BRANCH:-unknown}\` failed." + if [[ -n "$job_url" ]]; then + body="${body} See job: ${job_url}" + fi + if [[ -n "$FAILURE_CONTEXT" ]]; then + body="${body} + +${FAILURE_CONTEXT}" + fi + if [[ -n "$BACKPORT_MR_URL" ]]; then + body="${body} + +A backport MR was created for manual resolution: ${BACKPORT_MR_URL}" + fi + mr_remove_label "${SOURCE_MR_IID}" "${BACKPORT_TRIGGER_LABEL}" + mr_add_label "${SOURCE_MR_IID}" "${BACKPORT_FAILED_LABEL}" + mr_create_note "${SOURCE_MR_IID}" "${body}" +} + +# Report success feedback to the source MR (best-effort). +report_success() { + if [[ -z "${SOURCE_MR_IID}" ]]; then + echo "No source MR iid known; cannot report success feedback." >&2 + return 0 + fi + local body="Backport to \`${TARGET_BRANCH}\` successful: ${BACKPORT_MR_URL:-!(unknown)}" + mr_remove_label "${SOURCE_MR_IID}" "${BACKPORT_TRIGGER_LABEL}" + mr_add_label "${SOURCE_MR_IID}" "${BACKPORT_SUCCESS_LABEL}" + mr_create_note "${SOURCE_MR_IID}" "${body}" +} +# EXIT trap: route to success/failure feedback. Preserves the exit code on +# the failure path; feedback errors never override the backport outcome. +on_exit() { + local rc=$? + # Clean exit (success) — nothing more to do. + if [[ $rc -eq 0 || "${BACKPORT_RESULT}" == "success" ]]; then + exit 0 + fi + set +e + echo "Backport failed (exit code ${rc}); reporting failure feedback to source MR." >&2 + report_failure + exit "$rc" +} +trap on_exit EXIT + +# --------------------------------------------------------------------------- +# Resolve the source MR iid. +# --------------------------------------------------------------------------- SOURCE_MR_IID="${SOURCE_MR_IID:-${CI_MERGE_REQUEST_IID:-}}" if [[ -z "$SOURCE_MR_IID" ]]; then echo "ERROR: SOURCE_MR_IID is required (CI_MERGE_REQUEST_IID unset and no explicit var)." >&2 exit 1 fi - -echo "Backport target: ${TARGET_BRANCH}" echo "Source MR: !${SOURCE_MR_IID}" -# 1) Read source MR to get the merged commit SHA. +# --------------------------------------------------------------------------- +# Read source MR (merged commit SHA + milestone). +# --------------------------------------------------------------------------- mr_path="/projects/${CI_PROJECT_ID}/merge_requests/${SOURCE_MR_IID}" mr_json="$(api GET "${mr_path}")" + sha="$(printf '%s' "$mr_json" | jq -r '.merge_commit_sha // .sha // empty')" if [[ -z "$sha" || "$sha" == "null" ]]; then echo "ERROR: could not extract SHA from MR !${SOURCE_MR_IID} (not merged yet?)." >&2 + FAILURE_CONTEXT="could not extract merged commit SHA from MR !${SOURCE_MR_IID}" exit 1 fi echo "Source commit SHA: ${sha}" -# 2) Verify target branch exists (gives a clearer error than a git fetch failure). +milestone_title="$(printf '%s' "$mr_json" | jq -r '.milestone.title // empty')" + +# --------------------------------------------------------------------------- +# Resolve TARGET_BRANCH (priority: explicit var > milestone-based inference). +# --------------------------------------------------------------------------- +TARGET_BRANCH="${TARGET_BRANCH:-}" +if [[ -z "$TARGET_BRANCH" ]]; then + if [[ -z "$milestone_title" || "$milestone_title" == "null" ]]; then + echo "ERROR: source MR !${SOURCE_MR_IID} has no milestone; cannot derive target branch." >&2 + echo " Set TARGET_BRANCH or assign a milestone to the source MR." >&2 + FAILURE_CONTEXT="source MR !${SOURCE_MR_IID} has no milestone" + exit 1 + fi + # Match vX.Y.Z or X.Y.Z and keep the X.Y minor (mirrors the GitHub workflow). + if [[ "$milestone_title" =~ v?([0-9]+\.[0-9]+)\.[0-9]+ ]]; then + TARGET_BRANCH="release-${BASH_REMATCH[1]}" + else + echo "ERROR: milestone '${milestone_title}' does not match v?X.Y.Z format." >&2 + FAILURE_CONTEXT="invalid milestone format: ${milestone_title}" + exit 1 + fi +fi + +if ! [[ "$TARGET_BRANCH" =~ ^release-[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: TARGET_BRANCH='${TARGET_BRANCH}' does not match ^release-[0-9]+\\.[0-9]+\$" >&2 + FAILURE_CONTEXT="invalid TARGET_BRANCH format: ${TARGET_BRANCH}" + exit 1 +fi + +echo "Backport target: ${TARGET_BRANCH}" +if [[ -n "$milestone_title" && "$milestone_title" != "null" ]]; then + echo "Source MR milestone: ${milestone_title}" +fi + +# --------------------------------------------------------------------------- +# Verify target branch exists (clearer error than a git fetch failure). +# --------------------------------------------------------------------------- if ! api GET "/projects/${CI_PROJECT_ID}/repository/branches/${TARGET_BRANCH}" >/dev/null; then echo "ERROR: target branch '${TARGET_BRANCH}' does not exist in this project." >&2 + FAILURE_CONTEXT="target branch '${TARGET_BRANCH}' does not exist" exit 1 fi -# 3) Use the runner workspace (GitLab already cloned the repo here). -cd "${CI_PROJECT_DIR:?CI_PROJECT_DIR is required}" +# --------------------------------------------------------------------------- +# Cherry-pick the merged commit onto the target branch. +# --------------------------------------------------------------------------- +cd "${CI_PROJECT_DIR}" git config user.email "ci-backport@flant.com" git config user.name "GitLab CI Backport Bot" @@ -105,9 +254,11 @@ echo "Creating backport branch: ${BACKPORT_BRANCH}" git checkout -B "${BACKPORT_BRANCH}" "origin/${TARGET_BRANCH}" CONFLICT_MARKER="" -# 4) Cherry-pick. GIT_SEQUENCE_EDITOR=/dev/null drops the default commit message -# editor so the script can run unattended. +CONFLICTED=0 +# GIT_SEQUENCE_EDITOR=true drops the default commit message editor so the +# script can run unattended. GIT_SEQUENCE_EDITOR=true git cherry-pick -x "${sha}" || { + CONFLICTED=1 CONFLICT_MARKER=" **Conflicts detected** — please resolve manually, amend the commit, and force-push. " @@ -119,14 +270,15 @@ GIT_SEQUENCE_EDITOR=true git cherry-pick -x "${sha}" || { fi } -# 5) Push with push options to create MR in one step. +# --------------------------------------------------------------------------- +# Push the backport branch and ensure a backport MR exists. +# --------------------------------------------------------------------------- DESCRIPTION="Backport !${SOURCE_MR_IID} (${sha}) to ${TARGET_BRANCH}. Auto-generated by GitLab CI backport job. ${CONFLICT_MARKER}" -# Build payload for a single API call to create MR after push, because push -# options are limited and we want fine-grained control of body/labels. +# Push with merge_request.* push options to create the MR in one step. push_output="$(git push --force-with-lease \ -o merge_request.create \ -o merge_request.target="${TARGET_BRANCH}" \ @@ -139,13 +291,16 @@ push_output="$(git push --force-with-lease \ origin "${BACKPORT_BRANCH}" 2>&1 || true)" echo "${push_output}" -# 6) As a fallback (push options ignored on some GitLab versions), explicitly -# POST the MR via API if push did not create one. We search for an open MR -# from our branch first; if absent, we create it. +# Fallback (push options are ignored on some GitLab versions): search for an +# open MR from our branch; if absent, create it via API. +backport_mr_json="" existing_mr_iid="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests?source_branch=${BACKPORT_BRANCH}&state=opened" \ | jq -r 'if type == "array" and length > 0 then .[0].iid else empty end')" -if [[ -z "$existing_mr_iid" ]]; then +if [[ -n "$existing_mr_iid" ]]; then + echo "Backport MR already exists: !${existing_mr_iid}" + backport_mr_json="$(api GET "/projects/${CI_PROJECT_ID}/merge_requests/${existing_mr_iid}")" +else echo "Push options did not create MR; creating via API..." payload="$(jq -n \ --arg src "${BACKPORT_BRANCH}" \ @@ -153,8 +308,31 @@ if [[ -z "$existing_mr_iid" ]]; then --arg title "Backport !${SOURCE_MR_IID} to ${TARGET_BRANCH}" \ --arg desc "${DESCRIPTION}" \ '{source_branch: $src, target_branch: $tgt, title: $title, description: $desc, remove_source_branch: true, labels: "backport,auto"}')" - api POST "/projects/${CI_PROJECT_ID}/merge_requests" --data "${payload}" \ - | jq -r '"Created MR !\(.iid) at \(.web_url)"' -else - echo "Backport MR already exists: !${existing_mr_iid}" + backport_mr_json="$(api POST "/projects/${CI_PROJECT_ID}/merge_requests" --data "${payload}")" +fi + +BACKPORT_MR_IID="$(printf '%s' "$backport_mr_json" | jq -r '.iid // empty')" +BACKPORT_MR_URL="$(printf '%s' "$backport_mr_json" | jq -r '.web_url // empty')" +if [[ -z "$BACKPORT_MR_IID" || "$BACKPORT_MR_IID" == "null" ]]; then + echo "ERROR: could not determine backport MR iid." >&2 + FAILURE_CONTEXT="backport MR creation did not return an iid" + exit 1 fi +echo "Backport MR: !${BACKPORT_MR_IID} (${BACKPORT_MR_URL})" + +# --------------------------------------------------------------------------- +# Report outcome. +# --------------------------------------------------------------------------- +if [[ "$CONFLICTED" == "1" ]]; then + # A backport MR exists for manual resolution, but the cherry-pick had + # conflicts, so this is treated as a failure (matches the GitHub flow). + FAILURE_CONTEXT="cherry-pick had conflicts; resolve them in the backport MR." + exit 1 +fi + +BACKPORT_RESULT="success" +# Feedback failures must not mask the successful backport. +set +e +report_success +set -e +echo "Backport completed successfully." From bf2cbb57072d9ad222448ca89fe04ed8114ca280 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 15:08:54 +0300 Subject: [PATCH 49/60] test(ci): port real JS/Python script unit tests, drop smoke-only jobs The GitHub test_scripts_{js,python} jobs ran full unit suites, but the GitLab migration had degraded them to a JS smoke test and a Python py_compile check (m9e.5.16). The original GH *.test.js / charts_test.py all live under e2e/ (out of migration scope); the migrated scripts (mrs_notifier.mjs, changelog_collect.py, check_changelog_entry.py) never had upstream tests, so port = write real unit tests for them. Python (.gitlab/ci/scripts/python/tests/, 44 tests, stdlib unittest): - check_changelog_entry: block regex, parse_block, load_allowed_sections, validate_block (section/type/summary/impact_level rules incl. :low pin and the feature/fix-only ALLOWED_TYPES guard from 5.4). - changelog_collect: next_link, parse_changes_block, has_label, group_entries, yaml_summary_scalar quoting, render_yaml (deckhouse schema, mr-iid ordering, :low strip, unsupported-type skip), render_markdown, minor_version_from_tag. - test:scripts:python now runs py_compile + `unittest discover`, no longer allow_failure. JS (mrs_notifier): export shouldNotifyMR (open-MR filter) and formatUser as pure helpers; add unit tests for both (29 tests total). Drop the stale "smoke test / cannot be imported" framing in the job comment. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/test-scripts-js.yml | 11 +- .gitlab/ci/jobs/test-scripts-python.yml | 17 +- .gitlab/ci/scripts/python/tests/__init__.py | 13 ++ .../python/tests/test_changelog_collect.py | 190 ++++++++++++++++++ .../tests/test_check_changelog_entry.py | 144 +++++++++++++ .gitlab/scripts/js/mrs_notifier.mjs | 28 ++- .gitlab/scripts/js/mrs_notifier.test.mjs | 66 +++++- 7 files changed, 444 insertions(+), 25 deletions(-) create mode 100644 .gitlab/ci/scripts/python/tests/__init__.py create mode 100644 .gitlab/ci/scripts/python/tests/test_changelog_collect.py create mode 100644 .gitlab/ci/scripts/python/tests/test_check_changelog_entry.py diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml index c11884d603..98185f7f66 100644 --- a/.gitlab/ci/jobs/test-scripts-js.yml +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -12,16 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# JS smoke tests for .gitlab/scripts/js. +# JS unit tests for .gitlab/scripts/js. # # Ports the GitHub Actions `test_scripts_js` job from # .github/workflows/dev_module_build.yml, which ran `npm test` in # .github/scripts/js with Node.js 24. The GitLab counterpart lives in # .gitlab/scripts/js (mrs_notifier.mjs); its `npm test` runs node:test over -# mrs_notifier.test.mjs, a minimal smoke test (syntax check + structural -# assertions) because mrs_notifier.mjs auto-runs at import time and cannot -# yet be imported by a unit test. See mrs_notifier.test.mjs for the TODO -# on adding real unit tests. +# mrs_notifier.test.mjs. mrs_notifier.mjs guards run() behind a +# direct-invocation check and exports its pure helpers (classifyMR, +# extractUnresolvedThreads, shouldNotifyMR, formatUser), so the suite imports +# the module and exercises real classification/filter/formatting logic with +# synthetic data (no network), alongside structural smoke assertions. # # The shell runner host does not ship Node.js, so the job runs inside the # official `node:${NODE_VERSION}` image via `docker run`, mirroring how diff --git a/.gitlab/ci/jobs/test-scripts-python.yml b/.gitlab/ci/jobs/test-scripts-python.yml index 88cd410959..154f8d25ef 100644 --- a/.gitlab/ci/jobs/test-scripts-python.yml +++ b/.gitlab/ci/jobs/test-scripts-python.yml @@ -12,25 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Python syntax smoke check for .gitlab/ci/scripts/python. +# Python unit tests for .gitlab/ci/scripts/python. # # Ports the GitHub Actions `test_scripts_python` job from # .github/workflows/dev_module_build.yml, which ran `python -m unittest` in # .github/scripts/python. The GitLab Python scripts -# (changelog_collect.py, check_changelog_entry.py) currently have no unit -# tests, so this job is a non-blocking `py_compile` syntax check only. -# TODO: add real unit tests under .gitlab/ci/scripts/python/tests/ and -# switch this job to `python3 -m unittest discover`. +# (changelog_collect.py, check_changelog_entry.py) ship real unittest suites +# under .gitlab/ci/scripts/python/tests/ covering the pure parse/validate/ +# render helpers (no network access). A py_compile pass first catches syntax +# errors in any script not reached by an import; then unittest discover runs +# the suites. Standard library only — no third-party deps to install. # # MR-gated via .dev. test:scripts:python: stage: test interruptible: true - allow_failure: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh python3 script: - python3 -m py_compile .gitlab/ci/scripts/python/*.py + - >- + python3 -m unittest discover + -s .gitlab/ci/scripts/python/tests + -t .gitlab/ci/scripts/python + -p 'test_*.py' extends: - .dev diff --git a/.gitlab/ci/scripts/python/tests/__init__.py b/.gitlab/ci/scripts/python/tests/__init__.py new file mode 100644 index 0000000000..54b1119619 --- /dev/null +++ b/.gitlab/ci/scripts/python/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/.gitlab/ci/scripts/python/tests/test_changelog_collect.py b/.gitlab/ci/scripts/python/tests/test_changelog_collect.py new file mode 100644 index 0000000000..a2ab6404f3 --- /dev/null +++ b/.gitlab/ci/scripts/python/tests/test_changelog_collect.py @@ -0,0 +1,190 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for changelog_collect.py pure helpers. + +main() is guarded by `if __name__ == "__main__"`, so the import is side-effect +free. The network/git helpers (api_get_paginated, push_changelog_mr) are not +exercised here; everything below is the pure parse/group/render logic that +produces the deckhouse-schema CHANGELOG-*.yml and the .md summary. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import changelog_collect as cl # noqa: E402 + + +def entry(section, type_, summary, mr_iid, impact_level="high", mr_url=None): + return { + "section": section, + "type": type_, + "summary": summary, + "impact_level": impact_level, + "mr_iid": mr_iid, + "mr_title": f"MR {mr_iid}", + "mr_url": mr_url or f"https://gl/-/merge_requests/{mr_iid}", + "author": "alice", + } + + +class NextLinkTest(unittest.TestCase): + def test_extracts_next_url(self): + header = ( + '; rel="prev", ' + '; rel="next"' + ) + self.assertEqual(cl.next_link(header), "https://gl/api/v4/x?page=3") + + def test_no_next_returns_empty(self): + header = '; rel="prev"' + self.assertEqual(cl.next_link(header), "") + + def test_empty_header(self): + self.assertEqual(cl.next_link(""), "") + + +class ParseChangesBlockTest(unittest.TestCase): + def test_parses_required_keys(self): + parsed = cl.parse_changes_block("section: vm\ntype: fix\nsummary: did it") + self.assertEqual(parsed["section"], "vm") + self.assertEqual(parsed["type"], "fix") + self.assertEqual(parsed["summary"], "did it") + + def test_returns_none_when_required_key_missing(self): + # summary missing -> not a valid changes block. + self.assertIsNone(cl.parse_changes_block("section: vm\ntype: fix")) + + def test_keeps_optional_keys(self): + parsed = cl.parse_changes_block( + "section: vm\ntype: fix\nsummary: s\nimpact_level: low" + ) + self.assertEqual(parsed["impact_level"], "low") + + +class HasLabelTest(unittest.TestCase): + def test_string_labels(self): + self.assertTrue(cl.has_label({"labels": ["changelog", "auto"]}, "changelog")) + self.assertFalse(cl.has_label({"labels": ["auto"]}, "changelog")) + + def test_dict_labels(self): + self.assertTrue( + cl.has_label({"labels": [{"name": "changelog"}]}, "changelog") + ) + + def test_missing_labels_key(self): + self.assertFalse(cl.has_label({}, "changelog")) + + +class GroupEntriesTest(unittest.TestCase): + def test_groups_by_section(self): + entries = [ + entry("vm", "fix", "a", 1), + entry("vm", "feature", "b", 2), + entry("core", "fix", "c", 3), + ] + grouped = cl.group_entries(entries) + self.assertEqual(len(grouped["vm"]), 2) + self.assertEqual(len(grouped["core"]), 1) + + +class YamlSummaryScalarTest(unittest.TestCase): + def test_plain_when_safe(self): + self.assertEqual(cl.yaml_summary_scalar("simple summary"), "simple summary") + + def test_empty_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar(""), '""') + + def test_colon_space_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("fix: thing"), '"fix: thing"') + + def test_leading_special_char_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("- dash start"), '"- dash start"') + + def test_trailing_space_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("trailing "), '"trailing "') + + def test_hash_comment_is_quoted(self): + self.assertEqual(cl.yaml_summary_scalar("note #5"), '"note #5"') + + +class RenderYamlTest(unittest.TestCase): + def test_empty_entries_render_empty_mapping(self): + self.assertEqual(cl.render_yaml([], "v1.21.0"), "{}\n\n") + + def test_groups_into_features_and_fixes(self): + entries = [ + entry("vm", "feature", "added X", 10), + entry("vm", "fix", "fixed Y", 11), + ] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn("vm:", out) + self.assertIn(" features:", out) + self.assertIn(" fixes:", out) + self.assertIn(" - summary: added X", out) + self.assertIn(" pull_request: https://gl/-/merge_requests/10", out) + + def test_entries_ordered_by_mr_iid_descending(self): + entries = [ + entry("vm", "fix", "older", 5), + entry("vm", "fix", "newer", 9), + ] + out = cl.render_yaml(entries, "v1.21.0") + self.assertLess(out.index("newer"), out.index("older")) + + def test_sections_sorted_alphabetically(self): + entries = [ + entry("vm", "fix", "v", 1), + entry("core", "fix", "c", 2), + ] + out = cl.render_yaml(entries, "v1.21.0") + self.assertLess(out.index("core:"), out.index("vm:")) + + def test_low_suffix_stripped_from_section_key(self): + entries = [entry("ci:low", "fix", "tweak", 1, impact_level="low")] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn("ci:", out) + self.assertNotIn("ci:low:", out) + + def test_unsupported_type_is_skipped(self): + # 'chore' has no features/fixes bucket -> dropped from yaml output. + entries = [entry("vm", "chore", "noise", 1)] + self.assertEqual(cl.render_yaml(entries, "v1.21.0"), "{}\n\n") + + +class RenderMarkdownTest(unittest.TestCase): + def test_basic_structure(self): + entries = [entry("vm", "fix", "fixed Y", 11, impact_level="high")] + out = cl.render_markdown(entries, "v1.21.0", "v1.21") + self.assertIn("# Changelog v1.21", out) + self.assertIn("## vm", out) + self.assertIn("**fix** (high): fixed Y ([!11]", out) + + +class MinorVersionFromTagTest(unittest.TestCase): + def test_patch_version_truncated_to_minor(self): + self.assertEqual(cl.minor_version_from_tag("v1.21.3"), "v1.21") + + def test_minor_version_unchanged(self): + self.assertEqual(cl.minor_version_from_tag("v1.21"), "v1.21") + + def test_non_matching_returned_as_is(self): + self.assertEqual(cl.minor_version_from_tag("nightly"), "nightly") + + +if __name__ == "__main__": + unittest.main() diff --git a/.gitlab/ci/scripts/python/tests/test_check_changelog_entry.py b/.gitlab/ci/scripts/python/tests/test_check_changelog_entry.py new file mode 100644 index 0000000000..986fd0b6c1 --- /dev/null +++ b/.gitlab/ci/scripts/python/tests/test_check_changelog_entry.py @@ -0,0 +1,144 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for check_changelog_entry.py pure helpers. + +The module guards main() behind `if __name__ == "__main__"`, so importing it +runs no network calls and never exits. We exercise the block parsing and +validation logic directly with synthetic MR-description text — no GitLab API +access required. +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +# Put the python/ dir (parent of tests/) on the path so the scripts under test +# import cleanly regardless of the discover invocation's cwd. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import check_changelog_entry as cc # noqa: E402 + + +# A representative allowed-sections set used across the validation tests. +ALLOWED = {"vm", "core", "ci", "ci:low"} + + +class FindBlocksTest(unittest.TestCase): + def test_finds_single_block(self): + text = "intro\n```changes\nsection: vm\n```\noutro" + blocks = cc.CHANGES_BLOCK_RE.findall(text) + self.assertEqual(len(blocks), 1) + self.assertIn("section: vm", blocks[0]) + + def test_finds_multiple_blocks(self): + text = ( + "```changes\nsection: vm\n```\n" + "text in between\n" + "```changes\nsection: core\n```\n" + ) + blocks = cc.CHANGES_BLOCK_RE.findall(text) + self.assertEqual(len(blocks), 2) + + def test_no_block_returns_empty(self): + self.assertEqual(cc.CHANGES_BLOCK_RE.findall("no fenced block here"), []) + + +class ParseBlockTest(unittest.TestCase): + def test_parses_keys_lowercased(self): + fields = cc.parse_block("Section: vm\nType: fix\nSummary: did a thing") + self.assertEqual( + fields, {"section": "vm", "type": "fix", "summary": "did a thing"} + ) + + def test_ignores_non_keyvalue_lines(self): + fields = cc.parse_block("section: vm\nthis is prose\n\ntype: fix") + self.assertEqual(fields, {"section": "vm", "type": "fix"}) + + def test_trims_whitespace_around_value(self): + fields = cc.parse_block("section: vm ") + self.assertEqual(fields["section"], "vm") + + +class LoadAllowedSectionsTest(unittest.TestCase): + def test_strips_comments_and_blank_lines(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "sections.txt" + p.write_text( + "# a comment\n\nvm\ncore\n \n# another\nci:low\n", + encoding="utf-8", + ) + self.assertEqual( + cc.load_allowed_sections(p), {"vm", "core", "ci:low"} + ) + + +class ValidateBlockTest(unittest.TestCase): + def test_valid_block_has_no_errors(self): + block = "section: vm\ntype: fix\nsummary: fixed it\nimpact_level: high" + self.assertEqual(cc.validate_block(1, block, ALLOWED), []) + + def test_missing_section(self): + block = "type: fix\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'section'" in e for e in errors)) + + def test_section_not_allowed(self): + block = "section: bogus\ntype: fix\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("not in" in e and "allowed_sections" in e for e in errors)) + + def test_missing_type(self): + block = "section: vm\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'type'" in e for e in errors)) + + def test_type_not_allowed(self): + # 'chore' is intentionally rejected: deckhouse changelog renders only + # feature/fix (ALLOWED_TYPES). Guards the 5.4 fix from regressing. + block = "section: vm\ntype: chore\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("type 'chore' is not supported" in e for e in errors)) + + def test_allowed_types_is_feature_fix_only(self): + self.assertEqual(cc.ALLOWED_TYPES, {"feature", "fix"}) + + def test_missing_summary(self): + block = "section: vm\ntype: fix\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'summary'" in e for e in errors)) + + def test_missing_impact_level_when_not_low(self): + block = "section: vm\ntype: fix\nsummary: s" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("missing required key 'impact_level'" in e for e in errors)) + + def test_low_section_may_omit_impact_level(self): + block = "section: ci:low\ntype: fix\nsummary: s" + self.assertEqual(cc.validate_block(1, block, ALLOWED), []) + + def test_low_section_with_explicit_low_is_ok(self): + block = "section: ci:low\ntype: fix\nsummary: s\nimpact_level: low" + self.assertEqual(cc.validate_block(1, block, ALLOWED), []) + + def test_low_section_with_conflicting_impact_level(self): + block = "section: ci:low\ntype: fix\nsummary: s\nimpact_level: high" + errors = cc.validate_block(1, block, ALLOWED) + self.assertTrue(any("pinned to low" in e for e in errors)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.gitlab/scripts/js/mrs_notifier.mjs b/.gitlab/scripts/js/mrs_notifier.mjs index 9719ea3db0..cf00369976 100644 --- a/.gitlab/scripts/js/mrs_notifier.mjs +++ b/.gitlab/scripts/js/mrs_notifier.mjs @@ -76,6 +76,19 @@ function validateEnv() { } } +// Pure predicate: decide whether an open MR belongs in the review summary. +// Excludes drafts/WIP, release-* source branches, and autorelease/changelog +// bot MRs (these are not human review work). Exported for unit testing. +export function shouldNotifyMR(mr) { + if (mr.draft || mr.work_in_progress) return false; + const head = (mr.source_branch || '').toLowerCase(); + if (head.startsWith('release-')) return false; + const labels = (mr.labels || []).map((l) => l.toLowerCase()); + if (labels.some((l) => l.startsWith('autorelease'))) return false; + if (labels.includes('changelog')) return false; + return true; +} + async function fetchOpenMRs() { const { data } = await api.get(`/projects/${PROJECT_ID}/merge_requests`, { params: { @@ -85,15 +98,7 @@ async function fetchOpenMRs() { sort: 'asc', }, }); - return data.filter((mr) => { - if (mr.draft || mr.work_in_progress) return false; - const head = (mr.source_branch || '').toLowerCase(); - if (head.startsWith('release-')) return false; - const labels = (mr.labels || []).map((l) => l.toLowerCase()); - if (labels.some((l) => l.startsWith('autorelease'))) return false; - if (labels.includes('changelog')) return false; - return true; - }); + return data.filter(shouldNotifyMR); } async function fetchUser(id) { @@ -107,7 +112,10 @@ async function fetchUser(id) { } } -function formatUser(user, details) { +// Pure helper: render a Loop mention for a user. Prefers the profile name +// rendered as @first.last; falls back to the username with a nudge to set a +// real name. Exported for unit testing. +export function formatUser(user, details) { if (!user) return 'unknown'; if (details && details.name) { const loopName = details.name.replace(/ /g, '.').toLowerCase(); diff --git a/.gitlab/scripts/js/mrs_notifier.test.mjs b/.gitlab/scripts/js/mrs_notifier.test.mjs index 771a85b898..8ecf6b30e5 100644 --- a/.gitlab/scripts/js/mrs_notifier.test.mjs +++ b/.gitlab/scripts/js/mrs_notifier.test.mjs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Unit tests for mrs_notifier.mjs classification logic. +// Unit tests for mrs_notifier.mjs. // // mrs_notifier.mjs guards its `run()` call behind a direct-invocation check // and validates env vars only inside `run()`, so it can be safely imported // here without triggering network calls or process.exit. The pure helpers -// `classifyMR` and `extractUnresolvedThreads` are exercised directly with -// synthetic data — no network access required. +// `classifyMR`, `extractUnresolvedThreads`, `shouldNotifyMR`, and `formatUser` +// are exercised directly with synthetic data — no network access required. import { test } from 'node:test'; import assert from 'node:assert/strict'; @@ -27,7 +27,12 @@ import { execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import path from 'node:path'; -import { classifyMR, extractUnresolvedThreads } from './mrs_notifier.mjs'; +import { + classifyMR, + extractUnresolvedThreads, + shouldNotifyMR, + formatUser, +} from './mrs_notifier.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const MODULE_PATH = path.join(__dirname, 'mrs_notifier.mjs'); @@ -246,3 +251,56 @@ test('extractUnresolvedThreads + classifyMR integration: old unresolved => stuck assert.equal(classifyMR([], threads), 'changes_requested'); } }); + +// ---- shouldNotifyMR (open-MR filter) ---- + +test('shouldNotifyMR: plain open MR is included', () => { + assert.equal(shouldNotifyMR({ source_branch: 'feature/x', labels: [] }), true); +}); + +test('shouldNotifyMR: drafts and WIP are excluded', () => { + assert.equal(shouldNotifyMR({ draft: true, source_branch: 'x' }), false); + assert.equal(shouldNotifyMR({ work_in_progress: true, source_branch: 'x' }), false); +}); + +test('shouldNotifyMR: release-* source branches are excluded (case-insensitive)', () => { + assert.equal(shouldNotifyMR({ source_branch: 'release-1.2' }), false); + assert.equal(shouldNotifyMR({ source_branch: 'Release-1.2' }), false); +}); + +test('shouldNotifyMR: autorelease and changelog bot MRs are excluded', () => { + assert.equal(shouldNotifyMR({ source_branch: 'x', labels: ['autorelease'] }), false); + assert.equal(shouldNotifyMR({ source_branch: 'x', labels: ['Autorelease/v1'] }), false); + assert.equal(shouldNotifyMR({ source_branch: 'x', labels: ['changelog'] }), false); +}); + +test('shouldNotifyMR: missing source_branch/labels is safe', () => { + assert.equal(shouldNotifyMR({}), true); +}); + +// ---- formatUser (Loop mention rendering) ---- + +test('formatUser: null user => unknown', () => { + assert.equal(formatUser(null, null), 'unknown'); +}); + +test('formatUser: profile name rendered as @first.last lowercase', () => { + assert.equal( + formatUser({ username: 'jdoe' }, { name: 'John Doe' }), + '@john.doe', + ); +}); + +test('formatUser: no profile name => username with nudge', () => { + assert.equal( + formatUser({ username: 'jdoe' }, null), + 'jdoe (Set name in profile!)', + ); +}); + +test('formatUser: falls back to login when username absent', () => { + assert.equal( + formatUser({ login: 'ghuser' }, {}), + 'ghuser (Set name in profile!)', + ); +}); From dddd321565468ec614857a12f43f50bd3a1c4513 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 17:30:07 +0300 Subject: [PATCH 50/60] docs(ci): correct stale/contradictory comments in .gitlab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment-only audit of .gitlab/** surfaced one internal contradiction and three stale TODOs (no behavior change): - stages.yml: the "Stage semantics" legend listed prod_check after build/scan/deploy_dev, contradicting both the actual `stages:` order (prod_check is declared before build) and its own "Runs BEFORE build" note. Reordered to match, and clarified that prod_check runs ONLY in the manual web release-channel flow (prod:check-requirements, CI_PIPELINE_SOURCE == web) — its early position is a DAG-ordering artifact so prod:build:* can `needs:` it, not a dev/MR-pipeline stage. - backport.yml (x2), changelog.yml, manual-tools.yml: dropped the "TODO: webhook-listener" framing. Reactive webhook automation is an accepted permanent gap (no webhook-listener will be built); reworded accordingly, without citing a tracker id. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/backport.yml | 7 ++++--- .gitlab/ci/jobs/changelog.yml | 5 +++-- .gitlab/ci/jobs/manual-tools.yml | 3 ++- .gitlab/ci/stages.yml | 8 ++++++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index 6a605cfb2d..3340bc46dc 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -23,7 +23,8 @@ # 1. Manual pipeline (Run pipeline UI) with variable TARGET_BRANCH=release-1.21. # 2. Manual pipeline on an MR carrying the `status/backport` label. # GitLab does NOT auto-trigger pipelines on label change; the user must -# press "Run pipeline" on the MR (TODO: webhook-listener per plan §7). +# press "Run pipeline" on the MR. Webhook-driven auto-trigger is an +# accepted permanent gap (no webhook-listener will be built). # The target branch is derived from the source MR milestone title # (vX.Y.Z or X.Y.Z -> release-X.Y); TARGET_BRANCH overrides this. # @@ -55,8 +56,8 @@ backport: allow_failure: true # Mode 2: MR with the `status/backport` label. GitLab does NOT auto-run # pipelines on label change; user has to press "Run pipeline" on the MR - # (TODO: webhook-listener per migration plan §7). The target branch is - # derived from the source MR milestone by backport.sh. + # (webhook auto-trigger is an accepted permanent gap, not planned). The + # target branch is derived from the source MR milestone by backport.sh. - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,\s*)status\/backport(,|$)/ when: manual allow_failure: true diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index a6ad34412a..2f9292c385 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -21,8 +21,9 @@ # All three used ./.github/actions/milestone-changelog (composite action). # # Per migration plan §0(3) and §11.5.4 the GitLab jobs are manual + scheduled, -# NOT reactive. A webhook-listener for slash-command-equivalents and label -# events is a documented TODO. +# NOT reactive. Reactive webhook automation (slash-command-equivalents, +# label/milestone events) is an accepted permanent gap — no webhook-listener +# will be built. # # Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index a32b9e4616..24379bc367 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -21,7 +21,8 @@ # Migration plan §0(1) explicitly removes GitHub slash-command dispatch # (/.github/workflows/dispatch-slash-command.yml) and replaces it with # manual pipelines. The two manual jobs in this file are the closest -# equivalents. There is no automated webhook listener yet (TODO). +# equivalents. There is no automated webhook listener, and this is an +# accepted permanent gap (no webhook-listener will be built). # Variables for "Run pipeline" UI: # LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index cf5690445e..a8e72b58f2 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -13,11 +13,15 @@ # info — manifest printers, set_vars helper # lint — Go lint, helm lint, yaml lint, no-cyrillic, etc. # test — Go unit tests, hooks tests +# prod_check — release-channel requirements check +# (prod:check-requirements). Runs ONLY in the manual +# web release-channel flow (CI_PIPELINE_SOURCE == web), +# NOT in dev/MR pipelines. Declared BEFORE build purely +# so prod:build:* can `needs:` it — the early position +# is a DAG-ordering artifact, not a dev-pipeline stage. # build — werf build for dev / dev-tags / main / prod # scan — cve_scan, gitleaks, svace (owned by sibling issues) # deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid -# prod_check — requirements check before release-channel dispatch. -# Runs BEFORE build so prod:build:* can need it. # deploy_prod — prod release-channel deploy. ONE stage with independent # manual jobs per channel (tag-push flow, deploy-prod.yml) # and per-edition deploy from Run pipeline From f679b84b237c5761b7c1c1f1584e9dc050d59631 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 18:10:55 +0300 Subject: [PATCH 51/60] fix(ci): correct dev-tag deploy, prod secondary_repo, and main tag Three build/deploy parity fixes for the GitHub Actions -> GitLab CI migration, found by reviewing .github/workflows against the new .gitlab/ci/** files. - Drop the DEV release-channel fan-out. deploy_for_dev_tag crane-copied a dev tag into alpha/beta/early-access/stable/rock-solid, but the DEV registry has no release channels, only tags. The GitHub "Deploy Dev" workflow is build-only. Remove deploy-dev.yml, its include, and the now-unused deploy_dev stage; dev tags stay built by build_dev_tags. - Pass secondary_repo to prod builds. GitHub builds set secondary_repo=${DEV_MODULE_SOURCE}/${MODULE_NAME} so prod werf builds reuse already-built layers from the DEV registry. Add WERF_SECONDARY_REPO_1 to .dual_registry_login (a prod-only template), so it applies to build_prod and prod:build:* but never to dev builds. - Use "main" as the main-branch module tag instead of v0.0.0-main, to match GitHub set_vars (MODULES_MODULE_TAG = ref_name = "main"). Signed-off-by: Nikita Korolev --- .gitlab/ci/includes.yml | 5 ++- .gitlab/ci/jobs/deploy-dev.yml | 32 ------------------- .gitlab/ci/stages.yml | 4 +-- .gitlab/ci/templates/base/deploy.yml | 13 ++++---- .../ci/templates/base/dual_registry_login.yml | 12 ++++++- .gitlab/ci/templates/dev/dev_tags.yml | 7 ++-- .gitlab/ci/templates/dev/main.yml | 9 +++--- 7 files changed, 33 insertions(+), 49 deletions(-) delete mode 100644 .gitlab/ci/jobs/deploy-dev.yml diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 353fb7838f..5dce1ff657 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -84,7 +84,10 @@ include: - local: ".gitlab/ci/jobs/test-d8v-cli.yml" - local: ".gitlab/ci/jobs/build-dev.yml" - local: ".gitlab/ci/jobs/build-prod.yml" - - local: ".gitlab/ci/jobs/deploy-dev.yml" + # No deploy-dev.yml: the DEV registry has no release channels (only tags), + # so a dev tag is just built (build_dev_tags) and never crane-copied into + # alpha/beta/... The GitHub "Deploy Dev" workflow + # (dev_module_build-and-registration.yml) is build-only as well. - local: ".gitlab/ci/jobs/deploy-prod.yml" # Prod release-channels parity flow (manual single-channel dispatch): # requirements check, build/deploy per edition, version verification, diff --git a/.gitlab/ci/jobs/deploy-dev.yml b/.gitlab/ci/jobs/deploy-dev.yml deleted file mode 100644 index 3af1115dba..0000000000 --- a/.gitlab/ci/jobs/deploy-dev.yml +++ /dev/null @@ -1,32 +0,0 @@ -# DEV deploy job. -# -# Carries forward deploy_for_dev_tag from the previous root .gitlab-ci.yml. -# Fans out across the five release channels (alpha / beta / early-access / -# stable / rock-solid) once build_dev_tags has finished. -# -# Extends: -# - .deploy (upstream modules-gitlab-ci Deploy.gitlab-ci.yml — release-channel crane copy) -# - .dev_tags (this repo — DEV registry vars + tag-match rule) -# -# The upstream .deploy template only runs on $CI_COMMIT_TAG (rule from -# Deploy.gitlab-ci.yml), which matches .dev_tags's tag regex -# vX.Y.Z-dev.* — behavior preserved. -# -# Need: build_dev_tags -> deploy_for_dev_tag. (Kept as a `needs:` line — -# same as the previous root .gitlab-ci.yml.) - -deploy_for_dev_tag: - stage: deploy_dev - needs: ["build_dev_tags"] - extends: - - .base_deploy - - .dev_tags - - .dual_registry_login - parallel: - matrix: - - RELEASE_CHANNEL: - - alpha - - beta - - early-access - - stable - - rock-solid diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index a8e72b58f2..6db3044dff 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -20,8 +20,9 @@ # so prod:build:* can `needs:` it — the early position # is a DAG-ordering artifact, not a dev-pipeline stage. # build — werf build for dev / dev-tags / main / prod +# (dev tags are build-only: the DEV registry has no +# release channels, so there is no DEV deploy stage) # scan — cve_scan, gitleaks, svace (owned by sibling issues) -# deploy_dev — DEV tag deploy to alpha/beta/ea/stable/rock-solid # deploy_prod — prod release-channel deploy. ONE stage with independent # manual jobs per channel (tag-push flow, deploy-prod.yml) # and per-edition deploy from Run pipeline @@ -48,7 +49,6 @@ stages: - prod_check - build - scan - - deploy_dev - deploy_prod - prod_verify - prod_release diff --git a/.gitlab/ci/templates/base/deploy.yml b/.gitlab/ci/templates/base/deploy.yml index 45261baad8..90b527f25c 100644 --- a/.gitlab/ci/templates/base/deploy.yml +++ b/.gitlab/ci/templates/base/deploy.yml @@ -1,9 +1,11 @@ # Base deploy template (.base_deploy). # -# Naming: this is the BASE crane-copy deploy job body extended by -# deploy_for_dev_tag / deploy_to_prod_*. Previously named `.local_deploy`; -# renamed to `.base_deploy` for the same reason as `.base_build` ("local" -# was uninformative and clashed with `include: local:`). +# Naming: this is the BASE crane-copy deploy job body extended by the prod +# release-channel deploy jobs (deploy_to_prod_* and prod:deploy:*). Previously +# named `.local_deploy`; renamed to `.base_deploy` for the same reason as +# `.base_build` ("local" was uninformative and clashed with `include: local:`). +# Note: there is no DEV deploy — the DEV registry has no release channels, so +# dev tags are build-only (see build_dev_tags). # # Mirrors upstream modules-gitlab-ci Deploy.gitlab-ci.yml `.deploy` body # (verified in @@ -14,8 +16,7 @@ # # Upstream `.deploy` has `rules: [{if: $CI_COMMIT_TAG}]` + `when: manual` # baked in. Because GitLab CI merges parent rules with parent-first -# precedence, that turns deploy_for_dev_tag into a manual job on prod -# tags too (we want it auto on dev tags and absent on prod tags). +# precedence, those rules would override our own gating. # The prod-channel chain (deploy_to_prod_*) wants `when: manual` from # .prod_manual, so the upstream `when: manual` happens to match — but we # still want explicit control. Keeping the upstream body in `.base_deploy` diff --git a/.gitlab/ci/templates/base/dual_registry_login.yml b/.gitlab/ci/templates/base/dual_registry_login.yml index f42bebbfa6..6a7d52dbb5 100644 --- a/.gitlab/ci/templates/base/dual_registry_login.yml +++ b/.gitlab/ci/templates/base/dual_registry_login.yml @@ -13,10 +13,19 @@ # register into PROD) can extend both .prod_vars and .dual_registry_login # and end up with the right pair of `werf cr login` calls. # +# WERF_SECONDARY_REPO_1 is the reason the DEV login is needed at build time: +# prod werf builds reuse already-built layers from the DEV registry as a +# read-only secondary repo. This mirrors the GitHub build action input +# `secondary_repo: "${vars.DEV_MODULE_SOURCE}/${vars.MODULE_NAME}"`, which the +# action exports as WERF_SECONDARY_REPO_1 (see modules-actions/build/action.yml). +# It is set ONLY here, so it only applies to prod release build/deploy jobs that +# extend .dual_registry_login — dev builds never use a secondary repo. (Deploy +# jobs that extend this template ignore it: they crane-copy, not werf build.) +# # Usage: # extends: # - .prod_vars # sets MODULES_* = PROD_* -# - .dual_registry_login # adds DEV_MODULES_* = DEV_* +# - .dual_registry_login # adds DEV_MODULES_* = DEV_* + WERF_SECONDARY_REPO_1 # - .build # upstream werf build + bundle/release-channel copy # # TODO: confirm with a virt-test pipeline that the second login is actually @@ -29,3 +38,4 @@ DEV_MODULES_REGISTRY: "${DEV_REGISTRY}" DEV_MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" DEV_MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" + WERF_SECONDARY_REPO_1: "${DEV_MODULE_SOURCE}/${MODULE_NAME}" diff --git a/.gitlab/ci/templates/dev/dev_tags.yml b/.gitlab/ci/templates/dev/dev_tags.yml index e71976f967..532a807685 100644 --- a/.gitlab/ci/templates/dev/dev_tags.yml +++ b/.gitlab/ci/templates/dev/dev_tags.yml @@ -4,9 +4,10 @@ # Triggers on tags matching vX.Y.Z-dev* (regex kept identical to the # previous regex101 link in the original .gitlab-ci.yml). # -# Use case: a developer pushes v1.21.0-dev-build42 -> build_dev_tags runs, -# pushes images into DEV_REGISTRY tagged with the raw tag name, then -# deploy_for_dev_tag fans out across all release channels. +# Use case: a developer pushes v1.21.0-dev-build42 -> build_dev_tags runs and +# pushes images into DEV_REGISTRY tagged with the raw tag name. There is no +# deploy step: the DEV registry has no release channels, only tags (matching +# the build-only GitHub "Deploy Dev" workflow). .dev_tags: variables: diff --git a/.gitlab/ci/templates/dev/main.yml b/.gitlab/ci/templates/dev/main.yml index 7d6d63a07e..8840d42cb8 100644 --- a/.gitlab/ci/templates/dev/main.yml +++ b/.gitlab/ci/templates/dev/main.yml @@ -1,12 +1,13 @@ # DEV pipeline context: main-branch builds. # -# Carries forward the previous root .gitlab-ci.yml `.main` template. Uses a -# fixed `v0.0.0-main` tag so that every main push produces an -# overwritable slot in the DEV registry (the next push replaces the tag). +# Uses the `main` tag, matching GitHub dev_module_build.yml `set_vars` +# (MODULES_MODULE_TAG = github.ref_name = "main" for the default branch, +# see .github/workflows/dev_module_build.yml:121-122). Every main push +# overwrites this tag in the DEV registry, producing a single rolling slot. .main: variables: - MODULES_MODULE_TAG: v0.0.0-main + MODULES_MODULE_TAG: main extends: - .dev_vars rules: From d5fa9e40610915ce83670f7f49d26fda26e2179c Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 26 Jun 2026 18:27:09 +0300 Subject: [PATCH 52/60] fix(ci): correct cve release-in-dev, svace status, cleanup dry-run, changelog impact Fix the functional bugs found while reviewing the GitHub Actions -> GitLab CI migration (vs .github/workflows). - cve-scan: the manual scan set RELEASE_IN_DEV=${SCAN_TAG:-false}, passing the tag string into a boolean. Derive it via rules instead, mirroring the GitHub `startsWith(tag || 'main', 'release-')` expression: a release-* SCAN_TAG sets "true", anything else keeps "false". - svace: svace:notify reported status from CI_JOB_STATUS of the notify job itself (almost always success). Read the real svace:analyze status via the pipeline jobs API, and run notify on both success and failure with an on_success/on_failure rule pair (same idiom as prod:notify-loop). Drop GIT_STRATEGY: none so the api.sh helper is available. - cleanup: restore the explicit WERF_DRY_RUN="false" from dev_registry-cleanup.yml, so an inherited WERF_DRY_RUN=true cannot turn cleanup into a silent no-op. - changelog_collect.py: preserve the free-text `impact` migration note for high-impact entries (parse multi-line block values, emit `impact` after `pull_request`), and make CHANGELOG-.md cumulative across patch releases instead of overwriting it with only the current milestone. Add unit tests for all four behaviours. Signed-off-by: Nikita Korolev --- .gitlab/ci/jobs/cleanup.yml | 4 + .gitlab/ci/jobs/cve-scan.yml | 10 +- .gitlab/ci/jobs/svace.yml | 34 ++++-- .../ci/scripts/python/changelog_collect.py | 112 +++++++++++++++--- .../python/tests/test_changelog_collect.py | 78 +++++++++++- 5 files changed, 209 insertions(+), 29 deletions(-) diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml index e371d5491d..278becb7e4 100644 --- a/.gitlab/ci/jobs/cleanup.yml +++ b/.gitlab/ci/jobs/cleanup.yml @@ -34,6 +34,10 @@ cleanup: - .dev_vars variables: MODULES_MODULE_TAG: v0.0.0-main + # Explicitly disable werf dry-run, mirroring GitHub dev_registry-cleanup.yml + # (env WERF_DRY_RUN: "false"). Without this an inherited/global + # WERF_DRY_RUN=true would make cleanup silently prune nothing. + WERF_DRY_RUN: "false" rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "cleanup" script: diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml index b79c905ac2..1b8e733beb 100644 --- a/.gitlab/ci/jobs/cve-scan.yml +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -74,8 +74,16 @@ cve:scan:manual: EXTERNAL_MODULE_NAME: "virtualization" SCAN_SEVERAL_LATEST_RELEASES: "${SCAN_SEVERAL:-False}" LATEST_RELEASES_AMOUNT: "5" - RELEASE_IN_DEV: "${SCAN_TAG:-false}" + RELEASE_IN_DEV: "false" rules: + # RELEASE_IN_DEV is a boolean: release-* tags are built into the DEV + # registry, everything else into PROD. Mirror the GitHub expression + # `startsWith(tag || 'main', 'release-')` — a release-* SCAN_TAG flips it + # to "true"; an empty/other SCAN_TAG keeps the "false" default above. + - if: '$CI_PIPELINE_SOURCE == "web" && $SCAN_TAG =~ /^release-/' + when: manual + variables: + RELEASE_IN_DEV: "true" - if: '$CI_PIPELINE_SOURCE == "web"' when: manual - when: never diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index d797449d62..1fdfcd3978 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -135,20 +135,32 @@ svace:analyze: svace:notify: stage: cleanup interruptible: true - before_script: - - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq - variables: - GIT_STRATEGY: none + # needs svace:analyze (non-optional) so this job stays coupled to a real + # svace run: on an unrelated web pipeline svace:analyze is manual and never + # played, so it is skipped and this notify is skipped too. The on_success / + # on_failure rule pair (same idiom as prod:notify-loop) makes notify run on + # BOTH a passing and a failing svace:analyze. set-vars is pulled for the + # REF_NAME dotenv. needs: - job: svace:set-vars - - job: svace:build + artifacts: true - job: svace:analyze + before_script: + - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq script: - | set -euo pipefail + source .gitlab/ci/scripts/bash/lib/api.sh + gl_required_env CI_API_V4_URL GITLAB_API_TOKEN CI_PROJECT_ID CI_PIPELINE_ID LOOP_WEBHOOK_URL DATE=$(date '+%Y-%m-%d') - STATUS=":white_check_mark: SUCCESS!" - if [[ "${CI_JOB_STATUS:-success}" != "success" ]]; then + # Derive the result from the svace:analyze job, not this notify job: + # CI_JOB_STATUS would report THIS job and is almost always "success". + # Mirrors the GitHub `needs.analyze_build.result == 'success'` check. + ANALYZE_STATUS=$(api GET "/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100" \ + | jq -r '[.[] | select(.name == "svace:analyze")][0].status // ""') + if [[ "${ANALYZE_STATUS}" == "success" ]]; then + STATUS=":white_check_mark: SUCCESS!" + else STATUS=":x: FAIL!" fi MESSAGE="### :gear: **DVP ${DATE} Weekly Svace Analyze Report** @@ -162,6 +174,10 @@ svace:notify: "${LOOP_WEBHOOK_URL}" || echo "Loop webhook failed (non-fatal)" rules: - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' - when: always + when: on_success + - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "svace"' + when: on_failure - if: '$CI_PIPELINE_SOURCE == "web"' - when: always + when: on_success + - if: '$CI_PIPELINE_SOURCE == "web"' + when: on_failure diff --git a/.gitlab/ci/scripts/python/changelog_collect.py b/.gitlab/ci/scripts/python/changelog_collect.py index 7a9d240a29..1fea77386d 100644 --- a/.gitlab/ci/scripts/python/changelog_collect.py +++ b/.gitlab/ci/scripts/python/changelog_collect.py @@ -61,6 +61,11 @@ re.DOTALL, ) KEY_VALUE_RE = re.compile(r"^([A-Za-z_]+)\s*:\s*(.*)$") +# Only these keys start a new field in a ```changes block. Any other line is +# treated as a continuation of the current field, so multi-line values (most +# importantly `impact:`, the high-impact migration note) are preserved instead +# of being dropped. Mirrors the deckhouse/changelog-action block schema. +KNOWN_BLOCK_KEYS = {"section", "type", "summary", "impact", "impact_level"} # deckhouse/changelog-action@v2.6.0 only renders 'feature' (-> features) and 'fix' # (-> fixes) sections in CHANGELOG-*.yml. Keep in sync with # check_changelog_entry.py. @@ -125,13 +130,20 @@ def next_link(link_header: str) -> str: def parse_changes_block(block_text: str) -> dict[str, str] | None: fields: dict[str, str] = {} + current_key: str | None = None for raw_line in block_text.splitlines(): match = KEY_VALUE_RE.match(raw_line.rstrip()) - if not match: - continue - key = match.group(1).strip().lower() - value = match.group(2).strip() - fields[key] = value + if match and match.group(1).strip().lower() in KNOWN_BLOCK_KEYS: + key = match.group(1).strip().lower() + fields[key] = match.group(2).strip() + current_key = key + elif current_key is not None: + # Continuation line of the current field (e.g. a multi-line impact). + cont = raw_line.strip() + if cont: + fields[current_key] = ( + f"{fields[current_key]}\n{cont}" if fields[current_key] else cont + ) required = {"section", "type", "summary"} if not required.issubset(fields): return None @@ -192,6 +204,7 @@ def collect_entries_for_milestone( "section": section, "type": parsed["type"], "summary": parsed["summary"], + "impact": parsed.get("impact", ""), "impact_level": impact_level or "high", "mr_iid": mr["iid"], "mr_title": mr.get("title", ""), @@ -275,19 +288,34 @@ def render_yaml(entries: list[dict], milestone_title: str) -> str: for entry in items: lines.append(f" - summary: {yaml_summary_scalar(entry['summary'])}") lines.append(f" pull_request: {entry['mr_url']}") + # High-impact entries carry a free-text `impact` migration note. + # Preserve it (deckhouse/changelog-action emits it after + # pull_request); a multi-line note becomes a literal block. + impact = entry.get("impact", "") + if impact: + if "\n" in impact: + lines.append(" impact: |-") + for impact_line in impact.split("\n"): + lines.append(f" {impact_line}" if impact_line else "") + else: + lines.append(f" impact: {yaml_summary_scalar(impact)}") return "\n".join(lines) + "\n\n" -def render_markdown(entries: list[dict], milestone_title: str, minor_version: str) -> str: +def render_milestone_md_block(entries: list[dict], milestone_title: str) -> str: + """Render the markdown block for ONE milestone (patch version). + + Heading is `## ` so that + :func:`merge_minor_markdown` can merge multiple patch versions into the + cumulative `CHANGELOG-.md` idempotently. + """ grouped = group_entries(entries) - lines = [ - f"# Changelog {minor_version}", - "", - f"Auto-generated summary for milestone `{milestone_title}`.", - "", - ] + lines = [f"## {milestone_title}", ""] + if not grouped: + lines.append("_No changelog entries._") + return "\n".join(lines).rstrip() + "\n" for section in sorted(grouped.keys()): - lines.append(f"## {section}") + lines.append(f"### {section}") lines.append("") for entry in grouped[section]: lines.append( @@ -298,6 +326,57 @@ def render_markdown(entries: list[dict], milestone_title: str, minor_version: st return "\n".join(lines).rstrip() + "\n" +def md_version_sort_key(title: str) -> tuple[int, int, int]: + """Sort key for `## vX.Y.Z` headings; missing parts sort as 0.""" + m = re.match(r"^v?(\d+)\.(\d+)(?:\.(\d+))?", title) + if not m: + return (0, 0, 0) + return tuple(int(part) if part else 0 for part in m.groups()) # type: ignore[return-value] + + +def parse_minor_md_blocks(text: str) -> dict[str, str]: + """Split an existing CHANGELOG-.md into {milestone_title: block}. + + Content before the first `## ` heading (the file header) is dropped — it is + regenerated. Each block keeps its own `## ` heading. + """ + blocks: dict[str, str] = {} + current_title: str | None = None + current_lines: list[str] = [] + for line in text.splitlines(): + if line.startswith("## "): + if current_title is not None: + blocks[current_title] = "\n".join(current_lines).rstrip() + "\n" + current_title = line[3:].strip() + current_lines = [line] + elif current_title is not None: + current_lines.append(line) + if current_title is not None: + blocks[current_title] = "\n".join(current_lines).rstrip() + "\n" + return blocks + + +def merge_minor_markdown( + existing_text: str, minor_version: str, milestone_title: str, block: str +) -> str: + """Merge ``block`` for ``milestone_title`` into the cumulative minor file. + + Replaces this milestone's block if present (idempotent re-generation) or + inserts it, then re-emits all patch blocks newest-first. This is what keeps + CHANGELOG-<minor>.md cumulative across patch releases (the GitHub + changelog-action produced a cumulative ``branch_markdown``); rendering only + the current milestone would drop the earlier patches. + """ + blocks = parse_minor_md_blocks(existing_text) + blocks[milestone_title] = block + ordered = sorted(blocks.keys(), key=md_version_sort_key, reverse=True) + out = [f"# Changelog {minor_version}", ""] + for title in ordered: + out.append(blocks[title].rstrip()) + out.append("") + return "\n".join(out).rstrip() + "\n" + + def minor_version_from_tag(tag: str) -> str: """v1.21.3 -> v1.21, v1.21 -> v1.21.""" m = re.match(r"^v(\d+\.\d+)(?:\.\d+)?$", tag) @@ -317,8 +396,13 @@ def write_files( minor = minor_version_from_tag(milestone_title) md_path = changelog_dir / f"CHANGELOG-{minor}.md" yml_path.write_text(render_yaml(entries, milestone_title), encoding="utf-8") + # Merge this milestone's block into the cumulative minor markdown so earlier + # patch releases of the same minor are preserved. + existing_md = md_path.read_text(encoding="utf-8") if md_path.is_file() else "" + block = render_milestone_md_block(entries, milestone_title) md_path.write_text( - render_markdown(entries, milestone_title, minor), encoding="utf-8" + merge_minor_markdown(existing_md, minor, milestone_title, block), + encoding="utf-8", ) log(f"Wrote {yml_path.relative_to(project_dir)} and {md_path.relative_to(project_dir)}.") return yml_path, md_path diff --git a/.gitlab/ci/scripts/python/tests/test_changelog_collect.py b/.gitlab/ci/scripts/python/tests/test_changelog_collect.py index a2ab6404f3..161f9c499d 100644 --- a/.gitlab/ci/scripts/python/tests/test_changelog_collect.py +++ b/.gitlab/ci/scripts/python/tests/test_changelog_collect.py @@ -29,11 +29,12 @@ import changelog_collect as cl # noqa: E402 -def entry(section, type_, summary, mr_iid, impact_level="high", mr_url=None): +def entry(section, type_, summary, mr_iid, impact_level="high", mr_url=None, impact=""): return { "section": section, "type": type_, "summary": summary, + "impact": impact, "impact_level": impact_level, "mr_iid": mr_iid, "mr_title": f"MR {mr_iid}", @@ -75,6 +76,15 @@ def test_keeps_optional_keys(self): ) self.assertEqual(parsed["impact_level"], "low") + def test_multiline_impact_is_preserved(self): + parsed = cl.parse_changes_block( + "section: core\ntype: feature\nsummary: containerd v2\n" + "impact: First line.\nSecond line.\nimpact_level: high" + ) + self.assertEqual(parsed["impact"], "First line.\nSecond line.") + self.assertEqual(parsed["impact_level"], "high") + self.assertEqual(parsed["summary"], "containerd v2") + class HasLabelTest(unittest.TestCase): def test_string_labels(self): @@ -165,15 +175,73 @@ def test_unsupported_type_is_skipped(self): entries = [entry("vm", "chore", "noise", 1)] self.assertEqual(cl.render_yaml(entries, "v1.21.0"), "{}\n\n") + def test_single_line_impact_emitted_after_pull_request(self): + entries = [entry("core", "feature", "containerd v2", 9, impact="Recreate images.")] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn(" pull_request: https://gl/-/merge_requests/9", out) + self.assertIn(" impact: Recreate images.", out) + + def test_multiline_impact_emitted_as_literal_block(self): + entries = [entry("core", "feature", "containerd v2", 9, impact="L1\nL2")] + out = cl.render_yaml(entries, "v1.21.0") + self.assertIn(" impact: |-", out) + self.assertIn(" L1", out) + self.assertIn(" L2", out) + + def test_no_impact_means_no_impact_line(self): + entries = [entry("vm", "fix", "fixed Y", 11)] + out = cl.render_yaml(entries, "v1.21.0") + self.assertNotIn("impact:", out) -class RenderMarkdownTest(unittest.TestCase): + +class RenderMilestoneMdBlockTest(unittest.TestCase): def test_basic_structure(self): entries = [entry("vm", "fix", "fixed Y", 11, impact_level="high")] - out = cl.render_markdown(entries, "v1.21.0", "v1.21") - self.assertIn("# Changelog v1.21", out) - self.assertIn("## vm", out) + out = cl.render_milestone_md_block(entries, "v1.21.0") + self.assertIn("## v1.21.0", out) + self.assertIn("### vm", out) self.assertIn("**fix** (high): fixed Y ([!11]", out) + def test_empty_entries_render_placeholder(self): + out = cl.render_milestone_md_block([], "v1.21.0") + self.assertIn("## v1.21.0", out) + self.assertIn("_No changelog entries._", out) + + +class MergeMinorMarkdownTest(unittest.TestCase): + def test_new_file_creates_header_and_block(self): + block = cl.render_milestone_md_block( + [entry("vm", "fix", "a", 1)], "v1.21.0" + ) + out = cl.merge_minor_markdown("", "v1.21", "v1.21.0", block) + self.assertIn("# Changelog v1.21", out) + self.assertIn("## v1.21.0", out) + + def test_existing_patch_preserved_and_sorted_desc(self): + first = cl.merge_minor_markdown( + "", "v1.21", "v1.21.0", + cl.render_milestone_md_block([entry("vm", "fix", "older", 1)], "v1.21.0"), + ) + second = cl.merge_minor_markdown( + first, "v1.21", "v1.21.1", + cl.render_milestone_md_block([entry("vm", "fix", "newer", 2)], "v1.21.1"), + ) + # Both patch blocks are present (cumulative)... + self.assertIn("## v1.21.0", second) + self.assertIn("## v1.21.1", second) + self.assertIn("older", second) + self.assertIn("newer", second) + # ...and the newer patch is listed first. + self.assertLess(second.index("## v1.21.1"), second.index("## v1.21.0")) + + def test_regenerating_same_milestone_is_idempotent(self): + block_v0 = cl.render_milestone_md_block( + [entry("vm", "fix", "a", 1)], "v1.21.0" + ) + once = cl.merge_minor_markdown("", "v1.21", "v1.21.0", block_v0) + twice = cl.merge_minor_markdown(once, "v1.21", "v1.21.0", block_v0) + self.assertEqual(once, twice) + class MinorVersionFromTagTest(unittest.TestCase): def test_patch_version_truncated_to_minor(self): From 25092abc7aee98b19a809892da2ee888eb750d12 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 18:38:24 +0300 Subject: [PATCH 53/60] docs(ci): fix misplaced dedup comment in workflow.yml The "avoid duplicate pipelines" comment sat above the default-branch rule, which does not skip anything. The branch-pipeline skip is actually done by the `push && $CI_OPEN_MERGE_REQUESTS -> when: never` rule. Move the explanation onto that rule, and give the default-branch and tag rules their own accurate comments. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/ci/workflow.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.gitlab/ci/workflow.yml b/.gitlab/ci/workflow.yml index 8acfacd676..dc990d1a33 100644 --- a/.gitlab/ci/workflow.yml +++ b/.gitlab/ci/workflow.yml @@ -20,17 +20,22 @@ workflow: rules: # MR pipelines always run. - if: $CI_PIPELINE_SOURCE == "merge_request_event" - # Avoid duplicate pipelines: when a branch push has an open MR, GitLab - # would otherwise create both a branch pipeline and an MR pipeline. Skip - # the redundant branch pipeline and keep only the MR one. (The default - # branch and release/tag pushes are handled by the explicit rules below, - # which take precedence because rules are first-match.) + # Default-branch (main) pushes always create a pipeline. Listed before the + # `when: never` dedup rule below so a main push is never skipped. - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # Tag pushes (vX.Y.Z / vX.Y.Z-dev*) always create a pipeline (also matched + # before the dedup rule). - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(-dev.*)?$/ - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_PIPELINE_SOURCE == "web" - # Plain branch pushes (non-default, non-tag) only create a pipeline when - # there is no open MR for the branch, preventing duplicate pipelines. + # Avoid duplicate pipelines: a push to a non-default, non-tag branch that + # already has an open MR would otherwise create BOTH a branch pipeline and + # an MR pipeline. The predefined $CI_OPEN_MERGE_REQUESTS is non-empty only + # when the pushed branch is the source of an open MR, so THIS rule is what + # skips the redundant branch pipeline (keeping only the MR one). main / tags + # / schedule / web are matched by the earlier rules and never reach here + # (rules are first-match). - if: $CI_PIPELINE_SOURCE == "push" && $CI_OPEN_MERGE_REQUESTS when: never + # Plain branch push with no open MR: create the branch pipeline. - if: $CI_PIPELINE_SOURCE == "push" From a3080d812d4dfb4bdec3fe40464ebffe10582ea2 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 19:06:35 +0300 Subject: [PATCH 54/60] chore(ci): drop internal planning-doc references from CI comments Remove references to the internal planning document (section markers and tmp/ paths) from comments across .gitlab/** and the root .gitlab-ci.yml, and replace the absolute local checkout paths (/Users/...) with the canonical upstream repo name (deckhouse/3p/deckhouse/modules-gitlab-ci). Comments keep their substance; only the internal/local references are dropped. No behavior change. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab-ci.yml | 5 ++--- .gitlab/ci/includes.yml | 7 +++---- .gitlab/ci/jobs/auto-assign-author.yml | 2 +- .gitlab/ci/jobs/backport.yml | 6 +++--- .gitlab/ci/jobs/build-dev.yml | 2 +- .gitlab/ci/jobs/build-prod.yml | 2 +- .gitlab/ci/jobs/changelog.yml | 3 +-- .gitlab/ci/jobs/check-changelog.yml | 4 ++-- .gitlab/ci/jobs/deploy-prod.yml | 5 ++--- .gitlab/ci/jobs/lint-validate.yml | 3 +-- .gitlab/ci/jobs/manual-tools.yml | 6 +++--- .gitlab/ci/jobs/release-channels.yml | 2 +- .gitlab/ci/jobs/translate-changelog.yml | 7 +++---- .gitlab/ci/scripts/bash/auto-assign-author.sh | 4 ++-- .gitlab/ci/scripts/bash/backport.sh | 2 +- .gitlab/ci/scripts/bash/changelog-milestone.sh | 2 +- .gitlab/ci/scripts/bash/check-changelog-entry.sh | 2 +- .gitlab/ci/scripts/bash/check-milestone.sh | 2 +- .gitlab/ci/scripts/bash/gitlab-ci-lint.sh | 3 +-- .gitlab/ci/scripts/bash/lib/api.sh | 2 +- .gitlab/ci/scripts/bash/set-vars.sh | 2 +- .gitlab/ci/scripts/python/changelog_collect.py | 2 +- .gitlab/ci/scripts/python/check_changelog_entry.py | 2 +- .gitlab/ci/stages.yml | 9 ++++----- .gitlab/ci/templates/base/build.yml | 2 +- .gitlab/ci/templates/base/deploy.yml | 3 +-- .gitlab/ci/templates/base/dual_registry_login.yml | 6 +++--- .gitlab/ci/templates/dev/dev_vars.yml | 3 +-- .gitlab/ci/templates/release/prod_manual.yml | 11 +++++------ .gitlab/ci/templates/release/prod_vars.yml | 3 +-- .gitlab/ci/variables.yml | 2 +- .gitlab/ci/workflow.yml | 2 +- .gitlab/scripts/js/mrs_notifier.mjs | 2 +- 33 files changed, 54 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf8820a1e4..cd2ae98823 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,9 +9,8 @@ # .gitlab/ci/includes.yml (see that file for the full include list). # # Upstream templates are sourced from deckhouse/3p/deckhouse/modules-gitlab-ci -# at ref v13.0 (HEAD 006d51c35904b434eca2045a449aafb5e37a8827). The plan -# calls for swapping the branch ref for a pinned SHA after the first green -# pipeline on virt-test. +# at ref v13.0 (HEAD 006d51c35904b434eca2045a449aafb5e37a8827). Swap the +# branch ref for a pinned SHA after the first green pipeline on virt-test. include: - local: '.gitlab/ci/includes.yml' diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index 5dce1ff657..ab4e6758b6 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -7,8 +7,7 @@ # so the final list is the union of both. # # Upstream template names (verified in -# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci, -# branch v13.0, HEAD 006d51c): +# deckhouse/3p/deckhouse/modules-gitlab-ci, branch v13.0, HEAD 006d51c): # - Setup.gitlab-ci.yml (trdl + werf + ci-env + dual-registry login) # - Build.gitlab-ci.yml (extends .build — werf build + bundle copy + # release-channel copy + module registration) @@ -30,8 +29,8 @@ include: # --- Upstream modules-gitlab-ci (deckhouse/3p, ref v13.0) --- - # TODO: pin to SHA after first green pipeline on virt-test; see migration - # plan §11.2. Until then, branch ref v13.0 keeps fixes flowing. + # TODO: pin to SHA after first green pipeline on virt-test. Until then, + # branch ref v13.0 keeps fixes flowing. - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" ref: "v13.0" file: diff --git a/.gitlab/ci/jobs/auto-assign-author.yml b/.gitlab/ci/jobs/auto-assign-author.yml index defe0749ee..177d968d35 100644 --- a/.gitlab/ci/jobs/auto-assign-author.yml +++ b/.gitlab/ci/jobs/auto-assign-author.yml @@ -15,7 +15,7 @@ # Auto-assign MR author as MR assignee (GitLab API). # # Migration of .github/workflows/dev_auto-pr-author-assign.yml. -# Behaviour per migration plan §0(4): if MR already has an assignee, skip. +# Behaviour: if MR already has an assignee, skip. # Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). auto-assign-author: diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index 3340bc46dc..cf864eebd6 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -16,10 +16,10 @@ # # Migration of .github/workflows/on-pull-request-backport.yml which used # deckhouse/backport-action@v1.0.0 with automerge=true and direct push to -# the release branch. Per migration plan §0(6) and §11.9 we open a -# reviewable backport MR instead of pushing directly. +# the release branch. We open a reviewable backport MR instead of pushing +# directly. # -# Triggers (per plan §11.9.4): +# Triggers: # 1. Manual pipeline (Run pipeline UI) with variable TARGET_BRANCH=release-1.21. # 2. Manual pipeline on an MR carrying the `status/backport` label. # GitLab does NOT auto-trigger pipelines on label change; the user must diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml index a67b34c756..d89d2e8678 100644 --- a/.gitlab/ci/jobs/build-dev.yml +++ b/.gitlab/ci/jobs/build-dev.yml @@ -12,7 +12,7 @@ # # The previous root .gitlab-ci.yml had its own `.build` template with the # same body; it is replaced here by `.base_build` (verified against -# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci +# deckhouse/3p/deckhouse/modules-gitlab-ci # templates/Build.gitlab-ci.yml — werf build with --save-build-report, # bundle / release-channel-version crane copy, and crane append to # register the module). The upstream version uses WERF_REPO env instead of diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml index 7eb90ce4e2..6cdf29ff2d 100644 --- a/.gitlab/ci/jobs/build-prod.yml +++ b/.gitlab/ci/jobs/build-prod.yml @@ -27,7 +27,7 @@ # `.dual_registry_login` is included so the second `werf cr login` from # upstream Setup.gitlab-ci.yml also fires against DEV_REGISTRY; the # previous GH workflow did both logins via two consecutive -# modules-actions/setup steps, and the migration plan keeps that behavior. +# modules-actions/setup steps; we keep that behavior. build_prod: stage: build diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index 2f9292c385..ce49ed2723 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -20,8 +20,7 @@ # - .github/workflows/changelog-command.yml (repository_dispatch /changelog) # All three used ./.github/actions/milestone-changelog (composite action). # -# Per migration plan §0(3) and §11.5.4 the GitLab jobs are manual + scheduled, -# NOT reactive. Reactive webhook automation (slash-command-equivalents, +# The GitLab jobs are manual + scheduled, NOT reactive. Reactive webhook automation (slash-command-equivalents, # label/milestone events) is an accepted permanent gap — no webhook-listener # will be built. # diff --git a/.gitlab/ci/jobs/check-changelog.yml b/.gitlab/ci/jobs/check-changelog.yml index 04091bbb36..45886ff91c 100644 --- a/.gitlab/ci/jobs/check-changelog.yml +++ b/.gitlab/ci/jobs/check-changelog.yml @@ -16,8 +16,8 @@ # # Migration of .github/workflows/check-changelog-entry.yml which used # deckhouse/changelog-action@v2.6.0 with validate_only=true. -# Per migration plan §11.11 the validation is implemented in Python and -# uses .gitlab/ci/changelog-sections.txt as the source of truth. +# The validation is implemented in Python and uses +# .gitlab/ci/changelog-sections.txt as the source of truth. # Required CI/CD variable: GITLAB_API_TOKEN (Project Access Token, scope api). check:changelog: diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index 01ae5ac019..9c66229251 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -27,9 +27,8 @@ # with `when: manual`). # - .dual_registry_login (this repo — also login against DEV_REGISTRY). # -# TODO: per migration plan §11.4.1, the -# GH release_module_release-channels workflow exposes inputs (channel, -# ce/ee, tag, enableBuild, release_to_github, check_only, +# TODO: the GH release_module_release-channels workflow exposes inputs +# (channel, ce/ee, tag, enableBuild, release_to_github, check_only, # send_results_to_loop, requirements/version checks, github release # creation, Loop notification) that are not yet ported. # diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 857a174184..7a42fd96bc 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -4,8 +4,7 @@ # .github/workflows/dev_validation.yaml (paths_filter + no_cyrillic + # doc_changes + shellcheck + helm_templates + check_gens_files). # -# Per the migration plan (decision #5), the `actionlint` job from the GH -# workflow is intentionally NOT migrated. In its place we run the new +# The `actionlint` job from the GH workflow is intentionally NOT migrated. In its place we run the new # `lint:gitlab-ci` job that validates the GitLab CI configuration itself # via the GitLab CI Lint API. The legacy skip-label # `validation/skip/actionlint` is therefore obsolete and not honored. diff --git a/.gitlab/ci/jobs/manual-tools.yml b/.gitlab/ci/jobs/manual-tools.yml index 24379bc367..dc9ce41ca1 100644 --- a/.gitlab/ci/jobs/manual-tools.yml +++ b/.gitlab/ci/jobs/manual-tools.yml @@ -18,9 +18,9 @@ # - mrs:summary : post a Loop summary of open MRs (GitLab counterpart of # .github/workflows/dev_prs-summary.yml / .github/scripts/prs_notifier.mjs). # -# Migration plan §0(1) explicitly removes GitHub slash-command dispatch -# (/.github/workflows/dispatch-slash-command.yml) and replaces it with -# manual pipelines. The two manual jobs in this file are the closest +# GitHub slash-command dispatch +# (/.github/workflows/dispatch-slash-command.yml) is removed and replaced +# with manual pipelines. The two manual jobs in this file are the closest # equivalents. There is no automated webhook listener, and this is an # accepted permanent gap (no webhook-listener will be built). diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml index 93ef8208aa..a36faa4169 100644 --- a/.gitlab/ci/jobs/release-channels.yml +++ b/.gitlab/ci/jobs/release-channels.yml @@ -32,7 +32,7 @@ # fails fast instead of building the wrong ref. GitHub let `tag` differ # from the workflow ref (it checked out ref:inputs.tag); GitLab cannot # do that without overriding the inherited Setup before_script, which -# the migration plan forbids. +# we avoid. # - Per-(channel,tag) concurrency uses resource_group (a mutex); GitLab has # no cancel-in-progress equivalent, so concurrent same-(channel,tag) # dispatches serialize instead of cancelling. diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index 4d934e83f7..7beb137a0a 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -43,10 +43,9 @@ # RELEASE_TOKEN explicitly in project CI/CD variables if CI_JOB_TOKEN lacks # push / merge-request permissions. # -# Per migration plan §0(2) we delegate to the upstream GitLab CI template at -# https://fox.flant.com/deckhouse/3p/deckhouse/modules-gitlab-ci (local checkout -# at /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci, -# branch v13.0, HEAD 006d51c35904b434eca2045a449aafb5e37a8827). +# We delegate to the upstream GitLab CI template at +# https://fox.flant.com/deckhouse/3p/deckhouse/modules-gitlab-ci +# (branch v13.0, HEAD 006d51c35904b434eca2045a449aafb5e37a8827). include: - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" diff --git a/.gitlab/ci/scripts/bash/auto-assign-author.sh b/.gitlab/ci/scripts/bash/auto-assign-author.sh index c560891eb8..d465065beb 100644 --- a/.gitlab/ci/scripts/bash/auto-assign-author.sh +++ b/.gitlab/ci/scripts/bash/auto-assign-author.sh @@ -20,7 +20,7 @@ # Migration of .github/workflows/dev_auto-pr-author-assign.yml which used the # third-party toshimaru/auto-author-assign@v2.1.0 action. # -# Behaviour (per migration plan §0 / §11): +# Behaviour: # - Skip silently if MR already has at least one assignee. # - Otherwise assign the MR author (the user who opened the MR). # - Token: GITLAB_API_TOKEN (Project Access Token, scope api). @@ -57,7 +57,7 @@ assignee_count="$(printf '%s' "$mr_json" | jq -r '.assignees | length')" echo "Current assignee count: ${assignee_count}" if [[ "${assignee_count}" -gt 0 ]]; then - echo "MR already has ${assignee_count} assignee(s) — skipping auto-assign per plan §0(4)." + echo "MR already has ${assignee_count} assignee(s) — skipping auto-assign." exit 0 fi diff --git a/.gitlab/ci/scripts/bash/backport.sh b/.gitlab/ci/scripts/bash/backport.sh index f50aebf946..a706ef8994 100644 --- a/.gitlab/ci/scripts/bash/backport.sh +++ b/.gitlab/ci/scripts/bash/backport.sh @@ -20,7 +20,7 @@ # Migration of .github/workflows/on-pull-request-backport.yml which used # deckhouse/backport-action@v1.0.0 and direct cherry-pick to release branch. # -# Per migration plan §0(6) and §11.9 we DO NOT use the GitLab cherry-pick +# We DO NOT use the GitLab cherry-pick # REST endpoint (POST /repository/commits/:sha/cherry_pick) because it # bypasses code review. Instead we: # 1. clone the repo (or reuse the runner workspace), diff --git a/.gitlab/ci/scripts/bash/changelog-milestone.sh b/.gitlab/ci/scripts/bash/changelog-milestone.sh index 1e95490f9c..9ddc069689 100644 --- a/.gitlab/ci/scripts/bash/changelog-milestone.sh +++ b/.gitlab/ci/scripts/bash/changelog-milestone.sh @@ -16,7 +16,7 @@ # Thin wrapper around changelog_collect.py to keep the job yml language-agnostic. # # Selects python3 / python at runtime. The actual logic lives in Python -# (preferred per migration plan §11.5.3 Variant B). +# (Variant B). set -euo pipefail diff --git a/.gitlab/ci/scripts/bash/check-changelog-entry.sh b/.gitlab/ci/scripts/bash/check-changelog-entry.sh index ba603ef871..46aabfb3b4 100644 --- a/.gitlab/ci/scripts/bash/check-changelog-entry.sh +++ b/.gitlab/ci/scripts/bash/check-changelog-entry.sh @@ -16,7 +16,7 @@ # Thin bash wrapper around check_changelog_entry.py. # # Allows the job yml to call a single bash script while keeping the -# actual validation logic in Python (preferred per migration plan §11.11.2). +# actual validation logic in Python. # # Picks the first available interpreter: python3, python. # Required environment is documented in check_changelog_entry.py. diff --git a/.gitlab/ci/scripts/bash/check-milestone.sh b/.gitlab/ci/scripts/bash/check-milestone.sh index be96739a57..c7fca25b8a 100644 --- a/.gitlab/ci/scripts/bash/check-milestone.sh +++ b/.gitlab/ci/scripts/bash/check-milestone.sh @@ -20,7 +20,7 @@ # Migration of .github/workflows/check-pr-milestone.yml which used # actions/github-script@v6.4.1 to GET the PR and assert data.milestone. # -# Behaviour (per plan §0): +# Behaviour: # - On MR pipelines: GET MR via API, ensure milestone is present. # - On other pipelines: no-op (print "skipping"). # - Skip-labels respected (see rules in job yml). diff --git a/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh index 5b8db06531..47fc563a79 100755 --- a/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh +++ b/.gitlab/ci/scripts/bash/gitlab-ci-lint.sh @@ -21,8 +21,7 @@ # with a `content` payload assembled from the project files that make # up the effective CI configuration. # -# Migration plan §11.14 specified a single-document lint, which is what -# GitLab's lint API supports per request. We therefore lint the root +# A single-document lint is what GitLab's lint API supports per request. We therefore lint the root # `.gitlab-ci.yml` directly. The upstream project owner is responsible # for keeping `.gitlab/ci/includes.yml` and the `local:` job files # self-consistent; this script only checks that the *merged* file the diff --git a/.gitlab/ci/scripts/bash/lib/api.sh b/.gitlab/ci/scripts/bash/lib/api.sh index 96c4213b55..8c0ec1c0da 100755 --- a/.gitlab/ci/scripts/bash/lib/api.sh +++ b/.gitlab/ci/scripts/bash/lib/api.sh @@ -25,7 +25,7 @@ # gl_required_env -- fails if required env vars are missing. # gl_log_call -- echoes request line for log readability. # -# Conventions (see tmp/ai-summary/gitlab-ci-migration-plan.md §11.1): +# Conventions: # - Always CI_API_V4_URL (never hardcode the host). # - Always GITLAB_API_TOKEN (Project Access Token, scope api). # - Always CI_PROJECT_ID (numeric) and CI_MERGE_REQUEST_IID (iid, not id). diff --git a/.gitlab/ci/scripts/bash/set-vars.sh b/.gitlab/ci/scripts/bash/set-vars.sh index 4fa4a32ff1..141f4fbd8a 100755 --- a/.gitlab/ci/scripts/bash/set-vars.sh +++ b/.gitlab/ci/scripts/bash/set-vars.sh @@ -17,7 +17,7 @@ # set-vars.sh — derives per-pipeline variables for downstream jobs. # # Carries forward the responsibilities of the GH `set_vars` job from -# dev_module_build.yml (migration plan §11.3.4). Produces a dotenv +# dev_module_build.yml. Produces a dotenv # artifact that downstream jobs consume via `needs: [set_vars]` + # `artifacts.reports.dotenv`. # diff --git a/.gitlab/ci/scripts/python/changelog_collect.py b/.gitlab/ci/scripts/python/changelog_collect.py index 1fea77386d..c7586563af 100644 --- a/.gitlab/ci/scripts/python/changelog_collect.py +++ b/.gitlab/ci/scripts/python/changelog_collect.py @@ -18,7 +18,7 @@ Migration of .github/actions/milestone-changelog/action.yml (composite action) which used deckhouse/changelog-action@v2.6.0. -Strategy chosen per migration plan §11.5.3 (Variant B - rewrite in python). +Strategy: rewrite the parser in Python (Variant B). Behaviour: 1. Resolve target milestone from MILESTONE_TITLE or list open milestones. diff --git a/.gitlab/ci/scripts/python/check_changelog_entry.py b/.gitlab/ci/scripts/python/check_changelog_entry.py index 68a848dd35..4979e515f5 100644 --- a/.gitlab/ci/scripts/python/check_changelog_entry.py +++ b/.gitlab/ci/scripts/python/check_changelog_entry.py @@ -19,7 +19,7 @@ .github/workflows/check-changelog-entry.yml which used deckhouse/changelog-action@v2.6.0 with validate_only=true. -Behaviour (per migration plan §11.11): +Behaviour: - Fetch MR description via GitLab API (CI_API_V4_URL). - Locate fenced code blocks with language ``changes``. - For each block validate required keys: section, type, summary. diff --git a/.gitlab/ci/stages.yml b/.gitlab/ci/stages.yml index 6db3044dff..198f7ef040 100644 --- a/.gitlab/ci/stages.yml +++ b/.gitlab/ci/stages.yml @@ -1,12 +1,11 @@ # Pipeline stages for the deckhouse/virtualization module. # # This is the union of the stages used by the previous root .gitlab-ci.yml -# and the new stages introduced by the GitHub Actions migration plan -# (tmp/ai-summary/gitlab-ci-migration-plan.md §11.8). +# and the new stages introduced during the GitHub Actions migration. # -# We intentionally drop the `e2e` stage here: the migration plan explicitly -# excludes all e2e-* workflows from the GitLab CI migration. A child issue -# owns any future re-introduction of e2e stages under a separate file. +# We intentionally drop the `e2e` stage here: all e2e-* workflows are +# excluded from the GitLab CI migration. Any future re-introduction of e2e +# stages lives in a separate file. # # Stage semantics: # pre — upstream Translate_Changelog hidden template stage diff --git a/.gitlab/ci/templates/base/build.yml b/.gitlab/ci/templates/base/build.yml index 6f18857048..ae4b04c773 100644 --- a/.gitlab/ci/templates/base/build.yml +++ b/.gitlab/ci/templates/base/build.yml @@ -8,7 +8,7 @@ # # This template mirrors the upstream modules-gitlab-ci Build.gitlab-ci.yml # `.build` body verbatim (verified against -# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci +# deckhouse/3p/deckhouse/modules-gitlab-ci # templates/Build.gitlab-ci.yml, branch v13.0, HEAD 006d51c). # # Why a local copy instead of `extends: .build`: diff --git a/.gitlab/ci/templates/base/deploy.yml b/.gitlab/ci/templates/base/deploy.yml index 90b527f25c..6530fdfddc 100644 --- a/.gitlab/ci/templates/base/deploy.yml +++ b/.gitlab/ci/templates/base/deploy.yml @@ -8,8 +8,7 @@ # dev tags are build-only (see build_dev_tags). # # Mirrors upstream modules-gitlab-ci Deploy.gitlab-ci.yml `.deploy` body -# (verified in -# /Users/korolevn/repos/Virtualization-tasks/github/3p-deckhouse/modules-gitlab-ci +# (verified in deckhouse/3p/deckhouse/modules-gitlab-ci # templates/Deploy.gitlab-ci.yml, branch v13.0, HEAD 006d51c). # # Why a local copy (same reasoning as .base_build in build.yml): diff --git a/.gitlab/ci/templates/base/dual_registry_login.yml b/.gitlab/ci/templates/base/dual_registry_login.yml index 6a7d52dbb5..645cb28037 100644 --- a/.gitlab/ci/templates/base/dual_registry_login.yml +++ b/.gitlab/ci/templates/base/dual_registry_login.yml @@ -29,9 +29,9 @@ # - .build # upstream werf build + bundle/release-channel copy # # TODO: confirm with a virt-test pipeline that the second login is actually -# required for the build_prod / deploy_to_prod_* chain (the plan §11.7.3 -# flags this as an open question because the previous GH workflow did both -# logins via two consecutive `modules-actions/setup` steps). +# required for the build_prod / deploy_to_prod_* chain (an open question, +# because the previous GH workflow did both logins via two consecutive +# `modules-actions/setup` steps). .dual_registry_login: variables: diff --git a/.gitlab/ci/templates/dev/dev_vars.yml b/.gitlab/ci/templates/dev/dev_vars.yml index b878b97860..2ba08f5b56 100644 --- a/.gitlab/ci/templates/dev/dev_vars.yml +++ b/.gitlab/ci/templates/dev/dev_vars.yml @@ -3,8 +3,7 @@ # Carries forward the previous root .gitlab-ci.yml `.dev_vars` template. # Anything that uses `extends: .dev_vars` gets MODULES_REGISTRY, # MODULES_REGISTRY_LOGIN, MODULES_REGISTRY_PASSWORD, MODULES_MODULE_SOURCE -# and ENV=DEV, all sourced from the DEV_* Project Variables introduced in -# plan §11.6. +# and ENV=DEV, all sourced from the DEV_* Project Variables. # # The legacy EXTERNAL_MODULES_DEV_REGISTRY_* names are still accepted as # fallback to ease the variable-rename migration. diff --git a/.gitlab/ci/templates/release/prod_manual.yml b/.gitlab/ci/templates/release/prod_manual.yml index 2efecd26eb..55621be077 100644 --- a/.gitlab/ci/templates/release/prod_manual.yml +++ b/.gitlab/ci/templates/release/prod_manual.yml @@ -4,12 +4,11 @@ # Runs on tags matching vX.Y.Z (no -dev suffix), with `when: manual` so # each release channel deploy requires a human click. # -# TODO: per plan §11.4.1, the GH release_module_release-channels.yml -# workflow exposes inputs (channel, ce/ee, tag, enableBuild, -# release_to_github, check_only, ...) that we currently collapse into -# hardcoded matrix (RELEASE_CHANNEL x EDITION) in deploy-prod.yml. Once we -# decide whether to expose that flexibility (likely via "Run pipeline" -# variables), this template will gain extra variables. +# TODO: the GH release_module_release-channels.yml workflow exposes inputs +# (channel, ce/ee, tag, enableBuild, release_to_github, check_only, ...) that +# we currently collapse into a hardcoded matrix (RELEASE_CHANNEL x EDITION) in +# deploy-prod.yml. Once we decide whether to expose that flexibility (likely +# via "Run pipeline" variables), this template will gain extra variables. # # The single-channel dispatch with Run-pipeline inputs (channel, editions, # tag, enableBuild, check_only, release_to_gitlab, send_results_to_loop, diff --git a/.gitlab/ci/templates/release/prod_vars.yml b/.gitlab/ci/templates/release/prod_vars.yml index cfd8029f52..abe58a4bf5 100644 --- a/.gitlab/ci/templates/release/prod_vars.yml +++ b/.gitlab/ci/templates/release/prod_vars.yml @@ -3,8 +3,7 @@ # Carries forward the previous root .gitlab-ci.yml `.prod_vars` template. # Anything that uses `extends: .prod_vars` gets MODULES_REGISTRY, # MODULES_REGISTRY_LOGIN, MODULES_REGISTRY_PASSWORD, MODULES_MODULE_SOURCE -# and ENV=PROD, all sourced from the PROD_* Project Variables introduced in -# plan §11.6. +# and ENV=PROD, all sourced from the PROD_* Project Variables. # # MODULES_MODULE_SOURCE is composed dynamically from PROD_REGISTRY, # PROD_MODULE_SOURCE_NAME, and the matrix-driven EDITION variable so that diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index c5ff3f6685..341af5cb82 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -4,7 +4,7 @@ # docker configs) lives in GitLab Settings -> CI/CD -> Variables (masked # where appropriate) and is referenced via ${VAR} expansion below. # -# Legacy variable migration (plan §11.6): +# Legacy variable migration: # EXTERNAL_MODULES_DEV_REGISTRY_LOGIN -> DEV_MODULES_REGISTRY_LOGIN # EXTERNAL_MODULES_DEV_REGISTRY_PASSWORD -> DEV_MODULES_REGISTRY_PASSWORD # EXTERNAL_MODULES_PROD_REGISTRY_LOGIN -> PROD_MODULES_REGISTRY_LOGIN diff --git a/.gitlab/ci/workflow.yml b/.gitlab/ci/workflow.yml index dc990d1a33..412e9c1798 100644 --- a/.gitlab/ci/workflow.yml +++ b/.gitlab/ci/workflow.yml @@ -12,7 +12,7 @@ # - Manual/scheduled/web pipelines always run. # # TODO: once auto-cancel-redundant-pipelines is verified at the project -# level, add `interruptible: true` here as a project default (see plan §11.15). +# level, add `interruptible: true` here as a project default. # Per-job overrides will set `interruptible: false` for build_prod and the # prod-deploy chain. diff --git a/.gitlab/scripts/js/mrs_notifier.mjs b/.gitlab/scripts/js/mrs_notifier.mjs index cf00369976..edc84cca55 100644 --- a/.gitlab/scripts/js/mrs_notifier.mjs +++ b/.gitlab/scripts/js/mrs_notifier.mjs @@ -29,7 +29,7 @@ // MANAGER_LOOP_NAME (optional) @firstname.lastname of the manager. // Default "@yuriy.milyutin". // -// Mapping cheat-sheet (per migration plan §11.12.2): +// Mapping cheat-sheet: // octokit -> axios with PRIVATE-TOKEN // pr.draft -> mr.draft (or mr.work_in_progress for older GitLab) // pr.head.ref -> mr.source_branch From 72b6a6ffd193418b15472a53b09fc28d5a170ab0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 19:19:42 +0300 Subject: [PATCH 55/60] refactor(ci): consolidate registry-login templates and dedupe prod tag context Two template cleanups, both behavior-preserving (same variables/rules emitted). Registry login: - Replace the misleadingly named .dual_registry_login (it only configured the DEV registry, not "two") with a single base/registry-login.yml that names each login explicitly: .login_dev_registry and .login_prod_registry (primary registry select), .login_prod_read_registry (read-only checks), and .also_login_dev_registry (the extra DEV login + werf secondary repo a prod build composes to also read DEV). This mirrors the GitHub pipeline, which logged into the prod and dev registries explicitly. - .dev_vars / .prod_vars now extend the matching .login_*_registry, so the registry credentials live in one file and the vars bundles only add MODULES_MODULE_SOURCE + ENV. Move .prod_read_registry there as .login_prod_read_registry. Prod tag context: - prod_always.yml and prod_manual.yml were identical except the rule's `when:` (always vs manual). Merge them into prod_tag.yml with a shared .prod_tag base; .prod_always / .prod_manual now only carry their rule. Anchor names used by jobs (.dev_vars, .prod_vars, .prod_always, .prod_manual) are unchanged; only .dual_registry_login -> .also_login_dev_registry and .prod_read_registry -> .login_prod_read_registry references were updated. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/ci/includes.yml | 7 ++- .gitlab/ci/jobs/build-prod.yml | 4 +- .gitlab/ci/jobs/deploy-prod.yml | 12 ++-- .gitlab/ci/jobs/release-channels.yml | 20 +++---- .../ci/templates/base/dual_registry_login.yml | 41 ------------- .gitlab/ci/templates/base/registry-login.yml | 57 +++++++++++++++++++ .gitlab/ci/templates/dev/dev_vars.yml | 15 ++--- .gitlab/ci/templates/release/prod_always.yml | 17 ------ .gitlab/ci/templates/release/prod_manual.yml | 30 ---------- .gitlab/ci/templates/release/prod_tag.yml | 39 +++++++++++++ .gitlab/ci/templates/release/prod_vars.yml | 27 +++------ 11 files changed, 130 insertions(+), 139 deletions(-) delete mode 100644 .gitlab/ci/templates/base/dual_registry_login.yml create mode 100644 .gitlab/ci/templates/base/registry-login.yml delete mode 100644 .gitlab/ci/templates/release/prod_always.yml delete mode 100644 .gitlab/ci/templates/release/prod_manual.yml create mode 100644 .gitlab/ci/templates/release/prod_tag.yml diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index ab4e6758b6..da0a679736 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -62,16 +62,17 @@ include: # release/ - release process (PROD registry): tag-push + manual dispatch # Anchor names are unchanged (.base_build, .dev, .prod_vars, ...), so no job # `extends:` edits are needed. + # Registry-login helpers must load before dev_vars/prod_vars, which extend + # them (.login_dev_registry / .login_prod_registry). + - local: ".gitlab/ci/templates/base/registry-login.yml" - local: ".gitlab/ci/templates/dev/dev_vars.yml" - local: ".gitlab/ci/templates/release/prod_vars.yml" - local: ".gitlab/ci/templates/dev/dev.yml" - local: ".gitlab/ci/templates/dev/dev_tags.yml" - local: ".gitlab/ci/templates/dev/main.yml" - local: ".gitlab/ci/templates/dev/release.yml" - - local: ".gitlab/ci/templates/release/prod_manual.yml" - - local: ".gitlab/ci/templates/release/prod_always.yml" + - local: ".gitlab/ci/templates/release/prod_tag.yml" - local: ".gitlab/ci/templates/base/info.yml" - - local: ".gitlab/ci/templates/base/dual_registry_login.yml" - local: ".gitlab/ci/templates/base/build.yml" - local: ".gitlab/ci/templates/base/deploy.yml" diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml index 6cdf29ff2d..6d30340d12 100644 --- a/.gitlab/ci/jobs/build-prod.yml +++ b/.gitlab/ci/jobs/build-prod.yml @@ -24,7 +24,7 @@ # EDITION/modules), and WERF_REPO defaults to MODULES_MODULE_SOURCE/ # MODULES_MODULE_NAME in upstream Setup — correct for prod. # -# `.dual_registry_login` is included so the second `werf cr login` from +# `.also_login_dev_registry` is included so the second `werf cr login` from # upstream Setup.gitlab-ci.yml also fires against DEV_REGISTRY; the # previous GH workflow did both logins via two consecutive # modules-actions/setup steps; we keep that behavior. @@ -36,7 +36,7 @@ build_prod: extends: - .base_build - .prod_always - - .dual_registry_login + - .also_login_dev_registry parallel: matrix: - EDITION: ce diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index 9c66229251..692fd30a6d 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -25,7 +25,7 @@ # gating). # - .prod_manual (this repo — PROD registry vars + tag-match rule # with `when: manual`). -# - .dual_registry_login (this repo — also login against DEV_REGISTRY). +# - .also_login_dev_registry (this repo — also login against DEV_REGISTRY). # # TODO: the GH release_module_release-channels workflow exposes inputs # (channel, ce/ee, tag, enableBuild, release_to_github, check_only, @@ -47,7 +47,7 @@ deploy_to_prod_alpha: extends: - .base_deploy - .prod_manual - - .dual_registry_login + - .also_login_dev_registry parallel: matrix: - EDITION: ce @@ -67,7 +67,7 @@ deploy_to_prod_beta: extends: - .base_deploy - .prod_manual - - .dual_registry_login + - .also_login_dev_registry parallel: matrix: - EDITION: ce @@ -87,7 +87,7 @@ deploy_to_prod_ea: extends: - .base_deploy - .prod_manual - - .dual_registry_login + - .also_login_dev_registry parallel: matrix: - EDITION: ce @@ -107,7 +107,7 @@ deploy_to_prod_stable: extends: - .base_deploy - .prod_manual - - .dual_registry_login + - .also_login_dev_registry parallel: matrix: - EDITION: ce @@ -127,7 +127,7 @@ deploy_to_prod_rock_solid: extends: - .base_deploy - .prod_manual - - .dual_registry_login + - .also_login_dev_registry parallel: matrix: - EDITION: ce diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml index a36faa4169..e52d91bbf9 100644 --- a/.gitlab/ci/jobs/release-channels.yml +++ b/.gitlab/ci/jobs/release-channels.yml @@ -121,7 +121,7 @@ prod:check-requirements: stage: prod_check extends: - .prod_release_rules - - .prod_read_registry + - .login_prod_read_registry needs: ["prod:print-vars"] variables: MODULES_MODULE_TAG: "${RELEASE_TAG}" @@ -157,7 +157,7 @@ prod:build:ce: - .prod_release_rules - .base_build - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: - job: prod:check-requirements optional: true @@ -176,7 +176,7 @@ prod:deploy:ce: - .prod_release_rules - .prod_deploy_script - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: - job: prod:check-requirements optional: true @@ -198,7 +198,7 @@ prod:build:ee: - .prod_release_rules - .base_build - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: - job: prod:check-requirements optional: true @@ -217,7 +217,7 @@ prod:deploy:ee: - .prod_release_rules - .prod_deploy_script - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: - job: prod:check-requirements optional: true @@ -239,7 +239,7 @@ prod:build:se-plus: - .prod_release_rules - .base_build - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: ["prod:build:ee"] variables: EDITION: se-plus @@ -256,7 +256,7 @@ prod:deploy:se-plus: - .prod_release_rules - .prod_deploy_script - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: - job: prod:build:se-plus optional: true @@ -276,7 +276,7 @@ prod:build:fe: - .prod_release_rules - .base_build - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: ["prod:build:ee"] variables: EDITION: fe @@ -293,7 +293,7 @@ prod:deploy:fe: - .prod_release_rules - .prod_deploy_script - .prod_vars - - .dual_registry_login + - .also_login_dev_registry needs: - job: prod:build:fe optional: true @@ -312,7 +312,7 @@ prod:check-version: stage: prod_verify extends: - .prod_release_rules - - .prod_read_registry + - .login_prod_read_registry needs: - job: prod:deploy:ce optional: true diff --git a/.gitlab/ci/templates/base/dual_registry_login.yml b/.gitlab/ci/templates/base/dual_registry_login.yml deleted file mode 100644 index 645cb28037..0000000000 --- a/.gitlab/ci/templates/base/dual_registry_login.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Dual-registry login helper for prod release jobs. -# -# The upstream modules-gitlab-ci Setup.gitlab-ci.yml (templates/Setup.gitlab-ci.yml) -# does: -# werf cr login -u $MODULES_REGISTRY_LOGIN -p $MODULES_REGISTRY_PASSWORD $MODULES_REGISTRY -# if [[ -n $DEV_MODULES_REGISTRY_LOGIN && ... ]]; then -# werf cr login -u $DEV_MODULES_REGISTRY_LOGIN -p $DEV_MODULES_REGISTRY_PASSWORD $DEV_MODULES_REGISTRY -# fi -# -# That means a job only needs to set DEV_MODULES_REGISTRY_* in addition to -# MODULES_REGISTRY_* to get a second login. We expose DEV_MODULES_REGISTRY_* -# via this helper so prod release jobs (which need to copy from DEV and -# register into PROD) can extend both .prod_vars and .dual_registry_login -# and end up with the right pair of `werf cr login` calls. -# -# WERF_SECONDARY_REPO_1 is the reason the DEV login is needed at build time: -# prod werf builds reuse already-built layers from the DEV registry as a -# read-only secondary repo. This mirrors the GitHub build action input -# `secondary_repo: "${vars.DEV_MODULE_SOURCE}/${vars.MODULE_NAME}"`, which the -# action exports as WERF_SECONDARY_REPO_1 (see modules-actions/build/action.yml). -# It is set ONLY here, so it only applies to prod release build/deploy jobs that -# extend .dual_registry_login — dev builds never use a secondary repo. (Deploy -# jobs that extend this template ignore it: they crane-copy, not werf build.) -# -# Usage: -# extends: -# - .prod_vars # sets MODULES_* = PROD_* -# - .dual_registry_login # adds DEV_MODULES_* = DEV_* + WERF_SECONDARY_REPO_1 -# - .build # upstream werf build + bundle/release-channel copy -# -# TODO: confirm with a virt-test pipeline that the second login is actually -# required for the build_prod / deploy_to_prod_* chain (an open question, -# because the previous GH workflow did both logins via two consecutive -# `modules-actions/setup` steps). - -.dual_registry_login: - variables: - DEV_MODULES_REGISTRY: "${DEV_REGISTRY}" - DEV_MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" - DEV_MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" - WERF_SECONDARY_REPO_1: "${DEV_MODULE_SOURCE}/${MODULE_NAME}" diff --git a/.gitlab/ci/templates/base/registry-login.yml b/.gitlab/ci/templates/base/registry-login.yml new file mode 100644 index 0000000000..6e757c2766 --- /dev/null +++ b/.gitlab/ci/templates/base/registry-login.yml @@ -0,0 +1,57 @@ +# Registry authentication helpers — all `werf cr login` wiring in one place. +# +# The actual login happens in the upstream Setup.gitlab-ci.yml before_script, +# which runs for every job: +# 1. ALWAYS logs into $MODULES_REGISTRY using +# $MODULES_REGISTRY_LOGIN / $MODULES_REGISTRY_PASSWORD — the primary +# registry the job builds/pushes/pulls against. +# 2. ADDITIONALLY logs into $DEV_MODULES_REGISTRY when its credentials are +# set — a second registry used only to pull already-built layers. +# +# These templates just populate those variables, so `extends`-ing them selects +# which registries a job authenticates to. Pick the primary with exactly one of +# .login_dev_registry / .login_prod_registry / .login_prod_read_registry, and +# add .also_login_dev_registry when a prod job must also read the DEV registry. + +# Primary = DEV registry (write). Used by all dev builds +# (MR / main / release-* / dev tags); pulled in via .dev_vars. +.login_dev_registry: + variables: + MODULES_REGISTRY: "${DEV_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" + +# Primary = PROD registry (write). Used by prod release builds/deploys; +# pulled in via .prod_vars. +.login_prod_registry: + variables: + MODULES_REGISTRY: "${PROD_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${PROD_MODULES_REGISTRY_LOGIN}" + MODULES_REGISTRY_PASSWORD: "${PROD_MODULES_REGISTRY_PASSWORD}" + +# Primary = PROD read-only registry. Used by the version/requirements check +# jobs (prod:check-requirements, prod:check-version). NOTE: the Go/bash tools +# hardcode registry.deckhouse.io, so PROD_READ_REGISTRY must resolve to that +# host for crane auth to apply (see tools/moduleversions Taskfile.dist.yaml). +.login_prod_read_registry: + variables: + MODULES_REGISTRY: "${PROD_READ_REGISTRY}" + MODULES_REGISTRY_LOGIN: "${PROD_READ_REGISTRY_USER}" + MODULES_REGISTRY_PASSWORD: "${PROD_READ_REGISTRY_PASSWORD}" + +# Additional DEV login for prod jobs that must ALSO read the DEV registry. +# Compose with the prod primary, e.g.: +# extends: [.prod_vars, .also_login_dev_registry, .base_build] +# Prod werf builds reuse already-built layers from the DEV registry as a +# read-only secondary repo (WERF_SECONDARY_REPO_1). This mirrors the GitHub +# build action input +# `secondary_repo: "${vars.DEV_MODULE_SOURCE}/${vars.MODULE_NAME}"`, which the +# action exports as WERF_SECONDARY_REPO_1 (see modules-actions/build/action.yml). +# Dev builds never need this (their primary already IS the DEV registry); deploy +# jobs that compose it ignore the secondary repo (they crane-copy, not build). +.also_login_dev_registry: + variables: + DEV_MODULES_REGISTRY: "${DEV_REGISTRY}" + DEV_MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" + DEV_MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" + WERF_SECONDARY_REPO_1: "${DEV_MODULE_SOURCE}/${MODULE_NAME}" diff --git a/.gitlab/ci/templates/dev/dev_vars.yml b/.gitlab/ci/templates/dev/dev_vars.yml index 2ba08f5b56..4894eeba69 100644 --- a/.gitlab/ci/templates/dev/dev_vars.yml +++ b/.gitlab/ci/templates/dev/dev_vars.yml @@ -1,17 +1,12 @@ # DEV registry variable bundle. # -# Carries forward the previous root .gitlab-ci.yml `.dev_vars` template. -# Anything that uses `extends: .dev_vars` gets MODULES_REGISTRY, -# MODULES_REGISTRY_LOGIN, MODULES_REGISTRY_PASSWORD, MODULES_MODULE_SOURCE -# and ENV=DEV, all sourced from the DEV_* Project Variables. -# -# The legacy EXTERNAL_MODULES_DEV_REGISTRY_* names are still accepted as -# fallback to ease the variable-rename migration. +# Anything that uses `extends: .dev_vars` authenticates against the DEV +# registry (via .login_dev_registry) and gets MODULES_MODULE_SOURCE and +# ENV=DEV, all sourced from the DEV_* Project Variables. .dev_vars: + extends: + - .login_dev_registry variables: - MODULES_REGISTRY: "${DEV_REGISTRY}" - MODULES_REGISTRY_LOGIN: "${DEV_MODULES_REGISTRY_LOGIN}" - MODULES_REGISTRY_PASSWORD: "${DEV_MODULES_REGISTRY_PASSWORD}" MODULES_MODULE_SOURCE: "${DEV_MODULE_SOURCE}" ENV: DEV diff --git a/.gitlab/ci/templates/release/prod_always.yml b/.gitlab/ci/templates/release/prod_always.yml deleted file mode 100644 index cc46af3890..0000000000 --- a/.gitlab/ci/templates/release/prod_always.yml +++ /dev/null @@ -1,17 +0,0 @@ -# PROD pipeline context: automatic prod build on tag push. -# -# Carries forward the previous root .gitlab-ci.yml `.prod_always` template. -# Runs on every vX.Y.Z tag push, with `when: always` so that the build -# fires automatically once the tag is in place. The deploy step (in -# deploy-prod.yml) is still `when: manual` via .prod_manual. - -.prod_always: - variables: - MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} - extends: - - .prod_vars - rules: - # https://regex101.com/r/lToOvi/1 - - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' - when: always - - when: never diff --git a/.gitlab/ci/templates/release/prod_manual.yml b/.gitlab/ci/templates/release/prod_manual.yml deleted file mode 100644 index 55621be077..0000000000 --- a/.gitlab/ci/templates/release/prod_manual.yml +++ /dev/null @@ -1,30 +0,0 @@ -# PROD pipeline context: manual prod release deploy. -# -# Carries forward the previous root .gitlab-ci.yml `.prod_manual` template. -# Runs on tags matching vX.Y.Z (no -dev suffix), with `when: manual` so -# each release channel deploy requires a human click. -# -# TODO: the GH release_module_release-channels.yml workflow exposes inputs -# (channel, ce/ee, tag, enableBuild, release_to_github, check_only, ...) that -# we currently collapse into a hardcoded matrix (RELEASE_CHANNEL x EDITION) in -# deploy-prod.yml. Once we decide whether to expose that flexibility (likely -# via "Run pipeline" variables), this template will gain extra variables. -# -# The single-channel dispatch with Run-pipeline inputs (channel, editions, -# tag, enableBuild, check_only, release_to_gitlab, send_results_to_loop, -# requirements/version checks) is now implemented in -# .gitlab/ci/jobs/release-channels.yml (prod:* jobs). This template stays -# as the tag-push deploy context: each channel deploy is an independent -# `when: manual` job in the single `deploy_prod` stage (no forced -# alpha->...->rock-solid promotion chain). - -.prod_manual: - variables: - MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} - extends: - - .prod_vars - rules: - # https://regex101.com/r/lToOvi/1 - - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' - when: manual - - when: never diff --git a/.gitlab/ci/templates/release/prod_tag.yml b/.gitlab/ci/templates/release/prod_tag.yml new file mode 100644 index 0000000000..75882f407c --- /dev/null +++ b/.gitlab/ci/templates/release/prod_tag.yml @@ -0,0 +1,39 @@ +# PROD pipeline context: vX.Y.Z tag-push build & deploy. +# +# .prod_always (build) and .prod_manual (deploy) are identical except for the +# rule's `when:` — build fires automatically once the tag is pushed, each +# channel deploy waits for a human click. They share the common base .prod_tag +# (PROD registry vars + the tag as module tag). +# +# TODO: the GH release_module_release-channels.yml workflow exposes inputs +# (channel, ce/ee, tag, enableBuild, release_to_github, check_only, ...) that +# the tag-push flow collapses into a hardcoded matrix (RELEASE_CHANNEL x +# EDITION) in deploy-prod.yml. The single-channel dispatch with those inputs +# lives in release-channels.yml (prod:* jobs, manual Run pipeline); this +# template stays the tag-push context. + +.prod_tag: + extends: + - .prod_vars + variables: + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + +# Build context: auto on a vX.Y.Z tag push. +.prod_always: + extends: + - .prod_tag + rules: + # https://regex101.com/r/lToOvi/1 + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' + when: always + - when: never + +# Deploy context: same tag match, but each channel deploy is manual. +.prod_manual: + extends: + - .prod_tag + rules: + # https://regex101.com/r/lToOvi/1 + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' + when: manual + - when: never diff --git a/.gitlab/ci/templates/release/prod_vars.yml b/.gitlab/ci/templates/release/prod_vars.yml index abe58a4bf5..99ff573ed8 100644 --- a/.gitlab/ci/templates/release/prod_vars.yml +++ b/.gitlab/ci/templates/release/prod_vars.yml @@ -1,9 +1,8 @@ # PROD registry variable bundle. # -# Carries forward the previous root .gitlab-ci.yml `.prod_vars` template. -# Anything that uses `extends: .prod_vars` gets MODULES_REGISTRY, -# MODULES_REGISTRY_LOGIN, MODULES_REGISTRY_PASSWORD, MODULES_MODULE_SOURCE -# and ENV=PROD, all sourced from the PROD_* Project Variables. +# Anything that uses `extends: .prod_vars` authenticates against the PROD +# registry (via .login_prod_registry) and gets MODULES_MODULE_SOURCE and +# ENV=PROD, all sourced from the PROD_* Project Variables. # # MODULES_MODULE_SOURCE is composed dynamically from PROD_REGISTRY, # PROD_MODULE_SOURCE_NAME, and the matrix-driven EDITION variable so that @@ -15,25 +14,13 @@ # ee/se-plus/fe), matching .github/workflows/release_module_build-and-registration.yml. # .werf/consts.yaml defaults MODULE_EDITION to EE, so it MUST be set by the # matrix — otherwise the ce prod image is built/deployed as EE. - -# Read-only registry bundle for prod verification jobs -# (prod:check-requirements, prod:check-version in release-channels.yml). # -# Sets MODULES_REGISTRY_* to PROD_READ_REGISTRY_* so the upstream Setup -# `werf cr login` authenticates against the read-only registry used by the -# tools/moduleversions checks. NOTE: the Go/bash tools hardcode -# registry.deckhouse.io; PROD_READ_REGISTRY must resolve to that host for -# crane auth to apply (see tools/moduleversions Taskfile.dist.yaml). -.prod_read_registry: - variables: - MODULES_REGISTRY: "${PROD_READ_REGISTRY}" - MODULES_REGISTRY_LOGIN: "${PROD_READ_REGISTRY_USER}" - MODULES_REGISTRY_PASSWORD: "${PROD_READ_REGISTRY_PASSWORD}" +# The read-only registry login used by the prod verification jobs lives in +# base/registry-login.yml as .login_prod_read_registry. .prod_vars: + extends: + - .login_prod_registry variables: - MODULES_REGISTRY: "${PROD_REGISTRY}" - MODULES_REGISTRY_LOGIN: "${PROD_MODULES_REGISTRY_LOGIN}" - MODULES_REGISTRY_PASSWORD: "${PROD_MODULES_REGISTRY_PASSWORD}" MODULES_MODULE_SOURCE: "${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}/${EDITION}/modules" ENV: PROD From 157e8c198b913766e02dba0db8d2ae93a05d2d93 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 19:52:27 +0300 Subject: [PATCH 56/60] chore(ci): drop dead backport sweep mode and stale TODO comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - backport.yml: remove the scheduled "backport sweep" rule (Mode 3). It was beyond the GitHub workflow / migration scope and non-functional anyway (a `when: manual` job on a schedule pipeline is never triggered). Manual TARGET_BRANCH (Mode 1) and the status/backport label (Mode 2) remain the working backport paths. Drop its row from the SCHEDULE_TYPE table too. - deploy-prod.yml: remove the "inputs not yet ported" TODO — it contradicted the very next paragraph; those inputs are implemented in the web flow in release-channels.yml. - prod_tag.yml: demote the same "TODO" to a plain note (the design decision is made: tag-push uses the matrix, the input dispatch lives in release-channels). Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/ci/jobs/backport.yml | 6 ------ .gitlab/ci/jobs/deploy-prod.yml | 5 ----- .gitlab/ci/templates/release/prod_tag.yml | 12 ++++++------ .gitlab/ci/variables.yml | 1 - 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index cf864eebd6..f6fb4f767e 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -61,9 +61,3 @@ backport: - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,\s*)status\/backport(,|$)/ when: manual allow_failure: true - # Mode 3: scheduled backport sweep (TODO: future automation), under the - # dedicated backport-sweep pipeline schedule - # ($SCHEDULE_TYPE == "backport-sweep"). - - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "backport-sweep" - when: manual - allow_failure: true diff --git a/.gitlab/ci/jobs/deploy-prod.yml b/.gitlab/ci/jobs/deploy-prod.yml index 692fd30a6d..f77d36880d 100644 --- a/.gitlab/ci/jobs/deploy-prod.yml +++ b/.gitlab/ci/jobs/deploy-prod.yml @@ -27,11 +27,6 @@ # with `when: manual`). # - .also_login_dev_registry (this repo — also login against DEV_REGISTRY). # -# TODO: the GH release_module_release-channels workflow exposes inputs -# (channel, ce/ee, tag, enableBuild, release_to_github, check_only, -# send_results_to_loop, requirements/version checks, github release -# creation, Loop notification) that are not yet ported. -# # This file is the tag-push chained-deploy flow. The GitHub # release_module_release-channels parity flow (single-channel dispatch with # requirements/version checks, release creation, Loop notify) now lives in diff --git a/.gitlab/ci/templates/release/prod_tag.yml b/.gitlab/ci/templates/release/prod_tag.yml index 75882f407c..0ba308aab4 100644 --- a/.gitlab/ci/templates/release/prod_tag.yml +++ b/.gitlab/ci/templates/release/prod_tag.yml @@ -5,12 +5,12 @@ # channel deploy waits for a human click. They share the common base .prod_tag # (PROD registry vars + the tag as module tag). # -# TODO: the GH release_module_release-channels.yml workflow exposes inputs -# (channel, ce/ee, tag, enableBuild, release_to_github, check_only, ...) that -# the tag-push flow collapses into a hardcoded matrix (RELEASE_CHANNEL x -# EDITION) in deploy-prod.yml. The single-channel dispatch with those inputs -# lives in release-channels.yml (prod:* jobs, manual Run pipeline); this -# template stays the tag-push context. +# Note: the GH release_module_release-channels.yml inputs (channel, ce/ee, +# tag, enableBuild, release_to_github, check_only, ...) are collapsed here into +# a hardcoded matrix (RELEASE_CHANNEL x EDITION) in deploy-prod.yml. The +# single-channel dispatch that exposes those inputs lives in +# release-channels.yml (prod:* jobs, manual Run pipeline); this template stays +# the tag-push context. .prod_tag: extends: diff --git a/.gitlab/ci/variables.yml b/.gitlab/ci/variables.yml index 341af5cb82..31e30b93a3 100644 --- a/.gitlab/ci/variables.yml +++ b/.gitlab/ci/variables.yml @@ -110,7 +110,6 @@ variables: # changelog:all-active-milestones changelog-all-active-milestones nightly # mrs:summary mrs-summary 0 7 * * * # lint:no-cyrillic / lint:go / lint:gitlab-ci lint-validate as needed - # backport (scheduled sweep) backport-sweep as needed # # The global workflow:rules in .gitlab/ci/workflow.yml still create a # pipeline for any schedule; SCHEDULE_TYPE only narrows which jobs run From e94a0ea657515faf9d8f0490d96416666451e273 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 19:52:38 +0300 Subject: [PATCH 57/60] test(ci): commit package-lock.json and use npm ci for JS tests Pin the mrs_notifier JS test dependencies with a committed package-lock.json and switch the test:scripts:js job from `npm install` to `npm ci` for reproducible, lockfile-verified installs. package-lock.json is ignored globally at the repo root, so a scoped .gitlab/scripts/js/.gitignore re-includes this one. Verified locally: `npm ci` + `npm test` pass (29 tests). Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/ci/jobs/test-scripts-js.yml | 16 +- .gitlab/scripts/js/.gitignore | 4 + .gitlab/scripts/js/package-lock.json | 357 +++++++++++++++++++++++++++ 3 files changed, 369 insertions(+), 8 deletions(-) create mode 100644 .gitlab/scripts/js/.gitignore create mode 100644 .gitlab/scripts/js/package-lock.json diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml index 98185f7f66..254f63a6dc 100644 --- a/.gitlab/ci/jobs/test-scripts-js.yml +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -28,10 +28,10 @@ # official `node:${NODE_VERSION}` image via `docker run`, mirroring how # check:gens-files runs the vm-route-forge leg in cilium/ebpf-builder. The # repo is bind-mounted READ-ONLY at /src and copied to an ephemeral tmpdir -# inside the container, so node_modules/package-lock.json never pollute the -# shared shell-executor workspace (root-owned files there break `git clean` -# on subsequent checkouts of unrelated jobs). -# TODO: commit a package-lock.json and switch to `npm ci`. +# inside the container, so node_modules never pollutes the shared +# shell-executor workspace (root-owned files there break `git clean` on +# subsequent checkouts of unrelated jobs). The committed package-lock.json +# lets the job use `npm ci` for reproducible installs. test:scripts:js: stage: test @@ -42,8 +42,8 @@ test:scripts:js: - | set -e # Run npm in a throwaway tmpdir INSIDE the container so node_modules - # and package-lock.json never land in the shared shell-executor - # workspace. Previously npm ran directly against a read-write + # never lands in the shared shell-executor workspace. Previously npm + # ran directly against a read-write # bind-mount of the repo: under the default container uid (root) it # created root-owned node_modules/ that the runner-user could not # remove, which broke `git clean` on the next job's checkout and @@ -51,7 +51,7 @@ test:scripts:js: # "warning: failed to remove ...: Permission denied". # # The repo is mounted read-only at /src; the test sources are copied - # to an ephemeral tmpdir where npm install/npm test execute. + # to an ephemeral tmpdir where npm ci/npm test execute. docker run --rm --platform linux/amd64 \ -v "${PWD}:/src:ro" \ -u "$(id -u):$(id -g)" \ @@ -62,7 +62,7 @@ test:scripts:js: work="$(mktemp -d)" cp -a /src/.gitlab/scripts/js/. "$work"/ cd "$work" - npm install --no-audit --no-fund --cache /tmp/.npm + npm ci --no-audit --no-fund --cache /tmp/.npm npm test ' extends: diff --git a/.gitlab/scripts/js/.gitignore b/.gitlab/scripts/js/.gitignore new file mode 100644 index 0000000000..9111b9b12c --- /dev/null +++ b/.gitlab/scripts/js/.gitignore @@ -0,0 +1,4 @@ +# package-lock.json is ignored globally at the repo root (.gitignore), but the +# one here is intentionally committed so the test:scripts:js CI job can install +# with `npm ci` (reproducible, lockfile-pinned). +!package-lock.json diff --git a/.gitlab/scripts/js/package-lock.json b/.gitlab/scripts/js/package-lock.json new file mode 100644 index 0000000000..f3e1fd430d --- /dev/null +++ b/.gitlab/scripts/js/package-lock.json @@ -0,0 +1,357 @@ +{ + "name": "mrs-notifier-gitlab", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mrs-notifier-gitlab", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "axios": "^1.7.7", + "moment": "^2.30.1" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz", + "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.5", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + } + } +} From 163d7013d1e0cb3757bdbc19ec2712d6be29c661 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 20:07:47 +0300 Subject: [PATCH 58/60] refactor(ci): make interruptible:true the default, override false on release/mutating jobs Centralize the cancel-in-progress behavior (GitHub concurrency equivalent): - defaults.yml now sets `interruptible: true` for every job, so a new commit cancels the redundant in-flight pipeline once the project enables "Auto-cancel redundant pipelines". Drop the resolved TODO in workflow.yml. - Remove the now-redundant per-job `interruptible: true` lines. - Override `interruptible: false` where a run must not be cancelled mid-flight: prod release build/deploy (via .prod_vars) and the whole web release flow (via .prod_release_rules), plus state-mutating/long jobs: cleanup, changelog (both), backport, translate-changelog (already: precache, cve daily/manual, gitleaks full, svace build/analyze). build_prod drops its explicit false (inherited from .prod_vars). Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/ci/defaults.yml | 13 +++++++++---- .gitlab/ci/jobs/backport.yml | 2 ++ .gitlab/ci/jobs/build-dev.yml | 11 +++++------ .gitlab/ci/jobs/build-prod.yml | 2 +- .gitlab/ci/jobs/changelog.yml | 4 ++++ .gitlab/ci/jobs/cleanup.yml | 2 ++ .gitlab/ci/jobs/cve-scan.yml | 1 - .gitlab/ci/jobs/gitleaks.yml | 1 - .gitlab/ci/jobs/info.yml | 1 - .gitlab/ci/jobs/lint-dmt.yml | 1 - .gitlab/ci/jobs/lint-validate.yml | 14 +++----------- .gitlab/ci/jobs/release-channels.yml | 5 ++++- .gitlab/ci/jobs/svace.yml | 2 -- .gitlab/ci/jobs/test-d8v-cli.yml | 1 - .gitlab/ci/jobs/test-scripts-js.yml | 1 - .gitlab/ci/jobs/test-scripts-python.yml | 1 - .gitlab/ci/jobs/translate-changelog.yml | 2 ++ .gitlab/ci/templates/release/prod_vars.yml | 3 +++ .gitlab/ci/workflow.yml | 7 +++---- 19 files changed, 38 insertions(+), 36 deletions(-) diff --git a/.gitlab/ci/defaults.yml b/.gitlab/ci/defaults.yml index 308068516e..6a4f5e8fdf 100644 --- a/.gitlab/ci/defaults.yml +++ b/.gitlab/ci/defaults.yml @@ -5,11 +5,16 @@ # resources are not pinned to a specific tag set (we leave GitLab's # scheduling to runner tags only). # -# Per-job `interruptible` is intentionally NOT set here; long-running jobs -# (build_prod and the deploy chain) need `interruptible: false`, while -# short-lived lint/test/build_dev jobs benefit from `interruptible: true`. -# Each job file opts in explicitly. +# `interruptible: true` is the default so that pushing a new commit cancels +# the now-redundant in-flight pipeline (the GitLab equivalent of GitHub's +# concurrency cancel-in-progress; takes effect once "Auto-cancel redundant +# pipelines" is enabled at the project level). Jobs that must NOT be cancelled +# — prod release build/deploy/release and state-mutating jobs (registry +# cleanup, changelog/backport/translate MR creation, long scheduled scans) — +# override with `interruptible: false`, mostly via the .prod_vars / +# .prod_release_rules templates. default: tags: - deckhouse + interruptible: true diff --git a/.gitlab/ci/jobs/backport.yml b/.gitlab/ci/jobs/backport.yml index f6fb4f767e..b222177c0d 100644 --- a/.gitlab/ci/jobs/backport.yml +++ b/.gitlab/ci/jobs/backport.yml @@ -38,6 +38,8 @@ backport: stage: lint + # Mutates state (cherry-pick + opens an MR); never auto-cancel mid-run. + interruptible: false tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/build-dev.yml b/.gitlab/ci/jobs/build-dev.yml index d89d2e8678..ed13eed338 100644 --- a/.gitlab/ci/jobs/build-dev.yml +++ b/.gitlab/ci/jobs/build-dev.yml @@ -19,9 +19,9 @@ # an explicit --repo flag, which is equivalent because MODULES_MODULE_SOURCE # is the same in both cases. # -# build_main keeps `interruptible: true` so a new main push cancels an -# older main pipeline. build_dev and build_dev_tags inherit the project -# default (interruptible not set, so they can be cancelled manually). +# All dev build jobs inherit the global `interruptible: true` default +# (.gitlab/ci/defaults.yml), so a new push to the same MR/branch cancels the +# older in-flight build. # `needs: [set_vars]` (with artifacts) is intentional: it opts these jobs # into DAG mode and pulls ONLY the set_vars dotenv (MODULE_EDITION, @@ -59,7 +59,6 @@ build_dev_tags: build_main: stage: build - interruptible: true needs: - job: set_vars artifacts: true @@ -73,10 +72,10 @@ build_main: # squash-merge into a release-X.Y branch rebuilds the dev image tagged with the # branch name. Without this job GitLab created a pipeline on the merge push but # built nothing for release branches (only .dev/.main/.dev_tags had build jobs). -# interruptible: true so a newer release-branch push cancels an older build. +# Inherits the default interruptible: true, so a newer release-branch push +# cancels an older build. build_release: stage: build - interruptible: true needs: - job: set_vars artifacts: true diff --git a/.gitlab/ci/jobs/build-prod.yml b/.gitlab/ci/jobs/build-prod.yml index 6d30340d12..8292fc1aaf 100644 --- a/.gitlab/ci/jobs/build-prod.yml +++ b/.gitlab/ci/jobs/build-prod.yml @@ -32,7 +32,7 @@ build_prod: stage: build resource_group: prod - interruptible: false + # interruptible: false is inherited from .prod_vars (via .prod_always). extends: - .base_build - .prod_always diff --git a/.gitlab/ci/jobs/changelog.yml b/.gitlab/ci/jobs/changelog.yml index ce49ed2723..6b2e9e54c6 100644 --- a/.gitlab/ci/jobs/changelog.yml +++ b/.gitlab/ci/jobs/changelog.yml @@ -35,6 +35,8 @@ changelog:milestone: stage: lint + # Mutates state (commits + opens an MR); never auto-cancel mid-run. + interruptible: false tags: - deckhouse before_script: @@ -71,6 +73,8 @@ changelog:milestone: # MRs are opened. changelog:all-active-milestones: stage: lint + # Mutates state (commits + opens an MR); never auto-cancel mid-run. + interruptible: false tags: - deckhouse before_script: diff --git a/.gitlab/ci/jobs/cleanup.yml b/.gitlab/ci/jobs/cleanup.yml index 278becb7e4..707465e414 100644 --- a/.gitlab/ci/jobs/cleanup.yml +++ b/.gitlab/ci/jobs/cleanup.yml @@ -30,6 +30,8 @@ cleanup: stage: cleanup + # Mutates the registry (werf cleanup); never auto-cancel mid-run. + interruptible: false extends: - .dev_vars variables: diff --git a/.gitlab/ci/jobs/cve-scan.yml b/.gitlab/ci/jobs/cve-scan.yml index 1b8e733beb..83fa849035 100644 --- a/.gitlab/ci/jobs/cve-scan.yml +++ b/.gitlab/ci/jobs/cve-scan.yml @@ -103,7 +103,6 @@ cve:scan:mr: extends: - .cve_scan stage: scan - interruptible: true needs: - build_dev variables: diff --git a/.gitlab/ci/jobs/gitleaks.yml b/.gitlab/ci/jobs/gitleaks.yml index 821426bb7f..321b2edc9d 100644 --- a/.gitlab/ci/jobs/gitleaks.yml +++ b/.gitlab/ci/jobs/gitleaks.yml @@ -49,7 +49,6 @@ gitleaks_full_scheduled: gitleaks:diff: extends: .gitleaks_scan stage: scan - interruptible: true variables: SCAN_MODE: "diff" GIT_DEPTH: "0" diff --git a/.gitlab/ci/jobs/info.yml b/.gitlab/ci/jobs/info.yml index 6e4e6bbaac..98fa9cc4c0 100644 --- a/.gitlab/ci/jobs/info.yml +++ b/.gitlab/ci/jobs/info.yml @@ -43,7 +43,6 @@ show_main_manifest: set_vars: stage: info - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash script: diff --git a/.gitlab/ci/jobs/lint-dmt.yml b/.gitlab/ci/jobs/lint-dmt.yml index b5bf623b8b..76433a0991 100644 --- a/.gitlab/ci/jobs/lint-dmt.yml +++ b/.gitlab/ci/jobs/lint-dmt.yml @@ -35,7 +35,6 @@ lint:dmt: stage: lint - interruptible: true allow_failure: true script: - dmt lint ./ diff --git a/.gitlab/ci/jobs/lint-validate.yml b/.gitlab/ci/jobs/lint-validate.yml index 7a42fd96bc..3e789bfcb4 100644 --- a/.gitlab/ci/jobs/lint-validate.yml +++ b/.gitlab/ci/jobs/lint-validate.yml @@ -10,9 +10,9 @@ # `validation/skip/actionlint` is therefore obsolete and not honored. # # Each job: -# - inherits `tags: [deckhouse]` from .gitlab/ci/defaults.yml; -# - has `interruptible: true` where short-lived (safe to cancel on a new -# push) and `interruptible: false` where a long run is expected; +# - inherits `tags: [deckhouse]` and `interruptible: true` from +# .gitlab/ci/defaults.yml (these short-lived checks are safe to cancel on a +# new push); # - honors `validation/skip/<name>` labels via `when: never` rules. # # The job file assumes the upstream template include for @@ -33,7 +33,6 @@ lint:no-cyrillic: stage: lint - interruptible: true variables: GIT_DEPTH: "0" before_script: @@ -55,7 +54,6 @@ lint:no-cyrillic: lint:doc-changes: stage: lint - interruptible: true variables: GIT_DEPTH: "0" before_script: @@ -80,7 +78,6 @@ lint:doc-changes: lint:shellcheck: stage: lint - interruptible: true # The `task lint:shellcheck` target runs shellcheck inside the # koalaman/shellcheck-alpine Docker image, so the runner needs docker — # not a host-installed shellcheck binary. @@ -101,7 +98,6 @@ lint:shellcheck: lint:yaml: stage: lint - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task script: @@ -147,7 +143,6 @@ lint:yaml: lint:go: stage: lint - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go curl - | @@ -211,7 +206,6 @@ lint:go: lint:helm-templates: stage: lint - interruptible: true before_script: # go + task are host tools; docker runs kubeconform; git/curl/python3 are # needed by tools/kubeconform/kubeconform.sh (clone kubeconform.git, fetch @@ -289,7 +283,6 @@ lint:helm-templates: check:gens-files: stage: lint - interruptible: true before_script: # git is needed by check_diffs (git diff). go/task are host tools. # docker is needed by the vm-route-forge matrix leg, which runs bpf2go @@ -402,7 +395,6 @@ check:gens-files: lint:gitlab-ci: stage: lint - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh bash curl jq script: diff --git a/.gitlab/ci/jobs/release-channels.yml b/.gitlab/ci/jobs/release-channels.yml index e52d91bbf9..15adb05316 100644 --- a/.gitlab/ci/jobs/release-channels.yml +++ b/.gitlab/ci/jobs/release-channels.yml @@ -76,8 +76,11 @@ variables: options: ["true", "false"] description: "Send the release result summary to Loop via webhook" -# Common rule: only on manual web pipelines with a valid tag. +# Common rule: only on manual web pipelines with a valid tag. Every prod:* job +# extends this; interruptible:false keeps the whole manual release flow from +# being auto-cancelled (overrides the global interruptible:true default). .prod_release_rules: + interruptible: false rules: - if: $CI_PIPELINE_SOURCE == "web" && $RELEASE_TAG =~ /^v\d+\.\d+\.\d+$/ - when: never diff --git a/.gitlab/ci/jobs/svace.yml b/.gitlab/ci/jobs/svace.yml index 1fdfcd3978..6ea9ed5377 100644 --- a/.gitlab/ci/jobs/svace.yml +++ b/.gitlab/ci/jobs/svace.yml @@ -41,7 +41,6 @@ svace:set-vars: extends: - .info stage: info - interruptible: true # Only emit the svace tag dotenv in contexts where svace actually runs. # Without these rules this job ran on EVERY MR pipeline and leaked a # MODULES_MODULE_TAG=<branch>-svace dotenv artifact into build jobs, @@ -134,7 +133,6 @@ svace:analyze: svace:notify: stage: cleanup - interruptible: true # needs svace:analyze (non-optional) so this job stays coupled to a real # svace run: on an unrelated web pipeline svace:analyze is manual and never # played, so it is skipped and this notify is skipped too. The on_success / diff --git a/.gitlab/ci/jobs/test-d8v-cli.yml b/.gitlab/ci/jobs/test-d8v-cli.yml index 746b1d2f52..832e870e10 100644 --- a/.gitlab/ci/jobs/test-d8v-cli.yml +++ b/.gitlab/ci/jobs/test-d8v-cli.yml @@ -26,7 +26,6 @@ test:build:d8v-cli: stage: test - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh go task script: diff --git a/.gitlab/ci/jobs/test-scripts-js.yml b/.gitlab/ci/jobs/test-scripts-js.yml index 254f63a6dc..db6df14c5d 100644 --- a/.gitlab/ci/jobs/test-scripts-js.yml +++ b/.gitlab/ci/jobs/test-scripts-js.yml @@ -35,7 +35,6 @@ test:scripts:js: stage: test - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh docker script: diff --git a/.gitlab/ci/jobs/test-scripts-python.yml b/.gitlab/ci/jobs/test-scripts-python.yml index 154f8d25ef..eabbf084b8 100644 --- a/.gitlab/ci/jobs/test-scripts-python.yml +++ b/.gitlab/ci/jobs/test-scripts-python.yml @@ -27,7 +27,6 @@ test:scripts:python: stage: test - interruptible: true before_script: - bash .gitlab/ci/scripts/bash/check-runner-tools.sh python3 script: diff --git a/.gitlab/ci/jobs/translate-changelog.yml b/.gitlab/ci/jobs/translate-changelog.yml index 7beb137a0a..f928924739 100644 --- a/.gitlab/ci/jobs/translate-changelog.yml +++ b/.gitlab/ci/jobs/translate-changelog.yml @@ -60,6 +60,8 @@ include: translate:changelog: extends: .translate_and_create_mr stage: notify + # Mutates state (translates + opens an MR); never auto-cancel mid-run. + interruptible: false tags: - deckhouse rules: diff --git a/.gitlab/ci/templates/release/prod_vars.yml b/.gitlab/ci/templates/release/prod_vars.yml index 99ff573ed8..8b59bc4296 100644 --- a/.gitlab/ci/templates/release/prod_vars.yml +++ b/.gitlab/ci/templates/release/prod_vars.yml @@ -21,6 +21,9 @@ .prod_vars: extends: - .login_prod_registry + # Prod build/deploy jobs must never be auto-cancelled mid-release; override + # the global `interruptible: true` default for every job that extends this. + interruptible: false variables: MODULES_MODULE_SOURCE: "${PROD_REGISTRY}/${PROD_MODULE_SOURCE_NAME}/${EDITION}/modules" ENV: PROD diff --git a/.gitlab/ci/workflow.yml b/.gitlab/ci/workflow.yml index 412e9c1798..b8985f0583 100644 --- a/.gitlab/ci/workflow.yml +++ b/.gitlab/ci/workflow.yml @@ -11,10 +11,9 @@ # - Pushes of tags vX.Y.Z-dev* / vX.Y.Z create pipelines. # - Manual/scheduled/web pipelines always run. # -# TODO: once auto-cancel-redundant-pipelines is verified at the project -# level, add `interruptible: true` here as a project default. -# Per-job overrides will set `interruptible: false` for build_prod and the -# prod-deploy chain. +# Jobs default to `interruptible: true` (see .gitlab/ci/defaults.yml) so a +# new commit cancels the redundant in-flight pipeline once "Auto-cancel +# redundant pipelines" is enabled at the project level. workflow: rules: From f23c5a6d7eb03de2fbbd461050a1bf07141eeff1 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 20:53:55 +0300 Subject: [PATCH 59/60] docs(ci): drop upstream pin TODO, document the SHA in a comment The decision is to keep tracking the v13.0 branch (upstream fixes flow in automatically). Replace the "pin to SHA" TODO in includes.yml with a plain note that records the branch HEAD SHA to use if pinning is ever wanted. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/ci/includes.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/includes.yml b/.gitlab/ci/includes.yml index da0a679736..65c15b3e5b 100644 --- a/.gitlab/ci/includes.yml +++ b/.gitlab/ci/includes.yml @@ -29,8 +29,9 @@ include: # --- Upstream modules-gitlab-ci (deckhouse/3p, ref v13.0) --- - # TODO: pin to SHA after first green pipeline on virt-test. Until then, - # branch ref v13.0 keeps fixes flowing. + # Tracks the v13.0 branch so upstream fixes flow in automatically. To pin for + # reproducibility, swap the ref for the branch HEAD SHA + # (006d51c35904b434eca2045a449aafb5e37a8827 at the time of writing). - project: "deckhouse/3p/deckhouse/modules-gitlab-ci" ref: "v13.0" file: From a9328710fe32d4d13b2138c3146768a9e194d184 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 26 Jun 2026 20:57:37 +0300 Subject: [PATCH 60/60] fix(ci): set mrs_notifier DOC_REVIEWER to the GitLab username MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the leftover GitHub username default ("z9r5") with the GitLab handle "gitlab-virt-bot" — the one reviewer kept as a real @-mention in the Loop MR summary. Still overridable via the DOC_REVIEWER CI/CD variable. Removes the last open TODO in the GitLab CI scripts. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .gitlab/scripts/js/mrs_notifier.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/js/mrs_notifier.mjs b/.gitlab/scripts/js/mrs_notifier.mjs index edc84cca55..39ce2637f6 100644 --- a/.gitlab/scripts/js/mrs_notifier.mjs +++ b/.gitlab/scripts/js/mrs_notifier.mjs @@ -24,8 +24,9 @@ // CI_PROJECT_ID (required) numeric project id (use the variable, // not the slug, to survive renames). // LOOP_WEBHOOK_URL (required) Loop incoming webhook URL. -// DOC_REVIEWER (optional) GitLab username of doc reviewer. -// Default "z9r5" (TODO: confirm GitLab username). +// DOC_REVIEWER (optional) GitLab username of the doc reviewer (the one +// reviewer kept as a real @-mention). +// Default "gitlab-virt-bot". // MANAGER_LOOP_NAME (optional) @firstname.lastname of the manager. // Default "@yuriy.milyutin". // @@ -47,7 +48,7 @@ const PROJECT_ID = process.env.CI_PROJECT_ID; const API_BASE = (process.env.CI_API_V4_URL || '').replace(/\/+$/, ''); const TOKEN = process.env.GITLAB_API_TOKEN; const LOOP_URL = process.env.LOOP_WEBHOOK_URL; -const DOC_REVIEWER = process.env.DOC_REVIEWER || 'z9r5'; +const DOC_REVIEWER = process.env.DOC_REVIEWER || 'gitlab-virt-bot'; const MANAGER_LOOP_NAME = process.env.MANAGER_LOOP_NAME || '@yuriy.milyutin'; const PROJECT = ':dvp: DVP';