Skip to content

Commit 4e3f205

Browse files
rajiv-jain-netapprajiv-jain-netapp
committed
CSTACKEX-175: adding initial changes for CICD related workflows
1 parent 3a324c6 commit 4e3f205

22 files changed

Lines changed: 929 additions & 0 deletions

‎private-cicd/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Local overrides with secrets (templates *.example stay tracked)
2+
config/qa.yaml
3+
marvin/zones/*.cfg
4+
!marvin/zones/*.cfg.example

‎private-cicd/Jenkinsfile‎

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
/*
2+
* Private downstream Jenkins pipeline for Apache CloudStack.
3+
* Not for inclusion in apache/cloudstack upstream.
4+
*
5+
* Rollout: Phase -1 = preflight (validate private-cicd only). Phase 1 = build-only. Phases 2–3 = marvin / delivery — see private-cicd/docs/IMPLEMENTATION-PHASES.md
6+
*
7+
* Usage:
8+
* - Monorepo: Script Path = private-cicd/Jenkinsfile ; leave CLONE_SEPARATE = false.
9+
* - CI-only repo: set CLONE_SEPARATE = true and point CLOUDSTACK_* at your fork/tag.
10+
*/
11+
12+
pipeline {
13+
agent any
14+
15+
options {
16+
buildDiscarder(logRotator(numToKeepStr: '30'))
17+
timestamps()
18+
timeout(time: 4, unit: 'HOURS')
19+
}
20+
21+
parameters {
22+
choice(
23+
name: 'PIPELINE_PHASE',
24+
choices: ['build-ontap-fast', 'build-only', 'preflight', 'marvin', 'delivery'],
25+
description: 'build-ontap-fast = ONTAP plugin -pl -am test. preflight = Phase -1. build-only = full Maven. marvin / delivery reserved; see private-cicd/docs/IMPLEMENTATION-PHASES.md'
26+
)
27+
string(
28+
name: 'CONFIG_DEFAULTS_FILE',
29+
defaultValue: 'config/defaults.yaml',
30+
description: 'YAML path relative to CICD_ROOT (e.g. config/defaults.yaml or config/team-b.yaml).'
31+
)
32+
string(
33+
name: 'CONFIG_PROFILE',
34+
defaultValue: '',
35+
description: 'Optional: key under profiles in the defaults YAML (e.g. example-lts-branch). Blank = use only top-level cloudstack: from the file.'
36+
)
37+
booleanParam(
38+
name: 'CLONE_SEPARATE',
39+
defaultValue: false,
40+
description: 'If true, clone CloudStack into cloudstack-src/ (use when this job repo is CI-only). If false, expect CloudStack pom.xml at the workspace root (multibranch on your fork). Branch/url below apply only when this is true.'
41+
)
42+
string(
43+
name: 'CLOUDSTACK_GIT_URL',
44+
defaultValue: '',
45+
description: 'Override Git URL for CloudStack. Leave blank to use config/defaults.yaml (after optional CONFIG_PROFILE).'
46+
)
47+
string(
48+
name: 'CLOUDSTACK_GIT_BRANCH',
49+
defaultValue: '',
50+
description: 'Override branch or tag for clone. Leave blank to use config/defaults.yaml (after optional CONFIG_PROFILE). Ignored when CLONE_SEPARATE is false.'
51+
)
52+
booleanParam(
53+
name: 'ENABLE_NOREDIST',
54+
defaultValue: false,
55+
description: 'If true, run third-party non-OSS install script then mvn -Dnoredist (see README / legal for your org).'
56+
)
57+
booleanParam(
58+
name: 'SKIP_TESTS',
59+
defaultValue: true,
60+
description: 'If true, Maven uses -DskipTests=true (faster CI).'
61+
)
62+
booleanParam(
63+
name: 'INSTALL_APT_DEPS',
64+
defaultValue: true,
65+
description: 'If true, run private-cicd/scripts/install-build-deps-ubuntu.sh (requires sudo on agent).'
66+
)
67+
booleanParam(
68+
name: 'SETUP_IPMITOOL_WRAPPER',
69+
defaultValue: false,
70+
description: 'If true, install CloudStack-style ipmitool wrapper (requires sudo).'
71+
)
72+
}
73+
74+
environment {
75+
MAVEN_OPTS = '-Xmx3072m -XX:MaxMetaspaceSize=512m'
76+
}
77+
78+
stages {
79+
stage('Phase gate') {
80+
steps {
81+
script {
82+
switch (params.PIPELINE_PHASE) {
83+
case 'preflight':
84+
case 'build-ontap-fast':
85+
case 'build-only':
86+
break
87+
case 'marvin':
88+
case 'delivery':
89+
error("PIPELINE_PHASE='${params.PIPELINE_PHASE}' is not implemented yet. Use preflight, build-ontap-fast, or build-only. See private-cicd/docs/IMPLEMENTATION-PHASES.md")
90+
default:
91+
error("PIPELINE_PHASE='${params.PIPELINE_PHASE}' is not supported.")
92+
}
93+
}
94+
}
95+
}
96+
97+
stage('Resolve CloudStack directory') {
98+
steps {
99+
script {
100+
if (fileExists("${env.WORKSPACE}/private-cicd/scripts/install-build-deps-ubuntu.sh")) {
101+
env.CICD_SCRIPT_DIR = "${env.WORKSPACE}/private-cicd/scripts"
102+
env.CICD_ROOT = "${env.WORKSPACE}/private-cicd"
103+
} else if (fileExists("${env.WORKSPACE}/scripts/install-build-deps-ubuntu.sh")) {
104+
env.CICD_SCRIPT_DIR = "${env.WORKSPACE}/scripts"
105+
env.CICD_ROOT = env.WORKSPACE
106+
} else {
107+
error('Cannot find install-build-deps-ubuntu.sh (expected private-cicd/scripts or scripts/).')
108+
}
109+
if (params.CLONE_SEPARATE) {
110+
env.CLOUDSTACK_DIR = "${env.WORKSPACE}/cloudstack-src"
111+
} else {
112+
env.CLOUDSTACK_DIR = env.WORKSPACE
113+
}
114+
115+
def rel = params.CONFIG_DEFAULTS_FILE?.trim()
116+
if (!rel) {
117+
rel = 'config/defaults.yaml'
118+
}
119+
rel = rel.replaceAll('^/+', '')
120+
if (rel.contains('..')) {
121+
error('CONFIG_DEFAULTS_FILE must stay under CICD_ROOT (no .. segments).')
122+
}
123+
def cfgPath = "${env.CICD_ROOT}/${rel}"
124+
echo "Loading CI defaults from: ${cfgPath}"
125+
def baseCfg = [cloudstack: [git_url: 'https://github.com/apache/cloudstack.git', branch: 'main']]
126+
if (fileExists(cfgPath)) {
127+
baseCfg = readYaml(file: cfgPath) ?: baseCfg
128+
} else {
129+
echo "No ${cfgPath}; using built-in CloudStack URL/branch defaults."
130+
}
131+
132+
def cs = new LinkedHashMap((baseCfg.cloudstack ?: [:]) as Map)
133+
def profileName = params.CONFIG_PROFILE?.trim()
134+
if (profileName && baseCfg.profiles instanceof Map && baseCfg.profiles[profileName]) {
135+
def overlay = baseCfg.profiles[profileName].cloudstack
136+
if (overlay instanceof Map) {
137+
cs.putAll(overlay as Map)
138+
}
139+
}
140+
141+
def urlOverride = params.CLOUDSTACK_GIT_URL?.trim()
142+
def branchOverride = params.CLOUDSTACK_GIT_BRANCH?.trim()
143+
env.EFFECTIVE_CLOUDSTACK_GIT_URL = urlOverride ?: (cs.git_url as String ?: 'https://github.com/apache/cloudstack.git')
144+
env.EFFECTIVE_CLOUDSTACK_GIT_BRANCH = branchOverride ?: (cs.branch as String ?: 'main')
145+
146+
echo "CICD_ROOT=${env.CICD_ROOT} CICD_SCRIPT_DIR=${env.CICD_SCRIPT_DIR} CLOUDSTACK_DIR=${env.CLOUDSTACK_DIR}"
147+
echo "CONFIG_PROFILE=${profileName ?: '(none)'} EFFECTIVE_CLOUDSTACK_GIT_URL=${env.EFFECTIVE_CLOUDSTACK_GIT_URL} EFFECTIVE_CLOUDSTACK_GIT_BRANCH=${env.EFFECTIVE_CLOUDSTACK_GIT_BRANCH}"
148+
}
149+
}
150+
}
151+
152+
stage('Phase -1: validate private-cicd') {
153+
when {
154+
expression { return params.PIPELINE_PHASE == 'preflight' }
155+
}
156+
steps {
157+
sh """CICD_ROOT='${env.CICD_ROOT}' bash '${env.CICD_SCRIPT_DIR}/validate-local.sh'"""
158+
}
159+
}
160+
161+
stage('Clone CloudStack') {
162+
when {
163+
allOf {
164+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
165+
expression { return params.CLONE_SEPARATE }
166+
}
167+
}
168+
steps {
169+
sh 'rm -rf cloudstack-src && mkdir cloudstack-src'
170+
dir('cloudstack-src') {
171+
checkout(
172+
[
173+
$class : 'GitSCM',
174+
branches : [[name: "*/${env.EFFECTIVE_CLOUDSTACK_GIT_BRANCH}"]],
175+
doGenerateSubmoduleConfigurations: false,
176+
extensions : [],
177+
submoduleCfg : [],
178+
userRemoteConfigs : [[url: env.EFFECTIVE_CLOUDSTACK_GIT_URL]]
179+
]
180+
)
181+
}
182+
}
183+
}
184+
185+
stage('Validate tree') {
186+
when {
187+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
188+
}
189+
steps {
190+
script {
191+
def pom = "${env.CLOUDSTACK_DIR}/pom.xml"
192+
if (!fileExists(pom)) {
193+
error("Missing ${pom}. Enable CLONE_SEPARATE or check out CloudStack with Script Path private-cicd/Jenkinsfile at repo root.")
194+
}
195+
}
196+
}
197+
}
198+
199+
stage('Install OS build dependencies') {
200+
when {
201+
allOf {
202+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
203+
expression { return params.INSTALL_APT_DEPS }
204+
}
205+
}
206+
steps {
207+
sh """bash '${env.CICD_SCRIPT_DIR}/install-build-deps-ubuntu.sh'"""
208+
}
209+
}
210+
211+
stage('Optional ipmitool wrapper') {
212+
when {
213+
allOf {
214+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
215+
expression { return params.SETUP_IPMITOOL_WRAPPER }
216+
}
217+
}
218+
steps {
219+
sh """bash '${env.CICD_SCRIPT_DIR}/setup-ipmitool-wrapper.sh'"""
220+
}
221+
}
222+
223+
stage('Non-OSS (noredist)') {
224+
when {
225+
allOf {
226+
expression { return params.PIPELINE_PHASE == 'build-only' }
227+
expression { return params.ENABLE_NOREDIST }
228+
}
229+
}
230+
steps {
231+
dir("${env.CLOUDSTACK_DIR}") {
232+
sh '''
233+
set -e
234+
rm -rf nonoss
235+
git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
236+
cd nonoss && bash -x install-non-oss.sh
237+
cd ..
238+
rm -rf nonoss
239+
'''
240+
}
241+
}
242+
}
243+
244+
stage('Phase 1a: ONTAP fast build') {
245+
when {
246+
expression { return params.PIPELINE_PHASE == 'build-ontap-fast' }
247+
}
248+
steps {
249+
dir("${env.CLOUDSTACK_DIR}") {
250+
script {
251+
def skip = params.SKIP_TESTS ? 'true' : 'false'
252+
sh """SKIP_TESTS='${skip}' CLOUDSTACK_DIR='${env.CLOUDSTACK_DIR}' bash '${env.CICD_SCRIPT_DIR}/mvn-ontap-fast.sh'"""
253+
}
254+
}
255+
}
256+
}
257+
258+
stage('Phase 1: Maven build') {
259+
when {
260+
expression { return params.PIPELINE_PHASE == 'build-only' }
261+
}
262+
steps {
263+
dir("${env.CLOUDSTACK_DIR}") {
264+
script {
265+
def skip = params.SKIP_TESTS ? '-DskipTests=true' : ''
266+
def noredist = params.ENABLE_NOREDIST ? '-Dnoredist' : ''
267+
sh "mvn -B -P developer,systemvm -Dsimulator ${noredist} clean install ${skip} -T\$(nproc)"
268+
}
269+
}
270+
}
271+
}
272+
}
273+
274+
post {
275+
always {
276+
script {
277+
if (params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast']) {
278+
dir("${env.CLOUDSTACK_DIR}") {
279+
def junitGlob = '**/target/surefire-reports/*.xml'
280+
if (params.PIPELINE_PHASE == 'build-ontap-fast') {
281+
junitGlob = 'plugins/storage/volume/ontap/target/surefire-reports/*.xml'
282+
}
283+
junit testResults: junitGlob, allowEmptyResults: true
284+
}
285+
}
286+
}
287+
}
288+
failure {
289+
echo 'Pipeline failed — see console output.'
290+
}
291+
}
292+
}

‎private-cicd/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Private CI/CD (downstream only)
2+
3+
This directory is **not** part of Apache CloudStack upstream. Do **not** include it in pull requests to `apache/cloudstack`.
4+
5+
## Option A (default): committed on the NetApp fork
6+
7+
`private-cicd/` is **tracked on integration branches** (e.g. `dev_branch`, `netapp/main`). Jenkins uses the same checkout as CloudStack.
8+
9+
- **Script Path:** `private-cicd/Jenkinsfile`
10+
- **Clone CloudStack separately:** leave unchecked (`CLONE_SEPARATE = false`)
11+
- **Upstream PRs:** use branches without `private-cicd/` commits — see [`docs/BRANCH-STRATEGY.md`](docs/BRANCH-STRATEGY.md)
12+
- **Layout:** [`docs/FOLDER-LAYOUT.md`](docs/FOLDER-LAYOUT.md)
13+
14+
## Rollout phases
15+
16+
| Phase | `PIPELINE_PHASE` | Status |
17+
|-------|------------------|--------|
18+
| **-1 — Preflight** | `preflight` | Implemented |
19+
| **1a — ONTAP fast** | `build-ontap-fast` | Implemented (`scripts/mvn-ontap-fast.sh`) |
20+
| **1 — Full build** | `build-only` | Implemented |
21+
| **2 — Marvin** | `marvin` | Stub — see `config/marvin.yaml`, `scripts/marvin-run.sh` |
22+
| **3 — CD** | `delivery` | Planned |
23+
24+
## Quick start
25+
26+
```bash
27+
# Validate CI tree only (no Maven)
28+
./private-cicd/scripts/validate-local.sh
29+
30+
# ONTAP plugin compile + JUnit (from repo root; only ontap *Test.java, not whole tree)
31+
CLOUDSTACK_DIR=$PWD SKIP_TESTS=false ./private-cicd/scripts/mvn-ontap-fast.sh
32+
33+
# Full build (long)
34+
CLOUDSTACK_DIR=$PWD SKIP_TESTS=true ./private-cicd/scripts/mvn-full.sh
35+
```
36+
37+
Do not use `mvn -pl :cloud-plugin-storage-volume-ontap -am test` alone — `-am test` runs upstream module tests (e.g. `engine/schema`), which may call `sudo mount` and block on `Password:`.
38+
39+
## Configuration
40+
41+
| File | Purpose |
42+
|------|---------|
43+
| [`config/defaults.yaml`](config/defaults.yaml) | Git URL, branch, profiles |
44+
| [`config/build-fast.yaml`](config/build-fast.yaml) | ONTAP `-pl -am` settings |
45+
| [`config/marvin.yaml`](config/marvin.yaml) | Marvin phase (Phase 2) |
46+
| [`config/qa.yaml.example`](config/qa.yaml.example) | Copy to `qa.yaml` for local overrides (gitignored) |
47+
48+
## Marvin tests
49+
50+
- Product tests: `test/integration/plugins/ontap/`
51+
- CI bundles: [`marvin/bundles.txt`](marvin/bundles.txt)
52+
- Zone templates: [`marvin/zones/`](marvin/zones/)
53+
54+
## Alternative: separate CI repository
55+
56+
Copy this tree to a dedicated repo and set `CLONE_SEPARATE = true` in Jenkins. See historical note in git history if you migrate from Option A.
57+
58+
## License
59+
60+
Scripts and the Jenkinsfile are for internal use only; not submitted to the ASF as part of CloudStack.

‎private-cicd/config/build-fast.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Fast build profile: ONTAP storage plugin only (Maven reactor -pl -am).
2+
# Used by PIPELINE_PHASE=build-ontap-fast and scripts/mvn-ontap-fast.sh.
3+
4+
ontap:
5+
maven_artifact: cloud-plugin-storage-volume-ontap
6+
maven_pl: ":cloud-plugin-storage-volume-ontap"
7+
plugin_path: plugins/storage/volume/ontap
8+
9+
maven:
10+
profiles: developer
11+
skip_tests: false
12+
extra_args: ""
13+
# Two-step: (-am -DskipTests install) then (-pl ontap test). Never use "-am test"
14+
# or upstream modules (e.g. engine/schema) run sudo-mount tests and hang on Password:
15+
steps:
16+
- install_deps_skip_tests
17+
- test_ontap_only
18+
19+
# Paths that trigger a full build when changed (git diff); optional in Jenkins later.
20+
change_detection:
21+
full_build_paths:
22+
- client/
23+
- engine/
24+
- api/
25+
- framework/
26+
- plugins/storage/volume/default/
27+
- pom.xml

0 commit comments

Comments
 (0)