Skip to content

Commit b8047b8

Browse files
committed
SRE-2832 Improve portability
Update the build procedure to be more portable to other Jenkins environments. .dockerignore: New file Jenkinsfile: Minor fixes packaging.Dockerfile*: Support building behind a proxy. packaging.rpm_chrootbuild: Support building behind a proxy. packaging/scripts: Move code from Dockerfiles for easier maintenance. Signed-off-by: John E. Malmberg <john.malmberg@hpe.com>
1 parent ebb74da commit b8047b8

File tree

8 files changed

+264
-44
lines changed

8 files changed

+264
-44
lines changed

.dockerignore

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# .dockerignore file for DAOS.
2+
3+
# Firstly deny everything and then allow only directories and files that we're
4+
# interested in. Other files will not be required for the build and they
5+
# just generate noise and extra work for docker.
6+
*
7+
!packaging/scripts

Jenkinsfile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/usr/bin/groovy
2-
/* Copyright (C) 2019 Intel Corporation
2+
/* Copyright (C) 2019-2024 Intel Corporation
3+
* Copyright (C) 2025 Hewlett Packard Enterprise Development LP
34
* All rights reserved.
45
*
56
* Redistribution and use in source and binary forms, with or without
@@ -39,5 +40,6 @@
3940
// To use a test branch (i.e. PR) until it lands to master
4041
// I.e. for testing library changes
4142
//@Library(value="pipeline-lib@your_branch") _
43+
@Library(value='pipeline-lib@malmberg/sre-2832') _
4244
packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'],
43-
'test-tag': 'dfuse'])
45+
'test-tag': 'dfuse'])

packaging/Dockerfile.mockbuild

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#
22
# Copyright 2018-2024 Intel Corporation
3+
# Copyright 2025 Hewlett Packard Enterprise Development LP
34
#
45
# 'recipe' for Docker to build an RPM
56
#
@@ -13,15 +14,15 @@ LABEL maintainer="daos@daos.groups.io"
1314

1415
# Use local repo server if present
1516
ARG REPO_FILE_URL
16-
RUN if [ -n "$REPO_FILE_URL" ]; then \
17-
cd /etc/yum.repos.d/ && \
18-
curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \
19-
"$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \
20-
for file in *.repo; do \
21-
true > $file; \
22-
done; \
23-
mv daos_ci-fedora-artifactory.repo{.tmp,}; \
24-
fi
17+
ARG DAOS_LAB_CA_FILE_URL
18+
# script to install OS updates basic tools and daos dependencies
19+
# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh
20+
# script to setup local repo if available
21+
COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh
22+
23+
RUN chmod +x /tmp/repo-helper.sh && \
24+
/tmp/repo-helper.sh && \
25+
rm -f /tmp/repo-helper.sh
2526

2627
# Install basic tools
2728
RUN dnf -y install mock make \
@@ -33,8 +34,8 @@ RUN dnf -y install mock make \
3334
ARG UID=1000
3435

3536
# Add build user (to keep rpmbuild happy)
36-
ENV USER build
37-
ENV PASSWD build
37+
ENV USER=build
38+
ENV PASSWD=build
3839
# add the user to the mock group so it can run mock
3940
RUN if [ $UID != 0 ]; then \
4041
useradd -u $UID -ms /bin/bash $USER; \

packaging/Dockerfile.ubuntu

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#
2+
# Copyright 2019-2021, Intel Corporation
3+
# Copyright 2025 Hewlett Packard Enterprise Development LP
4+
#
5+
# 'recipe' for Docker to build an Debian package
6+
#
7+
# Pull base image
8+
ARG BASE_DISTRO=ubuntu:20.04
9+
FROM $BASE_DISTRO
10+
LABEL org.opencontainers.image.authors="daos@daos.groups.io"
11+
# Needed for later use of BASE_DISTRO
12+
ARG BASE_DISTRO
13+
14+
ARG REPO_FILE_URL
15+
ARG DAOS_LAB_CA_FILE_URL
16+
# script to setup local repo if available
17+
COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh
18+
19+
# Install basic tools
20+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
21+
autoconf bash ca-certificates curl debhelper dh-make \
22+
dpkg-dev dh-python doxygen gcc git git-buildpackage \
23+
javahelper locales make patch pbuilder pkg-config \
24+
python3-dev python3-distro python3-distutils rpm scons wget \
25+
cmake valgrind rpmdevtools
26+
27+
# use same UID as host and default value of 1000 if not specified
28+
ARG UID=1000
29+
30+
# Add build user (to keep chrootbuild happy)
31+
ENV USER=build
32+
RUN useradd -u $UID -ms /bin/bash $USER
33+
34+
# need to run the build command as root, as it needs to chroot
35+
RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \
36+
echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \
37+
fi; \
38+
echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \
39+
echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \
40+
echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \
41+
chmod 0440 /etc/sudoers.d/build; \
42+
visudo -c; \
43+
sudo -l -U build

packaging/Dockerfile.ubuntu.20.04

Lines changed: 13 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,23 @@
1+
# Keep Dockerfile.ubuntu the same as this file until all packaging
2+
# jobs are fixed to have a Dockerfile.ubuntu, and then the common
3+
# Jenkinsfile will be changed to use Dockerfile.ubuntu.
14
#
25
# Copyright 2019-2021, Intel Corporation
6+
# Copyright 2025 Hewlett Packard Enterprise Development LP
37
#
48
# 'recipe' for Docker to build an Debian package
59
#
610
# Pull base image
7-
FROM ubuntu:20.04
11+
ARG BASE_DISTRO=ubuntu:20.04
12+
FROM $BASE_DISTRO
813
LABEL org.opencontainers.image.authors="daos@daos.groups.io"
9-
10-
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
11-
curl gpg
14+
# Needed for later use of BASE_DISTRO
15+
ARG BASE_DISTRO
1216

1317
ARG REPO_FILE_URL
14-
RUN if [ -n "$REPO_FILE_URL" ]; then \
15-
cd /etc/apt/sources.list.d && \
16-
curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \
17-
"$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \
18-
true > ../sources.list && \
19-
mv daos_ci-ubuntu20.04-artifactory.list.tmp \
20-
daos_ci-ubuntu20.04-artifactory.list; \
21-
url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \
22-
else \
23-
url="https://downloads.linux.hpe.com/SDR/repo/ilorest/"; \
24-
fi; \
25-
cd -; \
26-
mkdir -p /usr/local/share/keyrings/; \
27-
curl -f -O "$url"GPG-KEY-hprest; \
28-
gpg --no-default-keyring --keyring ./temp-keyring.gpg \
29-
--import GPG-KEY-hprest; \
30-
gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \
31-
--output /usr/local/share/keyrings/hpe-sdr-public.gpg; \
32-
rm ./temp-keyring.gpg; \
33-
curl -f -O "$REPO_FILE_URL"esad_repo.key; \
34-
gpg --no-default-keyring --keyring ./temp-keyring.gpg \
35-
--import esad_repo.key; \
36-
gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \
37-
--output /usr/local/share/keyrings/daos-stack-public.gpg
18+
ARG DAOS_LAB_CA_FILE_URL
19+
# script to setup local repo if available
20+
COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh
3821

3922
# Install basic tools
4023
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
@@ -48,15 +31,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
4831
ARG UID=1000
4932

5033
# Add build user (to keep chrootbuild happy)
51-
ENV USER build
34+
ENV USER=build
5235
RUN useradd -u $UID -ms /bin/bash $USER
5336

5437
# need to run the build command as root, as it needs to chroot
5538
RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \
5639
echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \
5740
fi; \
5841
echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \
59-
echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \
42+
echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \
6043
echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \
6144
chmod 0440 /etc/sudoers.d/build; \
6245
visudo -c; \

packaging/rpm_chrootbuild

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
set -uex
44

5+
: "${HTTPS_PROXY:=}"
6+
: "${REPO_FILE_URL:=}"
7+
58
cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg
69

710
# Enable mock ccache plugin
@@ -10,6 +13,18 @@ config_opts['plugin_conf']['ccache_enable'] = True
1013
config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/"
1114
EOF
1215

16+
# Optionally add a proxy to mock
17+
if [ -n "$HTTPS_PROXY" ];then
18+
yum_proxy="http://${HTTPS_PROXY##*//}"
19+
echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg
20+
fi
21+
22+
# No proxy for local mirrors
23+
if [ -n "$REPO_FILE_URL" ]; then
24+
direct="${REPO_FILE_URL##*//}"
25+
direct="${direct%%/*}"
26+
echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg
27+
fi
1328

1429
if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then
1530
cat <<EOF >> mock.cfg
@@ -127,7 +142,7 @@ if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \
127142
fi
128143

129144
# Save the ccache
130-
if [ -d /scratch/ ]; then
145+
if [ -d /scratch/mock ]; then
131146
mkdir -p "$bs_dir"/
132147
if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then
133148
echo "Failed to save ccache. Plowing onward."
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
set -uex
3+
4+
# This script is used by Dockerfiles to optionally use
5+
# a local repository instead of a distro provided repository.
6+
7+
: "${REPO_FILE_URL:=}"
8+
: "${DAOS_LAB_CA_FILE_URL:=}"
9+
: "${FVERSION:=latest}"
10+
: "${REPOSITORY_NAME:=artifactory}"
11+
: "${archive:=}"
12+
if [ "$FVERSION" != "latest" ]; then
13+
archive="-archive"
14+
fi
15+
16+
# shellcheck disable=SC2120
17+
disable_repos () {
18+
local repos_dir="$1"
19+
shift
20+
local save_repos
21+
IFS=" " read -r -a save_repos <<< "${*:-} daos_ci-fedora${archive}-${REPOSITORY_NAME}"
22+
if [ -n "$REPO_FILE_URL" ]; then
23+
pushd "$repos_dir"
24+
local repo
25+
for repo in "${save_repos[@]}"; do
26+
mv "$repo".repo{,.tmp}
27+
done
28+
for file in *.repo; do
29+
true > "$file"
30+
done
31+
for repo in "${save_repos[@]}"; do
32+
mv "$repo".repo{.tmp,}
33+
done
34+
popd
35+
fi
36+
}
37+
38+
# Use local repo server if present
39+
install_curl() {
40+
:
41+
}
42+
43+
# Use local repo server if present
44+
install_optional_ca() {
45+
ca_storage="/etc/pki/ca-trust/source/anchors/"
46+
if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then
47+
curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \
48+
"$DAOS_LAB_CA_FILE_URL"
49+
update-ca-trust
50+
fi
51+
}
52+
53+
# Use local repo server if present
54+
# if a local repo server is present and the distro repo server can not
55+
# be reached, have to bootstrap in an environment to get curl installed
56+
# to then install the pre-built repo file.
57+
58+
if [ -n "$REPO_FILE_URL" ]; then
59+
install_curl
60+
install_optional_ca
61+
mkdir -p /etc/yum.repos.d
62+
pushd /etc/yum.repos.d/
63+
curl -k --noproxy '*' -sSf \
64+
-o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \
65+
"{$REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo"
66+
disable_repos /etc/yum.repos.d/
67+
popd
68+
fi
69+
dnf -y install dnf-plugins-core
70+
# This does not work in fedora/41 anymore -- needs investigation
71+
# dnf -y config-manager --save --setopt=assumeyes=True
72+
# dnf config-manager --save --setopt=install_weak_deps=False
73+
dnf clean all
74+
75+
disable_repos /etc/yum.repos.d/ "${save_repos[@]}"
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/bin/bash
2+
set -uex
3+
4+
# This script is used by dockerfiles to optionally use
5+
# a local repository instead of a distro provided repository.
6+
# It will also optionally allow running a /tmp/install script
7+
# for custom packages if present.
8+
9+
: "${REPO_FILE_URL:=}"
10+
: "${HTTPS_PROXY:=}"
11+
: "${DAOS_LAB_CA_FILE_UR:=}"
12+
: "${REPOSITORY_NAME:=artifactory}"
13+
14+
disable_repos () {
15+
if [ -e /etc/apt/sources.list.d/ubuntu.sources ];then
16+
mv /etc/apt/sources.list.d/ubuntu.sources \
17+
etc/apt/sources.list.d/ubuntu.sources.disabled
18+
elif [ -e /etc/apt/sources.list ];then
19+
mv /etc/apt/sources.list \
20+
etc/apt/sources.list.disabled
21+
fi
22+
}
23+
24+
# Use local repo server if present
25+
install_curl() {
26+
27+
if command -v curl; then
28+
echo "found curl!"
29+
return
30+
else
31+
apt-get update
32+
apt-get install curl ca-certificates gpg gpg-agent
33+
fi
34+
35+
if command -v wget; then
36+
echo "found wget!"
37+
return
38+
fi
39+
# If we don't find one of these, we are basically sunk for using
40+
# a local repository mirror.
41+
}
42+
43+
# Use local repo server if present
44+
install_optional_ca() {
45+
ca_storage="/usr/local/share/ca-certificates/"
46+
if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then
47+
curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \
48+
"$DAOS_LAB_CA_FILE_URL"
49+
update-ca-certificates
50+
fi
51+
}
52+
53+
echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt
54+
echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends
55+
if [ -n "$HTTPS_PROXY" ];then
56+
apt_proxy="http://${HTTPS_PROXY##*//}"
57+
echo "Acquire::http::Proxy \"$apt_proxy\";" > \
58+
/etc/apt/apt.conf.d/local_proxy
59+
if [ -n "$REPO_FILE_URL" ]; then
60+
direct="${REPO_FILE_URL##*//}"
61+
direct="${direct%%/*}"
62+
echo "Acquire::http::Proxy { $direct DIRECT; };" >> \
63+
/etc/apt/apt.conf.d/local_proxy
64+
fi
65+
fi
66+
67+
# Use local repo server if present
68+
# if a local repo server is present and the distro repo server can not
69+
# be reached, have to bootstrap in an environment to get curl installed
70+
# to then install the pre-built repo file.
71+
DISTRO_VERSION="${BASE_DISTRO##*:}"
72+
if [ -n "$REPO_FILE_URL" ]; then
73+
install_curl
74+
install_optional_ca
75+
curl -k --noproxy '*' -sSf \
76+
-o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \
77+
"${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list"
78+
disable_repos
79+
mkdir -p /usr/local/share/keyrings/
80+
curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key"
81+
gpg --no-default-keyring --keyring ./temp-keyring.gpg \
82+
--import esad_repo.key
83+
gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \
84+
--output /usr/local/share/keyrings/daos-stack-public.gpg
85+
fi
86+
87+
apt-get update
88+
apt-get upgrade
89+
apt-get install gpg-agent software-properties-common
90+
add-apt-repository ppa:longsleep/golang-backports
91+
apt-get update
92+
chmod +x /tmp/install.sh
93+
/tmp/install.sh
94+
apt-get clean all

0 commit comments

Comments
 (0)