Skip to content

Commit cd985ee

Browse files
purple4reinapurple4reina
committed
Use shared logic to install ddtrace from wheel during PR.
1 parent 89084d6 commit cd985ee

4 files changed

Lines changed: 166 additions & 75 deletions

File tree

‎ci/input_files/build.yaml.tpl‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,6 @@ stages:
77
- publish
88
- e2e
99

10-
.python-before-script: &python-before-script
11-
- pip install virtualenv
12-
- virtualenv venv
13-
- source venv/bin/activate
14-
- pip install .[dev]
15-
- pip install poetry
16-
1710
default:
1811
retry:
1912
max: 1
@@ -73,7 +66,8 @@ lint python:
7366
tags: ["arch:amd64"]
7467
image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
7568
cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
76-
before_script: *python-before-script
69+
before_script:
70+
- PYTHON_VERSION={{ $runtime.python_version }} ./scripts/setup_python_env.sh
7771
script:
7872
- source venv/bin/activate
7973
- ./scripts/check_format.sh
@@ -83,7 +77,8 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
8377
tags: ["arch:amd64"]
8478
image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
8579
cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
86-
before_script: *python-before-script
80+
before_script:
81+
- PYTHON_VERSION={{ $runtime.python_version }} ./scripts/setup_python_env.sh
8782
script:
8883
- source venv/bin/activate
8984
- pytest -vv
@@ -195,7 +190,8 @@ publish-pypi-package:
195190
stage: publish
196191
tags: ["arch:amd64"]
197192
image: registry.ddbuild.io/images/docker:20.10-py3
198-
before_script: *python-before-script
193+
before_script:
194+
- ./scripts/setup_python_env.sh
199195
cache: []
200196
rules:
201197
- if: '$CI_COMMIT_TAG =~ /^v.*/'

‎scripts/_spec_ddtrace_dep.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
7+
# Shared helpers for rewriting the ddtrace dependency in pyproject.toml.
8+
# Sourced by scripts/build_layers.sh and scripts/setup_python_env.sh, so the
9+
# layer build and the unit-test/lint/publish jobs use the same env-var
10+
# contract and resolve the dep in a single pip pass.
11+
#
12+
# Env-var contract (highest precedence first):
13+
# DD_TRACE_COMMIT Specific dd-trace-py commit SHA from GitHub.
14+
# DD_TRACE_COMMIT_BRANCH dd-trace-py branch name from GitHub.
15+
# DD_TRACE_WHEEL Path to a pre-built ddtrace .whl file.
16+
# UPSTREAM_PIPELINE_ID GitLab pipeline ID from dd-trace-py. Looks up the
17+
# matching wheel from S3, trying the smaller
18+
# serverless build first then falling back to the
19+
# standard manylinux2014 build.
20+
#
21+
# When none of these are set, spec_ddtrace_dep is a no-op.
22+
#
23+
# When UPSTREAM_PIPELINE_ID is set, also requires:
24+
# PYTHON_VERSION e.g. "3.12" (used to build the cpXY platform tag)
25+
# ARCH "amd64" (default) or "arm64"
26+
27+
# Replace the ddtrace dependency block in pyproject.toml.
28+
# Usage: replace_ddtrace_dep "ddtrace = { ... }"
29+
replace_ddtrace_dep() {
30+
echo "Replacing ddtrace dep with: $1"
31+
perl -i -0777 -pe "s|ddtrace = \[[^\]]*\]|$1|gs" pyproject.toml
32+
}
33+
34+
# Search S3 for a wheel matching basename + index, then rewrite the ddtrace
35+
# dep to point at the downloaded file. Globals required:
36+
# S3_BASE, PY_TAG, PLATFORM
37+
# Returns 0 on success, 1 if no matching wheel was found at the index.
38+
_search_and_spec_s3_wheel() {
39+
local basename=$1
40+
local index=$2
41+
local search_pattern="${basename}-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl"
42+
local index_url="${S3_BASE}/index-${index}.html"
43+
echo "Searching for wheel ${search_pattern} in ${index_url}"
44+
local wheel_file
45+
wheel_file=$(curl -sSfL "${index_url}" | grep -o "${search_pattern}" | head -n 1 || true)
46+
if [ -z "$wheel_file" ]; then
47+
return 1
48+
fi
49+
curl -sSfL "${S3_BASE}/${wheel_file}" -o "${wheel_file}"
50+
echo "Using S3 wheel: ${wheel_file}"
51+
replace_ddtrace_dep "${basename} = { file = \"${wheel_file}\" }"
52+
}
53+
54+
# Rewrite pyproject.toml's ddtrace dep based on the env-var precedence above.
55+
# No-op if no override env var is set. Returns non-zero if UPSTREAM_PIPELINE_ID
56+
# is set but no matching S3 wheel is found.
57+
spec_ddtrace_dep() {
58+
if [ -n "${DD_TRACE_COMMIT:-}" ]; then
59+
replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"${DD_TRACE_COMMIT}\" }"
60+
elif [ -n "${DD_TRACE_COMMIT_BRANCH:-}" ]; then
61+
replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"${DD_TRACE_COMMIT_BRANCH}\" }"
62+
elif [ -n "${DD_TRACE_WHEEL:-}" ]; then
63+
local basename
64+
basename=$(sed 's/^.*\///' <<< "${DD_TRACE_WHEEL%%-*}")
65+
replace_ddtrace_dep "${basename} = { file = \"${DD_TRACE_WHEEL}\" }"
66+
elif [ -n "${UPSTREAM_PIPELINE_ID:-}" ]; then
67+
if [ -z "${PYTHON_VERSION:-}" ]; then
68+
echo "ERROR: PYTHON_VERSION must be set when UPSTREAM_PIPELINE_ID is set" >&2
69+
return 1
70+
fi
71+
S3_BASE="https://dd-trace-py-builds.s3.amazonaws.com/${UPSTREAM_PIPELINE_ID}"
72+
PY_TAG="cp$(echo "$PYTHON_VERSION" | tr -d '.')"
73+
if [ "${ARCH:-amd64}" = "amd64" ]; then
74+
PLATFORM="manylinux2014_x86_64"
75+
else
76+
PLATFORM="manylinux2014_aarch64"
77+
fi
78+
_search_and_spec_s3_wheel "ddtrace_serverless" "serverless" \
79+
|| _search_and_spec_s3_wheel "ddtrace" "manylinux2014" \
80+
|| { echo "ERROR: No matching ddtrace wheel for ${PY_TAG} ${PLATFORM} in pipeline ${UPSTREAM_PIPELINE_ID}" >&2; return 1; }
81+
fi
82+
}

‎scripts/build_layers.sh‎

Lines changed: 9 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -88,68 +88,16 @@ cleanup() {
8888
}
8989
trap cleanup EXIT
9090

91-
# Helper: replace the multi-line ddtrace dependency in pyproject.toml.
92-
# Uses perl instead of sed -z for macOS/Linux portability.
93-
replace_ddtrace_dep() {
94-
echo "Replacing dep with $1"
95-
perl -i -0777 -pe "s|ddtrace = \[[^\]]*\]|$1|gs" pyproject.toml
96-
}
91+
# Source the shared ddtrace-dep specification logic. spec_ddtrace_dep reads
92+
# DD_TRACE_COMMIT / DD_TRACE_COMMIT_BRANCH / DD_TRACE_WHEEL / UPSTREAM_PIPELINE_ID
93+
# (PYTHON_VERSION + ARCH for the S3 path) and rewrites the ddtrace dep block
94+
# in pyproject.toml.
95+
source "$(dirname "$0")/_spec_ddtrace_dep.sh"
9796

9897
function make_path_absolute {
9998
echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
10099
}
101100

102-
function search_wheel {
103-
# Args: [wheel base name] [index]
104-
105-
WHEEL_BASENAME=$1
106-
INDEX=$2
107-
108-
SEARCH_PATTERN="${WHEEL_BASENAME}-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl"
109-
INDEX_URL="${S3_BASE}/index-${INDEX}.html"
110-
echo "Searching for wheel ${SEARCH_PATTERN}"
111-
export WHEEL_FILE=$(curl -sSfL ${INDEX_URL} | grep -o "$SEARCH_PATTERN" | head -n 1)
112-
if [ ! -z "${WHEEL_FILE}" ]; then
113-
curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}"
114-
echo "Using S3 wheel: ${WHEEL_FILE}"
115-
replace_ddtrace_dep "${WHEEL_BASENAME} = { file = \"${WHEEL_FILE}\" }"
116-
fi
117-
}
118-
119-
function find_and_spec_wheel {
120-
# Args: [python version] [wheel base name] [index]
121-
122-
arch=$2
123-
wheel_basename=$3
124-
index=$4
125-
126-
# Restore pyproject.toml to a clean state for each build iteration
127-
cp pyproject.toml.bak pyproject.toml
128-
129-
# Replace ddtrace source if necessary
130-
if [ -n "$DD_TRACE_COMMIT" ]; then
131-
replace_ddtrace_dep "${wheel_basename} = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }"
132-
elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then
133-
replace_ddtrace_dep "${wheel_basename} = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }"
134-
elif [ -n "$DD_TRACE_WHEEL" ]; then
135-
wheel_basename=$(sed 's/^.*\///' <<< ${DD_TRACE_WHEEL%%-*})
136-
replace_ddtrace_dep "${wheel_basename} = { file = \"$DD_TRACE_WHEEL\" }"
137-
elif [ -n "$UPSTREAM_PIPELINE_ID" ]; then
138-
S3_BASE="https://dd-trace-py-builds.s3.amazonaws.com/${UPSTREAM_PIPELINE_ID}"
139-
if [ "${arch}" = "amd64" ]; then
140-
PLATFORM="manylinux2014_x86_64"
141-
else
142-
PLATFORM="manylinux2014_aarch64"
143-
fi
144-
PY_TAG="cp$(echo "$1" | tr -d '.')"
145-
search_wheel ${wheel_basename} ${index}
146-
if [ -z "${WHEEL_FILE}" ]; then
147-
echo "No S3 wheel found for ${PY_TAG} ${PLATFORM}, using default pyproject.toml version"
148-
return 1
149-
fi
150-
fi
151-
}
152-
153101
function docker_build_zip {
154102
# Args: [python version] [zip destination]
155103

@@ -180,14 +128,10 @@ do
180128
for architecture in "${ARCHS[@]}"
181129
do
182130
echo "Building layer for Python ${python_version} arch=${architecture}"
183-
set +e
184-
find_and_spec_wheel ${python_version} ${architecture} "ddtrace_serverless" "serverless"
185-
FAILURE=$?
186-
if [ $FAILURE != 0 ]; then
187-
echo "Attempting layer build again with package ddtrace"
188-
find_and_spec_wheel ${python_version} ${architecture} "ddtrace" "manylinux2014"
189-
fi
190-
set -e
131+
# Restore pyproject.toml to a clean state before each iteration so the
132+
# rewrite is deterministic regardless of what the previous loop did.
133+
cp pyproject.toml.bak pyproject.toml
134+
PYTHON_VERSION="${python_version}" ARCH="${architecture}" spec_ddtrace_dep
191135
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-${architecture}-${python_version}.zip ${architecture}
192136
done
193137
done

‎scripts/setup_python_env.sh‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
7+
# Sets up the Python environment for the lint, unit-test, and publish-pypi
8+
# CI jobs (and for local repro of those flows). Replaces the inline
9+
# .python-before-script anchor previously defined in
10+
# ci/input_files/build.yaml.tpl.
11+
#
12+
# Steps:
13+
# 1. (Optional) Rewrite pyproject.toml's ddtrace dep based on the
14+
# env-var contract documented in scripts/_spec_ddtrace_dep.sh
15+
# (DD_TRACE_COMMIT / DD_TRACE_COMMIT_BRANCH / DD_TRACE_WHEEL /
16+
# UPSTREAM_PIPELINE_ID). When dd-trace-py's CI triggers this repo's
17+
# pipeline it sets UPSTREAM_PIPELINE_ID, so the unit-test job
18+
# exercises the PR's wheel rather than the released ddtrace.
19+
# 2. Create and activate a virtualenv ("venv/").
20+
# 3. Install lambda-python's runtime + dev dependencies. pip resolves the
21+
# whole graph in one pass against the (possibly rewritten) pyproject.toml,
22+
# so any version conflicts surface as install errors instead of
23+
# runtime surprises.
24+
# 4. Install poetry.
25+
#
26+
# Same dep-resolution path as scripts/build_layers.sh — both source
27+
# scripts/_spec_ddtrace_dep.sh.
28+
#
29+
# DD_TRACE_COMMIT / DD_TRACE_COMMIT_BRANCH build ddtrace from source, which
30+
# requires cargo, cmake, and a C/C++ toolchain — not present in the slim
31+
# Python runner images. They are intended for local repro / git-bisect
32+
# workflows. The dd-trace-py CI trigger uses UPSTREAM_PIPELINE_ID.
33+
#
34+
# Venv contract: this script sources venv/bin/activate inside its own
35+
# subshell, so the activation does NOT persist into the calling job. Calling
36+
# jobs must `source venv/bin/activate` themselves before running their
37+
# command (matching the existing pattern in build.yaml.tpl).
38+
#
39+
# Environment variables:
40+
# PYTHON_VERSION Python minor version (e.g. 3.12 or just 12). Required
41+
# when the UPSTREAM_PIPELINE_ID branch is taken.
42+
43+
set -e
44+
45+
# Normalize Python version shorthand (e.g. 12 -> 3.12, 3.12 -> 3.12)
46+
if [ -n "${PYTHON_VERSION:-}" ]; then
47+
if [[ "$PYTHON_VERSION" =~ ^[0-9]+$ ]]; then
48+
PYTHON_VERSION="3.${PYTHON_VERSION}"
49+
fi
50+
fi
51+
52+
# Backup pyproject.toml so the rewrite doesn't persist across runs (matters
53+
# for local invocations; CI runners are ephemeral but cheap to be tidy).
54+
cp pyproject.toml pyproject.toml.bak
55+
cleanup() {
56+
mv pyproject.toml.bak pyproject.toml 2>/dev/null || true
57+
}
58+
trap cleanup EXIT
59+
60+
source "$(dirname "$0")/_spec_ddtrace_dep.sh"
61+
spec_ddtrace_dep
62+
63+
pip install virtualenv
64+
virtualenv venv
65+
source venv/bin/activate
66+
pip install .[dev]
67+
pip install poetry
68+
69+
python -c "import ddtrace; print('ddtrace version:', ddtrace.__version__)"

0 commit comments

Comments
 (0)