From 1fb56c0a08bdc8dc73b889e678d906a81cece2ef Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 27 Apr 2026 16:46:35 -0400 Subject: [PATCH 01/11] fix --- .gitlab/scripts/poll_e2e.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh index 9e4591474..353c9b38a 100755 --- a/.gitlab/scripts/poll_e2e.sh +++ b/.gitlab/scripts/poll_e2e.sh @@ -49,6 +49,7 @@ fi echo "Triggered downstream pipeline: ${PIPELINE_URL} (ID: ${PIPELINE_ID})" +set +e while true; do STATUS=$(curl --silent \ --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ From bfa24e61821f9938e16fee5dea2377916144d50e Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 28 Apr 2026 10:00:46 -0400 Subject: [PATCH 02/11] only run e2e for arm --- .gitlab/templates/pipeline.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 7f2061d75..7c6ad0fc7 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -231,7 +231,7 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors {{ range $f := (ds "flavors").flavors }} -{{ if $f.needs_layer_publish }} +{{ if and $f.needs_layer_publish (eq $f.arch "arm64") }} {{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }} {{ with $environment := (ds "environments").environments.sandbox }} From e0eaebb4ec3b36cf963f770ae9d2cc76882a5420 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 28 Apr 2026 14:57:57 -0400 Subject: [PATCH 03/11] only run e2e for arm build --- .gitlab/templates/pipeline.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 7c6ad0fc7..89de38f57 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -231,7 +231,7 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors {{ range $f := (ds "flavors").flavors }} -{{ if and $f.needs_layer_publish (eq $f.arch "arm64") }} +{{ if and $f.needs_layer_publish (eq $f.arch "arm64") (not $f.fips) }} {{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }} {{ with $environment := (ds "environments").environments.sandbox }} From bdc1e842c363e8eb99abe7294bf017388fde4976 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 29 Apr 2026 12:45:47 -0400 Subject: [PATCH 04/11] fix architecture --- .gitlab/scripts/poll_e2e.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh index 353c9b38a..3c9f55ec3 100755 --- a/.gitlab/scripts/poll_e2e.sh +++ b/.gitlab/scripts/poll_e2e.sh @@ -28,6 +28,7 @@ fi echo "Triggering DataDog/serverless-e2e-tests pipeline (ref: ${E2E_REF})..." echo " EXTENSION_LAYER_ARN=${EXTENSION_LAYER_ARN}" +echo " ARCHITECTURE=${ARCHITECTURE:-}" TRIGGER_RESPONSE=$(curl --silent --request POST \ --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ @@ -35,7 +36,8 @@ TRIGGER_RESPONSE=$(curl --silent --request POST \ --data "$(jq -n \ --arg ref "$E2E_REF" \ --arg arn "$EXTENSION_LAYER_ARN" \ - '{ref: $ref, variables: [{key: "EXTENSION_VERSION", value: $arn}]}')" \ + --arg arch "${ARCHITECTURE:-}" \ + '{ref: $ref, variables: [{key: "EXTENSION_VERSION", value: $arn}, {key: "ARCHITECTURE", value: $arch}]}')" \ "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipeline") PIPELINE_ID=$(echo "$TRIGGER_RESPONSE" | jq -r '.id // empty') From 8049acd111a7d489b454834a73f5ec3832c5aa8f Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 30 Apr 2026 10:35:03 -0400 Subject: [PATCH 05/11] change e2e test trigger --- .gitlab/scripts/poll_e2e.sh | 89 ----------------------------- .gitlab/templates/pipeline.yaml.tpl | 14 ++--- 2 files changed, 7 insertions(+), 96 deletions(-) delete mode 100755 .gitlab/scripts/poll_e2e.sh diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh deleted file mode 100755 index 3c9f55ec3..000000000 --- a/.gitlab/scripts/poll_e2e.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -# Trigger serverless-e2e-tests pipeline and poll until completion. - -set -euo pipefail - -if [ -z "${EXTENSION_LAYER_ARN:-}" ]; then - echo "ERROR: EXTENSION_LAYER_ARN is not set or empty" - exit 1 -fi - -E2E_PROJECT_ENCODED="DataDog%2Fserverless-e2e-tests" -E2E_REF="${E2E_REF:-main}" - -curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv authanywhere-linux-amd64 /bin/authanywhere && chmod +x /bin/authanywhere - -BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci) - -BTI_RESPONSE=$(curl --silent --request GET \ - --header "$BTI_CI_API_TOKEN" \ - --header "Content-Type: application/vnd.api+json" \ - "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=serverless-e2e-tests") - -GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') -if [ -z "$GITLAB_TOKEN" ]; then - echo "ERROR: could not obtain GitLab token from BTI" - exit 1 -fi - -echo "Triggering DataDog/serverless-e2e-tests pipeline (ref: ${E2E_REF})..." -echo " EXTENSION_LAYER_ARN=${EXTENSION_LAYER_ARN}" -echo " ARCHITECTURE=${ARCHITECTURE:-}" - -TRIGGER_RESPONSE=$(curl --silent --request POST \ - --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ - --header "Content-Type: application/json" \ - --data "$(jq -n \ - --arg ref "$E2E_REF" \ - --arg arn "$EXTENSION_LAYER_ARN" \ - --arg arch "${ARCHITECTURE:-}" \ - '{ref: $ref, variables: [{key: "EXTENSION_VERSION", value: $arn}, {key: "ARCHITECTURE", value: $arch}]}')" \ - "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipeline") - -PIPELINE_ID=$(echo "$TRIGGER_RESPONSE" | jq -r '.id // empty') -PIPELINE_URL=$(echo "$TRIGGER_RESPONSE" | jq -r '.web_url // empty') - -if [ -z "$PIPELINE_ID" ] || [ "$PIPELINE_ID" = "null" ]; then - echo "ERROR: failed to trigger downstream pipeline" - echo "Response: $TRIGGER_RESPONSE" - exit 1 -fi - -echo "Triggered downstream pipeline: ${PIPELINE_URL} (ID: ${PIPELINE_ID})" - -set +e -while true; do - STATUS=$(curl --silent \ - --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ - "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipelines/${PIPELINE_ID}" \ - | jq -r '.status // empty') - - echo -n "E2E pipeline ${PIPELINE_ID} status: ${STATUS}, " - - case "$STATUS" in - success) - echo "E2E tests passed" - exit 0 - ;; - failed) - echo "E2E tests failed" - exit 1 - ;; - canceled|canceling) - echo "E2E tests canceled" - exit 1 - ;; - skipped) - echo "E2E tests skipped" - exit 0 - ;; - running|pending|created|waiting_for_resource|preparing) - echo "still running, checking again in 2 minutes..." - ;; - *) - echo "unknown status, checking again in 2 minutes..." - ;; - esac - - sleep 120 -done diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 89de38f57..baf5df8b2 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -270,13 +270,12 @@ publish layer e2e sandbox ({{ $f.name }}): script: - .gitlab/scripts/publish_layers.sh -e2e-suite ({{ $f.name }}): +e2e-test ({{ $f.name }}): stage: e2e - image: registry.ddbuild.io/images/docker:20.10-py3 - tags: ["arch:amd64"] - timeout: 3h + trigger: + project: DataDog/serverless-e2e-tests + strategy: depend rules: - # Do not block on release, in case E2E is flaky - if: '$CI_COMMIT_TAG =~ /^v.*/' when: on_success allow_failure: true @@ -284,8 +283,9 @@ e2e-suite ({{ $f.name }}): needs: - job: "publish layer e2e sandbox ({{ $f.name }})" artifacts: true - script: - - .gitlab/scripts/poll_e2e.sh + variables: + EXTENSION_VERSION: $EXTENSION_LAYER_ARN + ARCHITECTURE: {{ $f.arch }} {{ end }} {{ end }} From 7ac754d20e0aeb37df7007e012a863241244e072 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 30 Apr 2026 16:12:23 -0400 Subject: [PATCH 06/11] add e2e-status polling job --- .gitlab/scripts/poll_e2e.sh | 41 +++++++++++++++++++++++++++++ .gitlab/templates/pipeline.yaml.tpl | 8 ++++++ 2 files changed, 49 insertions(+) create mode 100755 .gitlab/scripts/poll_e2e.sh diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh new file mode 100755 index 000000000..20da27227 --- /dev/null +++ b/.gitlab/scripts/poll_e2e.sh @@ -0,0 +1,41 @@ +#!/bin/bash +curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" \ + && mv "authanywhere-linux-amd64" /bin/authanywhere \ + && chmod +x /bin/authanywhere + +BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci) + +BTI_RESPONSE=$(curl --silent --request GET \ + --header "$BTI_CI_API_TOKEN" \ + --header "Content-Type: application/vnd.api+json" \ + "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-extension") + +GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') + +URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" + +echo "Fetching E2E job status from: $URL" + +while true; do + RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL") + E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test (arm64)") | .downstream_pipeline.status') + echo -n "E2E job status: $E2E_JOB_STATUS, " + if [ "$E2E_JOB_STATUS" == "success" ]; then + echo "E2E tests completed successfully" + exit 0 + elif [ "$E2E_JOB_STATUS" == "failed" ]; then + echo "E2E tests failed" + exit 1 + elif [ "$E2E_JOB_STATUS" == "running" ]; then + echo "E2E tests are still running, retrying in 2 minutes..." + elif [ "$E2E_JOB_STATUS" == "canceled" ]; then + echo "E2E tests were canceled" + exit 1 + elif [ "$E2E_JOB_STATUS" == "skipped" ]; then + echo "E2E tests were skipped" + exit 0 + else + echo "Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..." + fi + sleep 120 +done diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index baf5df8b2..55817447b 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -287,6 +287,14 @@ e2e-test ({{ $f.name }}): EXTENSION_VERSION: $EXTENSION_LAYER_ARN ARCHITECTURE: {{ $f.arch }} +e2e-test-status: + stage: e2e + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:arm64"] + timeout: 3h + script: + - .gitlab/scripts/poll_e2e.sh + {{ end }} {{ end }} {{ end }} From abb905b222d23fd7140601a51a6eba6e713e533c Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 30 Apr 2026 18:40:30 -0400 Subject: [PATCH 07/11] fix poll script --- .gitlab/scripts/poll_e2e.sh | 13 ++++++++++++- .gitlab/templates/pipeline.yaml.tpl | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh index 20da27227..d4beea789 100755 --- a/.gitlab/scripts/poll_e2e.sh +++ b/.gitlab/scripts/poll_e2e.sh @@ -12,12 +12,23 @@ BTI_RESPONSE=$(curl --silent --request GET \ GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') +if [ -z "$GITLAB_TOKEN" ]; then + echo "ERROR: Failed to obtain GitLab token. BTI HTTP status: $(echo "$BTI_RESPONSE" | jq -r '.status // "unknown"')" + exit 1 +fi + URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" echo "Fetching E2E job status from: $URL" while true; do - RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL") + HTTP_STATUS=$(curl -s -o /tmp/e2e_response.json -w "%{http_code}" --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL") + if [ "$HTTP_STATUS" != "200" ]; then + echo "WARNING: GitLab API returned HTTP $HTTP_STATUS, retrying in 2 minutes..." + sleep 120 + continue + fi + RESPONSE=$(cat /tmp/e2e_response.json) E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test (arm64)") | .downstream_pipeline.status') echo -n "E2E job status: $E2E_JOB_STATUS, " if [ "$E2E_JOB_STATUS" == "success" ]; then diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 55817447b..f50a82bd3 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -290,7 +290,7 @@ e2e-test ({{ $f.name }}): e2e-test-status: stage: e2e image: registry.ddbuild.io/images/docker:20.10-py3 - tags: ["arch:arm64"] + tags: ["arch:amd64"] timeout: 3h script: - .gitlab/scripts/poll_e2e.sh From 1f87a14ec06ff7cde51d1fbfb8bdc5cf64d411c7 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Fri, 1 May 2026 08:25:36 -0400 Subject: [PATCH 08/11] also test arm fips --- .gitlab/scripts/poll_e2e.sh | 2 +- .gitlab/templates/pipeline.yaml.tpl | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh index d4beea789..f68fd7773 100755 --- a/.gitlab/scripts/poll_e2e.sh +++ b/.gitlab/scripts/poll_e2e.sh @@ -29,7 +29,7 @@ while true; do continue fi RESPONSE=$(cat /tmp/e2e_response.json) - E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test (arm64)") | .downstream_pipeline.status') + E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r --arg name "$E2E_JOB_NAME" '.[] | select(.name==$name) | .downstream_pipeline.status') echo -n "E2E job status: $E2E_JOB_STATUS, " if [ "$E2E_JOB_STATUS" == "success" ]; then echo "E2E tests completed successfully" diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index f50a82bd3..ff07318ef 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -231,7 +231,7 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors {{ range $f := (ds "flavors").flavors }} -{{ if and $f.needs_layer_publish (eq $f.arch "arm64") (not $f.fips) }} +{{ if and $f.needs_layer_publish (eq $f.arch "arm64") }} {{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }} {{ with $environment := (ds "environments").environments.sandbox }} @@ -287,11 +287,13 @@ e2e-test ({{ $f.name }}): EXTENSION_VERSION: $EXTENSION_LAYER_ARN ARCHITECTURE: {{ $f.arch }} -e2e-test-status: +e2e-test-status ({{ $f.name }}): stage: e2e image: registry.ddbuild.io/images/docker:20.10-py3 tags: ["arch:amd64"] timeout: 3h + variables: + E2E_JOB_NAME: "e2e-test ({{ $f.name }})" script: - .gitlab/scripts/poll_e2e.sh From 91de7e6beae97e1c5fa1b2c5bc67ad1bad46f74d Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Fri, 1 May 2026 13:00:04 -0400 Subject: [PATCH 09/11] fix log --- .gitlab/scripts/poll_e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh index f68fd7773..938fd06e3 100755 --- a/.gitlab/scripts/poll_e2e.sh +++ b/.gitlab/scripts/poll_e2e.sh @@ -13,7 +13,7 @@ BTI_RESPONSE=$(curl --silent --request GET \ GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') if [ -z "$GITLAB_TOKEN" ]; then - echo "ERROR: Failed to obtain GitLab token. BTI HTTP status: $(echo "$BTI_RESPONSE" | jq -r '.status // "unknown"')" + echo "ERROR: could not obtain GitLab token from BTI" exit 1 fi From 6db4dde42e24d6971c4b8a1fd02cba789145fee6 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 4 May 2026 13:32:14 -0400 Subject: [PATCH 10/11] fix job dependencies --- .gitlab/templates/pipeline.yaml.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index ff07318ef..77b22682f 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -294,6 +294,9 @@ e2e-test-status ({{ $f.name }}): timeout: 3h variables: E2E_JOB_NAME: "e2e-test ({{ $f.name }})" + needs: + - job: "publish layer e2e sandbox ({{ $f.name }})" + artifacts: false script: - .gitlab/scripts/poll_e2e.sh From d33e05910bc9ef4882c8ad69456e3cae735edd45 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 4 May 2026 15:42:56 -0400 Subject: [PATCH 11/11] change arch to amd64 --- .gitlab/templates/pipeline.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 77b22682f..93a390fdb 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -231,7 +231,7 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors {{ range $f := (ds "flavors").flavors }} -{{ if and $f.needs_layer_publish (eq $f.arch "arm64") }} +{{ if and $f.needs_layer_publish (eq $f.arch "amd64") }} {{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }} {{ with $environment := (ds "environments").environments.sandbox }}