Skip to content

Commit ff1a65e

Browse files
achmelorichard-salac
achmelo
authored and
richard-salac
committed
OpenTelemetry integration test validating resource attributes
Signed-off-by: Richard Salac <richard.salac@broadcom.com>
1 parent df498b2 commit ff1a65e

14 files changed

Lines changed: 438 additions & 37 deletions

File tree

‎.github/workflows/service-registration.yml‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,77 @@ jobs:
2929
- name: Build with Gradle
3030
run: >
3131
./gradlew clean build --info --scan
32-
- name: Run startup check
32+
33+
- name: Run startup check for microservices
3334
run: >
3435
./gradlew runStartUpCheck --info --scan -Denvironment.startServices=true
36+
37+
- name: Start OpenTelemetry docker containers
38+
run: |
39+
cd otel
40+
chmod -R 777 otel-*
41+
docker compose up -d
42+
43+
- name: Check OpenTelemetry containers
44+
run: |
45+
echo "Checking OpenTelemetry Golden Tester..."
46+
curl -s http://localhost:5318/v1/metrics -H "Content-Type: application/json" -d "{}" \
47+
--fail \
48+
--retry-all-errors \
49+
--retry-delay 2 \
50+
--retry 10
51+
echo ""
52+
echo "curl exit code: "$?
53+
echo ""
54+
55+
echo "Checking OpenTelemetry Collector..."
56+
curl -s http://localhost:4318/v1/metrics -H "Content-Type: application/json" -d "{}" \
57+
--fail \
58+
--retry-all-errors \
59+
--retry-delay 2 \
60+
--retry 10
61+
echo ""
62+
echo "curl exit code: "$?
63+
64+
- name: Run startup check for modulith
65+
run: >
66+
./gradlew runStartUpCheckWithOpenTelemetry --info --scan -Denvironment.startServices=true -Denvironment.modulith=true
67+
68+
- name: Verify OpenTelemetry data with Golden Tester
69+
if: always()
70+
run: |
71+
echo "Waiting for Golden Validator to finish..."
72+
73+
# This blocks until the golden container exits (success or timeout)
74+
EXIT_CODE=$(docker wait golden)
75+
76+
# Display logs to see the diff if it failed
77+
echo "Golden container logs:"
78+
docker logs golden
79+
80+
echo ""
81+
82+
if [ "$EXIT_CODE" -ne 0 ]; then
83+
echo "::error::OpenTelemetry data validation failed! See logs above for diff."
84+
exit 1
85+
fi
86+
echo "OpenTelemetry data validation passed!"
87+
88+
- name: Stop OpenTelemetry Collector and print logs
89+
if: always()
90+
run: |
91+
docker stop collector -t 60
92+
93+
echo "Collector container logs:"
94+
docker logs collector
95+
3596
- name: Store results
3697
uses: actions/upload-artifact@v4
3798
if: always()
3899
with:
39100
name: BuildAndTest-${{ env.JOB_ID }}
40101
path: |
41102
*/build/reports/**
103+
otel/**
42104
43105
- uses: ./.github/actions/teardown

‎api-catalog-package/src/main/resources/bin/start.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,13 @@ then
135135
ZOWE_CONSOLE_LOG_CHARSET=IBM-1047
136136
fi
137137
fi
138+
139+
#Set the external URL only if the variables are defined so the APIML can fallback if the property is null
140+
EXTERNAL_URL=""
141+
if [ -n "${externalProtocol}" ] && [ -n "${ZWE_zowe_externalDomains_0}" ] && [ -n "${ZWE_zowe_externalPort}" ]; then
142+
EXTERNAL_URL="-Dapiml.service.externalUrl=${externalProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}"
143+
fi
144+
138145
LIBPATH="$LIBPATH":"/lib"
139146
LIBPATH="$LIBPATH":"/usr/lib"
140147
LIBPATH="$LIBPATH":"${JAVA_HOME}"/bin
@@ -287,6 +294,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CATALOG_CODE} ${JAVA_BIN_DIR}java \
287294
${ADD_OPENS} \
288295
${LOGBACK} \
289296
${JVM_SECURITY_PROPERTIES} \
297+
${EXTERNAL_URL} \
290298
-Dibm.serversocket.recover=true \
291299
-Dfile.encoding=UTF-8 \
292300
-Dlogging.charset.console=${ZOWE_CONSOLE_LOG_CHARSET} \
@@ -298,7 +306,6 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CATALOG_CODE} ${JAVA_BIN_DIR}java \
298306
-Dapiml.service.gatewayHostname=${ZWE_GATEWAY_HOST:-${ZWE_haInstance_hostname:-localhost}} \
299307
-Dapiml.logs.location=${ZWE_zowe_logDirectory} \
300308
-Dapiml.health.protected=${ZWE_configs_apiml_health_protected:-true} \
301-
-Dapiml.service.externalUrl="${externalProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}" \
302309
-Dapiml.discovery.staticApiDefinitionsDirectories=${ZWE_STATIC_DEFINITIONS_DIR} \
303310
-Dapiml.security.ssl.verifySslCertificatesOfServices=${verifySslCertificatesOfServices:-false} \
304311
-Dapiml.security.ssl.nonStrictVerifySslCertificatesOfServices=${nonStrictVerifySslCertificatesOfServices:-false} \

‎apiml-package/src/main/resources/bin/start.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,12 @@ if [ -n "${ZWE_configs_storage_vsam_name}" ]; then
229229
VSAM_FILE_NAME=//\'${ZWE_configs_storage_vsam_name:-${ZWE_components_caching_service_storage_vsam_name}}\'
230230
fi
231231

232+
#Set the external URL only if the variables are defined so the APIML can fallback if the property is null
233+
EXTERNAL_URL=""
234+
if [ -n "${externalProtocol}" ] && [ -n "${ZWE_zowe_externalDomains_0}" ] && [ -n "${ZWE_zowe_externalPort}" ]; then
235+
EXTERNAL_URL="-Dapiml.service.externalUrl=${externalProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}"
236+
fi
237+
232238
LIBPATH="$LIBPATH":"/lib"
233239
LIBPATH="$LIBPATH":"/usr/lib"
234240
LIBPATH="$LIBPATH":"${JAVA_HOME}/bin"
@@ -340,6 +346,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${APIML_CODE} ${JAVA_BIN_DIR}java \
340346
${ADD_OPENS} \
341347
${LOGBACK} \
342348
${JVM_SECURITY_PROPERTIES} \
349+
${EXTERNAL_URL} \
343350
-Dapiml.cache.storage.location=${ZWE_zowe_workspaceDirectory}/api-mediation/${ZWE_haInstance_id:-localhost} \
344351
-Dapiml.catalog.customStyle.backgroundColor=${ZWE_components_apicatalog_apiml_catalog_customStyle_backgroundColor:-${ZWE_configs_apiml_catalog_customStyle_backgroundColor:-}} \
345352
-Dapiml.catalog.customStyle.docLink=${ZWE_components_apicatalog_apiml_catalog_customStyle_docLink:-${ZWE_configs_apiml_catalog_customStyle_docLink:-}} \
@@ -413,7 +420,6 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${APIML_CODE} ${JAVA_BIN_DIR}java \
413420
-Dapiml.service.apimlId=${ZWE_components_gateway_apimlId:-${ZWE_configs_apimlId:-}} \
414421
-Dapiml.service.corsEnabled=${ZWE_components_gateway_apiml_service_corsEnabled:-${ZWE_configs_apiml_service_corsEnabled:-false}} \
415422
-Dapiml.service.corsAllowedMethods=${ZWE_components_gateway_apiml_service_corsAllowedMethods:-${ZWE_configs_apiml_service_corsAllowedMethods:-GET,HEAD,POST,PATCH,DELETE,PUT,OPTIONS}} \
416-
-Dapiml.service.externalUrl="${externalProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}" \
417423
-Dapiml.service.forwardClientCertEnabled=${ZWE_components_gateway_apiml_security_x509_enabled:-${ZWE_configs_apiml_security_x509_enabled:-false}} \
418424
-Dapiml.service.hostname=${ZWE_haInstance_hostname:-localhost} \
419425
-Dapiml.service.port=${ZWE_components_gateway_port:-${ZWE_configs_port:-7554}} \

‎gateway-package/src/main/resources/bin/start.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,12 @@ else
191191
externalProtocol="http"
192192
fi
193193

194+
#Set the external URL only if the variables are defined so the APIML can fallback if the property is null
195+
EXTERNAL_URL=""
196+
if [ -n "${externalProtocol}" ] && [ -n "${ZWE_zowe_externalDomains_0}" ] && [ -n "${ZWE_zowe_externalPort}" ]; then
197+
EXTERNAL_URL="-Dapiml.service.externalUrl=${externalProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}"
198+
fi
199+
194200
LIBPATH="$LIBPATH":"/lib"
195201
LIBPATH="$LIBPATH":"/usr/lib"
196202
LIBPATH="$LIBPATH":"${JAVA_HOME}/bin"
@@ -306,6 +312,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${GATEWAY_CODE} ${JAVA_BIN_DIR}java \
306312
${ADD_OPENS} \
307313
${LOGBACK} \
308314
${JVM_SECURITY_PROPERTIES} \
315+
${EXTERNAL_URL} \
309316
-Dapiml.connection.idleConnectionTimeoutSeconds=${ZWE_configs_apiml_connection_idleConnectionTimeoutSeconds:-5} \
310317
-Dapiml.connection.timeout=${ZWE_configs_apiml_connection_timeout:-60000} \
311318
-Dapiml.connection.timeToLive=${ZWE_configs_apiml_connection_timeToLive:-10000} \
@@ -341,7 +348,6 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${GATEWAY_CODE} ${JAVA_BIN_DIR}java \
341348
-Dapiml.service.apimlId=${ZWE_configs_apimlId:-} \
342349
-Dapiml.service.corsEnabled=${ZWE_configs_apiml_service_corsEnabled:-false} \
343350
-Dapiml.service.corsAllowedMethods=${ZWE_configs_apiml_service_corsAllowedMethods:-GET,HEAD,POST,PATCH,DELETE,PUT,OPTIONS} \
344-
-Dapiml.service.externalUrl="${externalProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}" \
345351
-Dapiml.service.forwardClientCertEnabled=${ZWE_configs_apiml_security_x509_enabled:-false} \
346352
-Dapiml.service.hostname=${ZWE_haInstance_hostname:-localhost} \
347353
-Dapiml.service.port=${ZWE_configs_port:-7554} \

‎integration-tests/build.gradle‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,18 @@ task runStartUpCheck(type: Test) {
101101
group 'integration tests'
102102
description "Check that the API Mediation Layer is up and running"
103103

104+
systemProperties System.properties
105+
useJUnitPlatform {
106+
includeTags 'StartupCheck'
107+
excludeTags 'OpenTelemetryTest'
108+
}
109+
outputs.upToDateWhen { false }
110+
}
111+
112+
task runStartUpCheckWithOpenTelemetry(type: Test) {
113+
group 'integration tests'
114+
description "Check that the API Mediation Layer is up and running with graceful wait for OpenTelemetry"
115+
104116
systemProperties System.properties
105117
useJUnitPlatform {
106118
includeTags 'StartupCheck'

‎integration-tests/src/test/java/org/zowe/apiml/startup/ApiMediationLayerStartTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,15 @@
1010

1111
package org.zowe.apiml.startup;
1212

13+
import lombok.SneakyThrows;
1314
import org.junit.jupiter.api.BeforeEach;
1415
import org.junit.jupiter.api.Test;
1516
import org.zowe.apiml.startup.impl.ApiMediationLayerStartupChecker;
17+
import org.zowe.apiml.util.categories.OpenTelemetryTest;
1618
import org.zowe.apiml.util.categories.StartupCheck;
1719

20+
import java.time.Duration;
21+
1822
import static org.junit.jupiter.api.Assertions.assertTrue;
1923

2024
@StartupCheck
@@ -29,4 +33,15 @@ void setUp() {
2933
void checkApiMediationLayerStart() {
3034
assertTrue(true);
3135
}
36+
37+
@Test
38+
@OpenTelemetryTest
39+
@SneakyThrows
40+
void giveOpenTelemetryTimeToSendMetrics() {
41+
//The application has to run for a while to collect and send the telemetry data
42+
//so they can be evaluated in the OpenTelemetry Golden Tester
43+
Thread.sleep(Duration.ofSeconds(30).toMillis());
44+
assertTrue(true);
45+
}
46+
3247
}

‎integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,14 @@
3939
import static java.util.concurrent.TimeUnit.MINUTES;
4040
import static java.util.concurrent.TimeUnit.SECONDS;
4141
import static org.awaitility.Awaitility.await;
42+
import static org.zowe.apiml.util.config.ConfigReader.IS_MODULITH_ENABLED;
4243

4344
/**
4445
* Checks and waits until the testing environment is ready to be tested.
4546
*/
4647
@Slf4j
4748
public class ApiMediationLayerStartupChecker {
4849

49-
private static final boolean IS_MODULITH_ENABLED = Boolean.parseBoolean(System.getProperty("environment.modulith"));
50-
5150
private final GatewayServiceConfiguration gatewayConfiguration;
5251
private final DiscoverableClientConfiguration discoverableClientConfiguration;
5352
private final DiscoveryServiceConfiguration discoveryServiceConfiguration;

‎integration-tests/src/test/java/org/zowe/apiml/util/categories/OpenTelemetryTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* This program and the accompanying materials are made available under the terms of the
3+
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
4+
* https://www.eclipse.org/legal/epl-v20.html
5+
*
6+
* SPDX-License-Identifier: EPL-2.0
7+
*
8+
* Copyright Contributors to the Zowe Project.
9+
*/
10+
11+
package org.zowe.apiml.util.categories;
12+
13+
import org.junit.jupiter.api.Tag;
14+
15+
import java.lang.annotation.Retention;
16+
import java.lang.annotation.RetentionPolicy;
17+
import java.lang.annotation.Target;
18+
19+
import static java.lang.annotation.ElementType.METHOD;
20+
import static java.lang.annotation.ElementType.TYPE;
21+
22+
@Tag("OpenTelemetryTest")
23+
@Target({ TYPE, METHOD })
24+
@Retention(RetentionPolicy.RUNTIME)
25+
public @interface OpenTelemetryTest {
26+
}
27+

‎integration-tests/src/test/java/org/zowe/apiml/util/service/FullApiMediationLayer.java‎

Lines changed: 46 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,14 @@
2424
import java.util.Map;
2525
import java.util.Optional;
2626

27+
import static org.zowe.apiml.util.config.ConfigReader.IS_MODULITH_ENABLED;
28+
2729
//TODO this class doesn't lend itself well to switching of configurations.
2830
//attls is integrated in a kludgy way, and deserves a rewrite
2931

3032
@Slf4j
3133
public class FullApiMediationLayer {
34+
3235
private RunningService discoveryService;
3336
private RunningService gatewayService;
3437
private RunningService apiCatalogService;
@@ -52,14 +55,19 @@ public class FullApiMediationLayer {
5255
private FullApiMediationLayer() {
5356
env = ConfigReader.environmentConfiguration().getInstanceEnv();
5457

58+
if (IS_MODULITH_ENABLED) {
59+
prepareApiml();
60+
} else {
61+
prepareGateway();
62+
prepareDiscovery();
63+
prepareCaching();
64+
prepareZaas();
65+
}
66+
67+
prepareMockServices();
5568
prepareCatalog();
5669
prepareDiscoverableClient();
57-
prepareGateway();
58-
prepareMockServices();
59-
prepareDiscovery();
60-
prepareCaching();
61-
prepareZaas();
62-
prepareApiml();
70+
6371
if (!attlsEnabled) {
6472
prepareNodeJsSampleApp();
6573
}
@@ -130,25 +138,28 @@ private void prepareDiscoverableClient() {
130138

131139
public void start() {
132140
try {
133-
var discoveryEnv = new HashMap<>(env);
134-
discoveryEnv.put("ZWE_configs_port", "10011");
135-
discoveryService.startWithScript("discovery-package/src/main/resources/bin", discoveryEnv);
136-
var gatewayEnv = new HashMap<>(env);
137-
gatewayEnv.put("ZWE_configs_port", "10010");
138-
gatewayService.startWithScript("gateway-package/src/main/resources/bin", gatewayEnv);
139-
var catalogEnv = new HashMap<>(env);
140-
catalogEnv.put("ZWE_configs_port", "10014");
141-
apiCatalogService.startWithScript("api-catalog-package/src/main/resources/bin", catalogEnv);
142-
var cachingEnv = new HashMap<>(env);
143-
cachingEnv.put("ZWE_configs_port", "10016");
144-
cachingService.startWithScript("caching-service-package/src/main/resources/bin", cachingEnv);
145-
var zaasEnv = new HashMap<>(env);
146-
zaasEnv.put("ZWE_configs_port", "10023");
147-
zaasService.startWithScript("zaas-package/src/main/resources/bin", zaasEnv);
148-
var apimlModulithEnv = new HashMap<>(env);
149-
apimlModulithEnv.put("ZWE_configs_port", "10020");
150-
apimlModulithEnv.put("ZWE_configs_internal_discovery_port", "10021");
151-
apimlService.startWithScript("apiml-package/src/main/resources/bin", apimlModulithEnv);
141+
if (IS_MODULITH_ENABLED) {
142+
var apimlModulithEnv = new HashMap<>(env);
143+
apimlModulithEnv.put("ZWE_configs_port", "10010");
144+
apimlModulithEnv.put("ZWE_configs_internal_discovery_port", "10011");
145+
apimlService.startWithScript("apiml-package/src/main/resources/bin", apimlModulithEnv);
146+
} else {
147+
var discoveryEnv = new HashMap<>(env);
148+
discoveryEnv.put("ZWE_configs_port", "10011");
149+
discoveryService.startWithScript("discovery-package/src/main/resources/bin", discoveryEnv);
150+
var gatewayEnv = new HashMap<>(env);
151+
gatewayEnv.put("ZWE_configs_port", "10010");
152+
gatewayService.startWithScript("gateway-package/src/main/resources/bin", gatewayEnv);
153+
var catalogEnv = new HashMap<>(env);
154+
catalogEnv.put("ZWE_configs_port", "10014");
155+
apiCatalogService.startWithScript("api-catalog-package/src/main/resources/bin", catalogEnv);
156+
var cachingEnv = new HashMap<>(env);
157+
cachingEnv.put("ZWE_configs_port", "10016");
158+
cachingService.startWithScript("caching-service-package/src/main/resources/bin", cachingEnv);
159+
var zaasEnv = new HashMap<>(env);
160+
zaasEnv.put("ZWE_configs_port", "10023");
161+
zaasService.startWithScript("zaas-package/src/main/resources/bin", zaasEnv);
162+
}
152163

153164
if (!attlsEnabled) {
154165
nodeJsSampleApp = nodeJsBuilder.start();
@@ -176,18 +187,22 @@ private String formatEnv() {
176187

177188
public void stop() {
178189
try {
179-
discoveryService.stop();
180-
gatewayService.stop();
181190
mockZosmfService.stop();
182-
183-
apiCatalogService.stop();
184191
discoverableClientService.stop();
185192

186-
cachingService.stop();
187-
zaasService.stop();
188193
if (!attlsEnabled && startServices()) {
189194
nodeJsSampleApp.destroy();
190195
}
196+
197+
if (IS_MODULITH_ENABLED) {
198+
apimlService.stop();
199+
} else {
200+
discoveryService.stop();
201+
gatewayService.stop();
202+
apiCatalogService.stop();
203+
cachingService.stop();
204+
zaasService.stop();
205+
}
191206
} catch (Exception e) {
192207
e.printStackTrace();
193208
}

‎integration-tests/src/test/resources/environment-configuration.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,10 @@ instanceEnv:
105105
ZWE_configs_apiml_security_authorization_provider: endpoint
106106
# set the value to "authentication" if you want to test the sticky session load balancing
107107
APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: headerRequest
108+
OTEL_SDK_DISABLED: false
109+
OTEL_RESOURCE_ATTRIBUTES_DEPLOYMENT_ENVIRONMENT: dev
110+
OTEL_RESOURCE_ATTRIBUTES_SERVICE_NAME: apiml
111+
OTEL_RESOURCE_ATTRIBUTES_ZOS_SMF_ID: SYS1
112+
OTEL_RESOURCE_ATTRIBUTES_ZOS_SYSPLEX_NAME: SYSPLEX1
113+
OTEL_RESOURCE_ATTRIBUTES_MAINFRAME_LPAR_NAME: LPAR01
114+
OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:4318

0 commit comments

Comments
 (0)