From c39793ab20e82bac482800d9a4d89b764b1998f4 Mon Sep 17 00:00:00 2001 From: Patryk Wasielewski Date: Fri, 17 Apr 2026 13:14:51 +0200 Subject: [PATCH] feat: parametrize index and ingestion test suite --- docs/IntegrationTesting.md | 25 +++- ...dex_and_ingestion_separation_suite_test.go | 114 +++++++++++++----- test/testenv/util.go | 8 ++ 3 files changed, 114 insertions(+), 33 deletions(-) diff --git a/docs/IntegrationTesting.md b/docs/IntegrationTesting.md index 30b4bae05..6d2994b83 100644 --- a/docs/IntegrationTesting.md +++ b/docs/IntegrationTesting.md @@ -492,9 +492,30 @@ These tests require pre-populated cloud storage buckets with Splunk app tarballs Environment variables are defined in `test/env.sh`. -**10. _(Index/ingestion separation tests only)_ Provision AWS resources** +**10. _(Index/ingestion separation tests only)_ Configure AWS resources** -These tests require dedicated SQS queues and an S3 bucket in `us-west-2`. Resource names are hardcoded in the test suite file and the S3 bucket name is globally unique, so these tests can only be run by the SOK team against the team's AWS account. +These tests require SQS queues and an S3 bucket. By default, they use SOK-team–owned resources in `us-west-2`. To run against your own AWS account, set the following environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `TEST_SQS_QUEUE_NAME` | `index-ingest-separation-test-q` | SQS queue name | +| `TEST_SQS_DLQ_NAME` | `index-ingest-separation-test-dlq` | SQS dead letter queue name | +| `TEST_INGEST_S3_BUCKET` | `index-ingest-separation-test-bucket` | S3 bucket name | +| `TEST_INGEST_S3_PREFIX` | `smartbus-test` | S3 prefix/path within the bucket | +| `TEST_AWS_REGION` | `us-west-2` | AWS region for SQS and S3 | +| `TEST_SQS_ENDPOINT` | `https://sqs..amazonaws.com` | SQS endpoint (derived from region) | +| `TEST_S3_ENDPOINT` | `https://s3..amazonaws.com` | S3 endpoint (derived from region) | + +Example: + +```bash +export TEST_SQS_QUEUE_NAME=my-test-queue +export TEST_SQS_DLQ_NAME=my-test-dlq +export TEST_INGEST_S3_BUCKET=my-unique-bucket-name +export TEST_AWS_REGION=us-east-1 +``` + +If no variables are set, tests fall back to the default SOK-team resources (backward compatible). ### Run All Integration Tests via Makefile diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go index c25f3716e..958b8824d 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go @@ -33,41 +33,99 @@ const ( // ConsistentPollInterval is the interval to use to consistently check a state is stable ConsistentPollInterval = 200 * time.Millisecond ConsistentDuration = 2000 * time.Millisecond + + // Default values for AWS resources (used when environment variables are not set) + defaultSQSQueueName = "index-ingest-separation-test-q" + defaultSQSDLQName = "index-ingest-separation-test-dlq" + defaultS3Bucket = "index-ingest-separation-test-bucket" + defaultS3Prefix = "smartbus-test" + defaultAWSRegion = "us-west-2" +) + +// AWS resource configuration - populated in init() from environment variables +var ( + // Configurable AWS resources via environment variables + sqsQueueName string + sqsDLQName string + s3Bucket string + s3Prefix string + awsRegion string + sqsEndpoint string + s3Endpoint string ) var ( testenvInstance *testenv.TestEnv testSuiteName = "indingsep-" + testenv.RandomDNSName(3) + queue enterpriseApi.QueueSpec + objectStorage enterpriseApi.ObjectStorageSpec + + serviceAccountName = "index-ingest-sa" + + inputs []string + outputs []string + defaultsAll []string + defaultsIngest []string + awsEnvVars []string + inputsShouldNotContain []string + outputsShouldNotContain []string + + testDataS3Bucket = os.Getenv("TEST_BUCKET") + testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") + currDir, _ = os.Getwd() + downloadDirV1 = filepath.Join(currDir, "icappfwV1-"+testenv.RandomDNSName(4)) + appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) + s3TestDir = "icappfw-" + testenv.RandomDNSName(4) + appListV1 = testenv.BasicApps + s3AppDirV1 = testenv.AppLocationV1 +) + +func init() { + // Initialize AWS resource configuration from environment variables with defaults + sqsQueueName = testenv.GetEnvWithDefault("TEST_SQS_QUEUE_NAME", defaultSQSQueueName) + sqsDLQName = testenv.GetEnvWithDefault("TEST_SQS_DLQ_NAME", defaultSQSDLQName) + s3Bucket = testenv.GetEnvWithDefault("TEST_INGEST_S3_BUCKET", defaultS3Bucket) + s3Prefix = testenv.GetEnvWithDefault("TEST_INGEST_S3_PREFIX", defaultS3Prefix) + awsRegion = testenv.GetEnvWithDefault("TEST_AWS_REGION", defaultAWSRegion) + sqsEndpoint = testenv.GetEnvWithDefault("TEST_SQS_ENDPOINT", "https://sqs."+awsRegion+".amazonaws.com") + s3Endpoint = testenv.GetEnvWithDefault("TEST_S3_ENDPOINT", "https://s3."+awsRegion+".amazonaws.com") + + // Build queue spec queue = enterpriseApi.QueueSpec{ Provider: "sqs", SQS: enterpriseApi.SQSSpec{ - Name: "index-ingest-separation-test-q", - AuthRegion: "us-west-2", - Endpoint: "https://sqs.us-west-2.amazonaws.com", - DLQ: "index-ingest-separation-test-dlq", + Name: sqsQueueName, + AuthRegion: awsRegion, + Endpoint: sqsEndpoint, + DLQ: sqsDLQName, }, } + + // Build object storage spec objectStorage = enterpriseApi.ObjectStorageSpec{ Provider: "s3", S3: enterpriseApi.S3Spec{ - Endpoint: "https://s3.us-west-2.amazonaws.com", - Path: "index-ingest-separation-test-bucket/smartbus-test", + Endpoint: s3Endpoint, + Path: s3Bucket + "/" + s3Prefix, }, } - serviceAccountName = "index-ingest-sa" + // Build inputs configuration inputs = []string{ - "[remote_queue:index-ingest-separation-test-q]", + "[remote_queue:" + sqsQueueName + "]", "remote_queue.type = sqs_smartbus", - "remote_queue.sqs_smartbus.auth_region = us-west-2", - "remote_queue.sqs_smartbus.dead_letter_queue.name = index-ingest-separation-test-dlq", - "remote_queue.sqs_smartbus.endpoint = https://sqs.us-west-2.amazonaws.com", - "remote_queue.sqs_smartbus.large_message_store.endpoint = https://s3.us-west-2.amazonaws.com", - "remote_queue.sqs_smartbus.large_message_store.path = s3://index-ingest-separation-test-bucket/smartbus-test", + "remote_queue.sqs_smartbus.auth_region = " + awsRegion, + "remote_queue.sqs_smartbus.dead_letter_queue.name = " + sqsDLQName, + "remote_queue.sqs_smartbus.endpoint = " + sqsEndpoint, + "remote_queue.sqs_smartbus.large_message_store.endpoint = " + s3Endpoint, + "remote_queue.sqs_smartbus.large_message_store.path = s3://" + s3Bucket + "/" + s3Prefix, "remote_queue.sqs_smartbus.retry_policy = max_count", - "remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4"} - outputs = append(inputs, "remote_queue.sqs_smartbus.encoding_format = s2s", "remote_queue.sqs_smartbus.send_interval = 5s") + "remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4", + } + outputs = append(inputs, "remote_queue.sqs_smartbus.encoding_format = s2s", "remote_queue.sqs_smartbus.send_interval = 5s") + + // Build defaults configuration defaultsAll = []string{ "[pipeline:remotequeueruleset]\ndisabled = false", "[pipeline:ruleset]\ndisabled = true", @@ -77,31 +135,25 @@ var ( } defaultsIngest = append(defaultsAll, "[pipeline:indexerPipe]\ndisabled = true") + // Build AWS environment variables for pods awsEnvVars = []string{ - "AWS_REGION=us-west-2", - "AWS_DEFAULT_REGION=us-west-2", + "AWS_REGION=" + awsRegion, + "AWS_DEFAULT_REGION=" + awsRegion, "AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/eks.amazonaws.com/serviceaccount/token", "AWS_ROLE_ARN=arn:aws:iam::", "AWS_STS_REGIONAL_ENDPOINTS=regional", } + // Build negative test assertions inputsShouldNotContain = []string{ - "[remote_queue:index-ingest-separation-test-q]", - "remote_queue.sqs_smartbus.dead_letter_queue.name = index-ingest-separation-test-dlq", - "remote_queue.sqs_smartbus.large_message_store.path = s3://index-ingest-separation-test-bucket/smartbus-test", + "[remote_queue:" + sqsQueueName + "]", + "remote_queue.sqs_smartbus.dead_letter_queue.name = " + sqsDLQName, + "remote_queue.sqs_smartbus.large_message_store.path = s3://" + s3Bucket + "/" + s3Prefix, "remote_queue.sqs_smartbus.retry_policy = max_count", - "remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4"} + "remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4", + } outputsShouldNotContain = append(inputs, "remote_queue.sqs_smartbus.send_interval = 5s") - - testDataS3Bucket = os.Getenv("TEST_BUCKET") - testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "icappfwV1-"+testenv.RandomDNSName(4)) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - s3TestDir = "icappfw-" + testenv.RandomDNSName(4) - appListV1 = testenv.BasicApps - s3AppDirV1 = testenv.AppLocationV1 -) +} // TestBasic is the main entry point func TestBasic(t *testing.T) { diff --git a/test/testenv/util.go b/test/testenv/util.go index 8ef0c6a4e..3ebaeaea5 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -49,6 +49,14 @@ const ( sidecarName = "sok-debug" ) +// GetEnvWithDefault returns the environment variable value or the default if not set +func GetEnvWithDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + func init() { rand.Seed(time.Now().UnixNano()) opts := zap.Options{