Skip to content

[DRAFT] Feature/parametrize index and ingestion suite #1853

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
patrykw-splunk wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

[DRAFT] Feature/parametrize index and ingestion suite #1853

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions docs/IntegrationTesting.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,9 +492,30 @@ These tests require pre-populated cloud storage buckets with Splunk app tarballs

Environment variables are defined in `test/env.sh`.

**10. _(Index/ingestion separation tests only)_ Provision AWS resources**
**10. _(Index/ingestion separation tests only)_ Configure AWS resources**

These tests require dedicated SQS queues and an S3 bucket in `us-west-2`. Resource names are hardcoded in the test suite file and the S3 bucket name is globally unique, so these tests can only be run by the SOK team against the team's AWS account.
These tests require SQS queues and an S3 bucket. By default, they use SOK-team–owned resources in `us-west-2`. To run against your own AWS account, set the following environment variables:

| Variable | Default | Description |
|----------|---------|-------------|
| `TEST_SQS_QUEUE_NAME` | `index-ingest-separation-test-q` | SQS queue name |
| `TEST_SQS_DLQ_NAME` | `index-ingest-separation-test-dlq` | SQS dead letter queue name |
| `TEST_INGEST_S3_BUCKET` | `index-ingest-separation-test-bucket` | S3 bucket name |
| `TEST_INGEST_S3_PREFIX` | `smartbus-test` | S3 prefix/path within the bucket |
| `TEST_AWS_REGION` | `us-west-2` | AWS region for SQS and S3 |
| `TEST_SQS_ENDPOINT` | `https://sqs.<region>.amazonaws.com` | SQS endpoint (derived from region) |
| `TEST_S3_ENDPOINT` | `https://s3.<region>.amazonaws.com` | S3 endpoint (derived from region) |

Example:

```bash
export TEST_SQS_QUEUE_NAME=my-test-queue
export TEST_SQS_DLQ_NAME=my-test-dlq
export TEST_INGEST_S3_BUCKET=my-unique-bucket-name
export TEST_AWS_REGION=us-east-1
```

If no variables are set, tests fall back to the default SOK-team resources (backward compatible).

### Run All Integration Tests via Makefile

Expand Down
114 changes: 83 additions & 31 deletions test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,41 +33,99 @@ const (
// ConsistentPollInterval is the interval to use to consistently check a state is stable
ConsistentPollInterval = 200 * time.Millisecond
ConsistentDuration = 2000 * time.Millisecond

// Default values for AWS resources (used when environment variables are not set)
defaultSQSQueueName = "index-ingest-separation-test-q"
defaultSQSDLQName = "index-ingest-separation-test-dlq"
defaultS3Bucket = "index-ingest-separation-test-bucket"
defaultS3Prefix = "smartbus-test"
defaultAWSRegion = "us-west-2"
)

// AWS resource configuration - populated in init() from environment variables
var (
// Configurable AWS resources via environment variables
sqsQueueName string
sqsDLQName string
s3Bucket string
s3Prefix string
awsRegion string
sqsEndpoint string
s3Endpoint string
)

var (
testenvInstance *testenv.TestEnv
testSuiteName = "indingsep-" + testenv.RandomDNSName(3)

queue enterpriseApi.QueueSpec
objectStorage enterpriseApi.ObjectStorageSpec

serviceAccountName = "index-ingest-sa"

inputs []string
outputs []string
defaultsAll []string
defaultsIngest []string
awsEnvVars []string
inputsShouldNotContain []string
outputsShouldNotContain []string

testDataS3Bucket = os.Getenv("TEST_BUCKET")
testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET")
currDir, _ = os.Getwd()
downloadDirV1 = filepath.Join(currDir, "icappfwV1-"+testenv.RandomDNSName(4))
appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3)
s3TestDir = "icappfw-" + testenv.RandomDNSName(4)
appListV1 = testenv.BasicApps
s3AppDirV1 = testenv.AppLocationV1
)

func init() {
// Initialize AWS resource configuration from environment variables with defaults
sqsQueueName = testenv.GetEnvWithDefault("TEST_SQS_QUEUE_NAME", defaultSQSQueueName)
sqsDLQName = testenv.GetEnvWithDefault("TEST_SQS_DLQ_NAME", defaultSQSDLQName)
s3Bucket = testenv.GetEnvWithDefault("TEST_INGEST_S3_BUCKET", defaultS3Bucket)
s3Prefix = testenv.GetEnvWithDefault("TEST_INGEST_S3_PREFIX", defaultS3Prefix)
awsRegion = testenv.GetEnvWithDefault("TEST_AWS_REGION", defaultAWSRegion)
sqsEndpoint = testenv.GetEnvWithDefault("TEST_SQS_ENDPOINT", "https://sqs."+awsRegion+".amazonaws.com")
s3Endpoint = testenv.GetEnvWithDefault("TEST_S3_ENDPOINT", "https://s3."+awsRegion+".amazonaws.com")

// Build queue spec
queue = enterpriseApi.QueueSpec{
Provider: "sqs",
SQS: enterpriseApi.SQSSpec{
Name: "index-ingest-separation-test-q",
AuthRegion: "us-west-2",
Endpoint: "https://sqs.us-west-2.amazonaws.com",
DLQ: "index-ingest-separation-test-dlq",
Name: sqsQueueName,
AuthRegion: awsRegion,
Endpoint: sqsEndpoint,
DLQ: sqsDLQName,
},
}

// Build object storage spec
objectStorage = enterpriseApi.ObjectStorageSpec{
Provider: "s3",
S3: enterpriseApi.S3Spec{
Endpoint: "https://s3.us-west-2.amazonaws.com",
Path: "index-ingest-separation-test-bucket/smartbus-test",
Endpoint: s3Endpoint,
Path: s3Bucket + "/" + s3Prefix,
},
}
serviceAccountName = "index-ingest-sa"

// Build inputs configuration
inputs = []string{
"[remote_queue:index-ingest-separation-test-q]",
"[remote_queue:" + sqsQueueName + "]",
"remote_queue.type = sqs_smartbus",
"remote_queue.sqs_smartbus.auth_region = us-west-2",
"remote_queue.sqs_smartbus.dead_letter_queue.name = index-ingest-separation-test-dlq",
"remote_queue.sqs_smartbus.endpoint = https://sqs.us-west-2.amazonaws.com",
"remote_queue.sqs_smartbus.large_message_store.endpoint = https://s3.us-west-2.amazonaws.com",
"remote_queue.sqs_smartbus.large_message_store.path = s3://index-ingest-separation-test-bucket/smartbus-test",
"remote_queue.sqs_smartbus.auth_region = " + awsRegion,
"remote_queue.sqs_smartbus.dead_letter_queue.name = " + sqsDLQName,
"remote_queue.sqs_smartbus.endpoint = " + sqsEndpoint,
"remote_queue.sqs_smartbus.large_message_store.endpoint = " + s3Endpoint,
"remote_queue.sqs_smartbus.large_message_store.path = s3://" + s3Bucket + "/" + s3Prefix,
"remote_queue.sqs_smartbus.retry_policy = max_count",
"remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4"}
outputs = append(inputs, "remote_queue.sqs_smartbus.encoding_format = s2s", "remote_queue.sqs_smartbus.send_interval = 5s")
"remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4",
}
outputs = append(inputs, "remote_queue.sqs_smartbus.encoding_format = s2s", "remote_queue.sqs_smartbus.send_interval = 5s")

// Build defaults configuration
defaultsAll = []string{
"[pipeline:remotequeueruleset]\ndisabled = false",
"[pipeline:ruleset]\ndisabled = true",
Expand All @@ -77,31 +135,25 @@ var (
}
defaultsIngest = append(defaultsAll, "[pipeline:indexerPipe]\ndisabled = true")

// Build AWS environment variables for pods
awsEnvVars = []string{
"AWS_REGION=us-west-2",
"AWS_DEFAULT_REGION=us-west-2",
"AWS_REGION=" + awsRegion,
"AWS_DEFAULT_REGION=" + awsRegion,
"AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/eks.amazonaws.com/serviceaccount/token",
"AWS_ROLE_ARN=arn:aws:iam::",
"AWS_STS_REGIONAL_ENDPOINTS=regional",
}

// Build negative test assertions
inputsShouldNotContain = []string{
"[remote_queue:index-ingest-separation-test-q]",
"remote_queue.sqs_smartbus.dead_letter_queue.name = index-ingest-separation-test-dlq",
"remote_queue.sqs_smartbus.large_message_store.path = s3://index-ingest-separation-test-bucket/smartbus-test",
"[remote_queue:" + sqsQueueName + "]",
"remote_queue.sqs_smartbus.dead_letter_queue.name = " + sqsDLQName,
"remote_queue.sqs_smartbus.large_message_store.path = s3://" + s3Bucket + "/" + s3Prefix,
"remote_queue.sqs_smartbus.retry_policy = max_count",
"remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4"}
"remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4",
}
outputsShouldNotContain = append(inputs, "remote_queue.sqs_smartbus.send_interval = 5s")

testDataS3Bucket = os.Getenv("TEST_BUCKET")
testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET")
currDir, _ = os.Getwd()
downloadDirV1 = filepath.Join(currDir, "icappfwV1-"+testenv.RandomDNSName(4))
appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3)
s3TestDir = "icappfw-" + testenv.RandomDNSName(4)
appListV1 = testenv.BasicApps
s3AppDirV1 = testenv.AppLocationV1
)
}

// TestBasic is the main entry point
func TestBasic(t *testing.T) {
Expand Down
8 changes: 8 additions & 0 deletions test/testenv/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ const (
sidecarName = "sok-debug"
)

// GetEnvWithDefault returns the environment variable value or the default if not set
func GetEnvWithDefault(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}

func init() {
rand.Seed(time.Now().UnixNano())
opts := zap.Options{
Expand Down
Loading