diff --git a/artifacts/docs/README.md b/artifacts/docs/README.md index 8e68816d..ef34911f 100644 --- a/artifacts/docs/README.md +++ b/artifacts/docs/README.md @@ -16,7 +16,7 @@ Services: adminer, cloudflared, portainer, portainer-agent, postgraphile, postgr ### event-streaming -Services: debezium, debezium-postgres-connector, postgres, redpanda, redpanda-console, sqitch, traefik +Services: debezium, debezium-postgres-connector, gizmosql, postgres, redpanda, redpanda-connect, redpanda-console, sqitch, traefik ### recommendation @@ -73,6 +73,10 @@ You cannot access the search engine via a web interface. You cannot access the ip geolocator via a web interface. +### gizmosql + +You can query the event stream's parquet lakehouse using any Arrow Flight SQL client at gizmosql:31337. + ### grafana You can access the observation dashboard at [grafana.app.localhost](https://grafana.app.localhost/). @@ -142,6 +146,10 @@ You cannot access the caching system via a web interface. You can access the event streaming platform's ui as described under `redpanda-console`. +### redpanda-connect + +You can track the event stream ingestion using the pipeline's readiness endpoint. + ### redpanda-console You can access the event streaming platform's ui at [redpanda.app.localhost](https://redpanda.app.localhost/). diff --git a/src/development/.env.template b/src/development/.env.template index 546de5a7..45277401 100644 --- a/src/development/.env.template +++ b/src/development/.env.template @@ -1,4 +1,5 @@ RECCOOM_ADMIN_ACCOUNT_IDS= +S3_ENDPOINT_URL= TUSD_BUCKET=vibetype-images TUSD_ENDPOINT=http://minio:9000 TUSD_MAX_SIZE=10485760 diff --git a/src/development/gizmosql/compose.yaml b/src/development/gizmosql/compose.yaml new file mode 100644 index 00000000..661caf43 --- /dev/null +++ b/src/development/gizmosql/compose.yaml @@ -0,0 +1,56 @@ +secrets: + gizmosql-password: + # The gizmosql server's password. + file: ~~/artifacts/secrets/gizmosql-password.secret + s3-account-id: + # The event stream analytics' cloud storage account identifier. + file: ~~/artifacts/secrets/s3-account-id.secret + s3-aws-credentials-access-key: + # The event stream analytics' cloud storage secret access key. + file: ~~/artifacts/secrets/s3-aws-credentials-access-key.secret + s3-aws-credentials-access-key-id: + # The event stream analytics' cloud storage access key identifier. + file: ~~/artifacts/secrets/s3-aws-credentials-access-key-id.secret +services: + gizmosql: + # You can query the event stream's parquet lakehouse using Grafana or any Arrow Flight SQL client. + command: /run/entrypoint.sh + deploy: + labels: + - dargstack.profiles=event-streaming + entrypoint: sh + environment: + GIZMOSQL_USERNAME: gizmosql + INIT_SQL_COMMANDS_FILE: /tmp/gizmosql-init.sql + PRINT_QUERIES: "1" + TLS_ENABLED: "0" + healthcheck: + test: ["CMD-SHELL", "GIZMOSQL_PASSWORD=$$(cat /run/environment-variables/GIZMOSQL_PASSWORD) gizmosql_client --host 127.0.0.1 --port 31337 --username gizmosql --command 'SELECT 1'"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + image: gizmodata/gizmosql:v1.35.1 + secrets: + - source: gizmosql-password + target: /run/environment-variables/GIZMOSQL_PASSWORD + - source: s3-account-id + target: /run/environment-variables/S3_ACCOUNT_ID + - source: s3-aws-credentials-access-key-id + target: /run/environment-variables/S3_ACCESS_KEY_ID + - source: s3-aws-credentials-access-key + target: /run/environment-variables/S3_SECRET_ACCESS_KEY + volumes: + - ./configurations/entrypoint.sh:/run/entrypoint.sh:ro + - ./configurations/init.sql.template:/run/init.sql.template:ro +x-dargstack: + secrets: + gizmosql-password: + special_characters: false + type: random_string + s3-account-id: + type: third_party + s3-aws-credentials-access-key: + type: third_party + s3-aws-credentials-access-key-id: + type: third_party diff --git a/src/development/gizmosql/configurations/entrypoint.sh b/src/development/gizmosql/configurations/entrypoint.sh new file mode 100644 index 00000000..b19cef3f --- /dev/null +++ b/src/development/gizmosql/configurations/entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +# START of maevsi entrypoint script customization +ENVIRONMENT_VARIABLES_PATH="/run/environment-variables" + +is_valid_var_name() { + case "$1" in + ''|[!a-zA-Z_]*|*[!a-zA-Z0-9_]*) return 1 ;; + *) return 0 ;; + esac +} + +load_env_file() { + file="$1" + name=$(basename "$file") + is_valid_var_name "$name" || return 0 + value=$(cat "$file") + export "$name=$value" +} + +load_environment_variables() { + [ -d "$ENVIRONMENT_VARIABLES_PATH" ] || return 0 + set -- "$ENVIRONMENT_VARIABLES_PATH"/* + [ -e "$1" ] || return 0 + + for file in "$ENVIRONMENT_VARIABLES_PATH"/*; do + [ -f "$file" ] && load_env_file "$file" + done +} + +load_environment_variables +# END of maevsi entrypoint script customization + +# DuckDB in this image has no getenv() function, so the S3 values are rendered +# into init.sql. sed treats `&` and `\` specially in replacements, so escape +# them first (S3 keys are hex today, but this keeps arbitrary values safe). +escape_sed_replacement() { + printf '%s\n' "$1" | sed 's/[&\\]/\\&/g' +} + +S3_ACCOUNT_ID="$(escape_sed_replacement "${S3_ACCOUNT_ID}")" +S3_ACCESS_KEY_ID="$(escape_sed_replacement "${S3_ACCESS_KEY_ID}")" +S3_SECRET_ACCESS_KEY="$(escape_sed_replacement "${S3_SECRET_ACCESS_KEY}")" + +sed -e "s|__S3_ACCOUNT_ID__|${S3_ACCOUNT_ID}|g" \ + -e "s|__S3_ACCESS_KEY_ID__|${S3_ACCESS_KEY_ID}|g" \ + -e "s|__S3_SECRET_ACCESS_KEY__|${S3_SECRET_ACCESS_KEY}|g" \ + /run/init.sql.template > /tmp/gizmosql-init.sql + +exec /opt/gizmosql/scripts/start_gizmosql.sh diff --git a/src/development/gizmosql/configurations/init.sql.template b/src/development/gizmosql/configurations/init.sql.template new file mode 100644 index 00000000..674cd9b8 --- /dev/null +++ b/src/development/gizmosql/configurations/init.sql.template @@ -0,0 +1,38 @@ +INSTALL httpfs; +LOAD httpfs; + +CREATE SECRET s3_secret (TYPE r2, KEY_ID '__S3_ACCESS_KEY_ID__', SECRET '__S3_SECRET_ACCESS_KEY__', ACCOUNT_ID '__S3_ACCOUNT_ID__'); + +CREATE SCHEMA IF NOT EXISTS analytics; + +-- The event stream's parquet lakehouse, written by redpanda-connect to S3 +-- (bucket "test", prefix "events/"). The raw parquet columns are strings, so +-- ids and timestamps are cast to proper types (try_cast keeps bad values NULL). +CREATE OR REPLACE VIEW analytics.events AS +SELECT + try_cast(id AS UUID) AS id, + name, + slug, + try_cast(start AS TIMESTAMPTZ) AS start, + try_cast("end" AS TIMESTAMPTZ) AS "end", + visibility, + is_archived, + is_in_person, + is_remote, + language, + url, + try_cast(address_id AS UUID) AS address_id, + guest_count_maximum, + try_cast(created_at AS TIMESTAMPTZ) AS created_at, + try_cast(created_by AS UUID) AS created_by, + operation +FROM read_parquet('r2://test/events/*.parquet'); + +-- Example aggregate over the lakehouse. +CREATE OR REPLACE VIEW analytics.events_daily AS +SELECT + try_cast(created_at AS DATE) AS day, + count(*) AS events +FROM read_parquet('r2://test/events/*.parquet') +GROUP BY 1 +ORDER BY 1; diff --git a/src/development/redpanda-connect/compose.yaml b/src/development/redpanda-connect/compose.yaml new file mode 100644 index 00000000..a54594da --- /dev/null +++ b/src/development/redpanda-connect/compose.yaml @@ -0,0 +1,30 @@ +secrets: + s3-aws-credentials: + # The event stream ingestion's s3 credentials file. + file: ~~/artifacts/secrets/s3-aws-credentials.secret +services: + redpanda-connect: + # You can track the event stream's parquet ingestion using the pipeline's readiness endpoint. + deploy: + labels: + - dargstack.profiles=event-streaming + environment: + AWS_SHARED_CREDENTIALS_FILE: /run/secrets/s3-aws-credentials + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:4195/ready"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + image: redpandadata/connect:4.103.1 + secrets: + - s3-aws-credentials + volumes: + - ./configurations/config.yaml:/connect.yaml:ro +x-dargstack: + secrets: + s3-aws-credentials: + template: | + [default] + aws_access_key_id = {{secret:s3-aws-credentials-access-key-id}} + aws_secret_access_key = {{secret:s3-aws-credentials-access-key}} diff --git a/src/development/redpanda-connect/configurations/config.yaml b/src/development/redpanda-connect/configurations/config.yaml new file mode 100644 index 00000000..afe99800 --- /dev/null +++ b/src/development/redpanda-connect/configurations/config.yaml @@ -0,0 +1,77 @@ +input: + kafka_franz: + seed_brokers: ["redpanda:9092"] + topics: ["vibetype.vibetype.event"] + consumer_group: "rp_connect_s3_ingest" +pipeline: + processors: + # Tombstone messages (Debezium deletes) carry no payload and must be dropped. + - bloblang: | + root = if this.payload.after.or(this.after) == null { deleted() } + - mapping: | + root = this.payload.after.or(this.after) + root.operation = this.payload.op.or(this.op) +output: + aws_s3: + bucket: test + endpoint: ${S3_ENDPOINT_URL} + region: weur + force_path_style_urls: true + credentials: + profile: default + path: 'events/${! timestamp_unix_nano() }-${! uuid_v4() }.parquet' + batching: + count: 10000 + period: 15s + processors: + - parquet_encode: + schema: + - name: id + type: UTF8 + optional: true + - name: name + type: UTF8 + optional: true + - name: slug + type: UTF8 + optional: true + - name: start + type: UTF8 + optional: true + - name: end + type: UTF8 + optional: true + - name: visibility + type: UTF8 + optional: true + - name: is_archived + type: BOOLEAN + optional: true + - name: is_in_person + type: BOOLEAN + optional: true + - name: is_remote + type: BOOLEAN + optional: true + - name: language + type: UTF8 + optional: true + - name: url + type: UTF8 + optional: true + - name: address_id + type: UTF8 + optional: true + - name: guest_count_maximum + type: INT64 + optional: true + - name: created_at + type: UTF8 + optional: true + - name: created_by + type: UTF8 + optional: true + - name: operation + type: UTF8 + optional: true + default_compression: zstd diff --git a/src/production/gizmosql/compose.yaml b/src/production/gizmosql/compose.yaml new file mode 100644 index 00000000..0d63a3f1 --- /dev/null +++ b/src/production/gizmosql/compose.yaml @@ -0,0 +1,5 @@ +services: + gizmosql: + deploy: + update_config: + order: stop-first diff --git a/src/production/redpanda-connect/compose.yaml b/src/production/redpanda-connect/compose.yaml new file mode 100644 index 00000000..4364f20d --- /dev/null +++ b/src/production/redpanda-connect/compose.yaml @@ -0,0 +1,5 @@ +services: + redpanda-connect: + deploy: + update_config: + order: stop-first