From ffe33547ecd89196a632a3c51c6d6a52888b2662 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Mon, 6 Jul 2026 10:55:25 +1000 Subject: [PATCH 1/2] telemetry: add OTLP traces pipeline (Vector otlp_in -> OTLP sink) Ports the traces pipeline from ethpandaops/blob-devnets (devnet-0): Vector gains an otlp_in source (gRPC :4317, HTTP :4318) with use_otlp_decoding, plus an otlp_traces sink forwarding span batches to the prod analytics OTLP gateway. geth exports RPC spans via --rpc.telemetry, lighthouse via --telemetry-collector-url. Vector joins the shared docker network so clients can reach it by name. --- .../devnet-0/group_vars/all/all.yaml | 35 +++++++++++++++++-- .../devnet-0/group_vars/bootnode.yaml | 4 +++ .../inventories/devnet-0/group_vars/geth.yaml | 5 +++ .../devnet-0/group_vars/lighthouse.yaml | 2 ++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/ansible/inventories/devnet-0/group_vars/all/all.yaml b/ansible/inventories/devnet-0/group_vars/all/all.yaml index f9d33aa..61a5ebf 100644 --- a/ansible/inventories/devnet-0/group_vars/all/all.yaml +++ b/ansible/inventories/devnet-0/group_vars/all/all.yaml @@ -276,15 +276,18 @@ docker_nginx_proxy_wildcard_cert: "{{ network_server_subdomain }}" docker_nginx_proxy_wildcard_cert_url: "http://cert.{{ network_server_subdomain }}/{{ network_server_subdomain }}-latest.tar.enc" docker_nginx_proxy_wildcard_cert_psk: "{{ secret_cert_encryption_psk }}" -# OTLP egress to the prod analytics gateway (Vector ships container logs here). +# OTLP egress to the prod analytics gateway (shared by Vector's logs + traces sinks). # Ingress identity (auth username + ingress_user tag) is the current devnet name # rather than secret_loki.username: the sops username drifts stale between devnet # iterations, while the gateway only validates the (constant) password — so using -# the network name keeps log attribution correct no matter what sops holds. +# the network name keeps log/trace attribution correct no matter what sops holds. otlp_endpoint: "https://otlp.analytics.production.platform.ethpandaops.io" otlp_deployment_env: production # role: ethpandaops.general.vector +# Shared docker network so the clients can reach Vector's OTLP listeners and +# Vector can read the other containers' logs. +vector_container_networks: "{{ docker_networks_shared }}" vector_config: | # Docker container logs (clean per-container metadata straight from the Docker API) [sources.in] @@ -298,6 +301,19 @@ vector_config: | "snooper-", ] + # OTLP traces pushed by the clients (beacon -> :4317 grpc, geth -> :4318 http). + # use_otlp_decoding preserves the resourceSpans envelope so the sink forwards + # natively with the client's batching intact (1 request in = 1 request out). + [sources.otlp_in] + type = "opentelemetry" + use_otlp_decoding = true + + [sources.otlp_in.grpc] + address = "[::]:4317" + + [sources.otlp_in.http] + address = "[::]:4318" + # Shape docker_logs events into an OTLP resourceLogs envelope with full metadata. [transforms.otel_shape] type = "remap" @@ -423,3 +439,18 @@ vector_config: | # max_events MUST be 1 — OTLP/HTTP allows one envelope per request. batch.max_events = 1 batch.timeout_secs = 5 + + [sinks.otlp_traces] + type = "opentelemetry" + inputs = ["otlp_in.traces"] + [sinks.otlp_traces.protocol] + type = "http" + uri = "{{ otlp_endpoint }}/v1/traces" + method = "post" + encoding.codec = "otlp" + auth.strategy = "basic" + auth.user = "{{ ethereum_network_name }}" + auth.password = "{{ secret_loki.password }}" + # use_otlp_decoding => one event already carries the client's full span batch. + batch.max_events = 1 + batch.timeout_secs = 5 diff --git a/ansible/inventories/devnet-0/group_vars/bootnode.yaml b/ansible/inventories/devnet-0/group_vars/bootnode.yaml index 3d3cdae..7f3e6dc 100644 --- a/ansible/inventories/devnet-0/group_vars/bootnode.yaml +++ b/ansible/inventories/devnet-0/group_vars/bootnode.yaml @@ -104,6 +104,10 @@ geth_container_command_extra_args: - --syncmode=full - --gcmode=archive - --state.scheme=hash + # OTLP traces → local otelcol sidecar (handles upstream auth + endpoint). + - --rpc.telemetry + - --rpc.telemetry.endpoint=http://vector:4318/v1/traces + - --rpc.telemetry.instance-id={{ ethereum_network_name }}-{{ inventory_hostname }} - >- --bootnodes={{ ( diff --git a/ansible/inventories/devnet-0/group_vars/geth.yaml b/ansible/inventories/devnet-0/group_vars/geth.yaml index 7d46a8a..9da7888 100644 --- a/ansible/inventories/devnet-0/group_vars/geth.yaml +++ b/ansible/inventories/devnet-0/group_vars/geth.yaml @@ -25,6 +25,11 @@ geth_container_command_extra_args: - --networkid={{ ethereum_network_id }} - --syncmode=full - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --rpc.telemetry + - --rpc.telemetry.endpoint=http://vector:4318/v1/traces + - --rpc.telemetry.sample-ratio=1 # Required until geth ships the fix for the SampleRatio default https://github.com/ethereum/go-ethereum/pull/34948 + - --rpc.telemetry.instance-id={{ ethereum_network_name }}-{{ inventory_hostname }} + - --rpc.telemetry.tags=execution_client={{ ethereum_node_el }},consensus_client={{ ethereum_node_cl }},supernode={{ ethereum_node_cl_supernode_enabled | bool | default(false) }},network={{ ethereum_network_name }},instance-id={{ ethereum_network_name }}-{{ inventory_hostname }} geth_container_pull: true diff --git a/ansible/inventories/devnet-0/group_vars/lighthouse.yaml b/ansible/inventories/devnet-0/group_vars/lighthouse.yaml index e51fe69..08dd440 100644 --- a/ansible/inventories/devnet-0/group_vars/lighthouse.yaml +++ b/ansible/inventories/devnet-0/group_vars/lighthouse.yaml @@ -38,6 +38,8 @@ lighthouse_container_command_extra_simple_args: - --testnet-dir=/network-config - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }} - --allow-insecure-genesis-sync + - --telemetry-collector-url=http://vector:4317 + - --telemetry-service-name={{ ethereum_network_name }}-{{ inventory_hostname }} lighthouse_validator_container_volumes: - "{{ lighthouse_validator_datadir }}:/validator-data" - "{{ eth_testnet_config_dir }}:/network-config:ro" From 8f68d188069d56870a8dcfd79e4b8bae36b383db Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Mon, 6 Jul 2026 12:58:37 +1000 Subject: [PATCH 2/2] telemetry: wire reth/prysm/grandine OTLP traces --- ansible/inventories/devnet-0/group_vars/grandine.yaml | 2 ++ ansible/inventories/devnet-0/group_vars/prysm.yaml | 4 ++++ ansible/inventories/devnet-0/group_vars/reth.yaml | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/ansible/inventories/devnet-0/group_vars/grandine.yaml b/ansible/inventories/devnet-0/group_vars/grandine.yaml index 23e6844..f91c8d3 100644 --- a/ansible/inventories/devnet-0/group_vars/grandine.yaml +++ b/ansible/inventories/devnet-0/group_vars/grandine.yaml @@ -43,6 +43,8 @@ grandine_container_command_extra_args: - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }} - --graffiti={{ ansible_hostname }} - --features=LogHttpRequests + - --telemetry-metrics-url=http://vector:4317 # despite the name this exports tracing spans (grandine #487); gRPC only + - --telemetry-service-name={{ ethereum_network_name }}-{{ inventory_hostname }} grandine_validator_container_volumes: - "{{ grandine_validator_datadir }}:/validator-data" diff --git a/ansible/inventories/devnet-0/group_vars/prysm.yaml b/ansible/inventories/devnet-0/group_vars/prysm.yaml index bbc86ea..19ccede 100644 --- a/ansible/inventories/devnet-0/group_vars/prysm.yaml +++ b/ansible/inventories/devnet-0/group_vars/prysm.yaml @@ -49,6 +49,10 @@ prysm_container_command_extra_simple_args: - --min-sync-peers=1 - --verbosity=debug - --subscribe-all-subnets + - --enable-tracing + - --tracing-endpoint=http://vector:4318/v1/traces # the flag default is a stale Jaeger URL; always set explicitly + - --tracing-process-name={{ ethereum_network_name }}-{{ inventory_hostname }} + - --trace-sample-fraction=1.0 # default samples only 20% prysm_container_command_extra_bootnode_args: >- {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }} diff --git a/ansible/inventories/devnet-0/group_vars/reth.yaml b/ansible/inventories/devnet-0/group_vars/reth.yaml index 642621b..1d56c05 100644 --- a/ansible/inventories/devnet-0/group_vars/reth.yaml +++ b/ansible/inventories/devnet-0/group_vars/reth.yaml @@ -14,6 +14,8 @@ reth_container_env: VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" RUST_BACKTRACE: "full" + # reth hardcodes service.name=reth; the instance id has to come in via resource attributes + OTEL_RESOURCE_ATTRIBUTES: "service.instance.id={{ ethereum_network_name }}-{{ inventory_hostname }}" reth_container_volumes: - "{{ reth_datadir }}:/data" - "{{ reth_auth_jwt_path }}:/execution-auth.jwt:ro" @@ -22,6 +24,9 @@ reth_container_command_extra_args: - --chain=/network-config/genesis.json - --bootnodes={{ ethereum_el_bootnodes | join(',') }} - --http.api=trace,rpc,eth,net,debug,web3,admin,txpool + - --tracing-otlp=http://vector:4318/v1/traces + - --tracing-otlp.filter=info # the default 'debug' produced 50GB of spans during a sync (reth #21608) + - --tracing-otlp.sample-ratio=1.0 prometheus_config: | global: scrape_interval: 30s