telemetry: add OTLP traces pipeline

ansible/inventories/devnet-0/group_vars/all/all.yaml

@@ -276,15 +276,18 @@ docker_nginx_proxy_wildcard_cert: "{{ network_server_subdomain }}"
 docker_nginx_proxy_wildcard_cert_url: "http://cert.{{ network_server_subdomain }}/{{ network_server_subdomain }}-latest.tar.enc"
 docker_nginx_proxy_wildcard_cert_psk: "{{ secret_cert_encryption_psk }}"
 
-# OTLP egress to the prod analytics gateway (Vector ships container logs here).
+# OTLP egress to the prod analytics gateway (shared by Vector's logs + traces sinks).
 # Ingress identity (auth username + ingress_user tag) is the current devnet name
 # rather than secret_loki.username: the sops username drifts stale between devnet
 # iterations, while the gateway only validates the (constant) password — so using
-# the network name keeps log attribution correct no matter what sops holds.
+# the network name keeps log/trace attribution correct no matter what sops holds.
 otlp_endpoint: "https://otlp.analytics.production.platform.ethpandaops.io"
 otlp_deployment_env: production
 
 # role: ethpandaops.general.vector
+# Shared docker network so the clients can reach Vector's OTLP listeners and
+# Vector can read the other containers' logs.
+vector_container_networks: "{{ docker_networks_shared }}"
 vector_config: |
   # Docker container logs (clean per-container metadata straight from the Docker API)
   [sources.in]
@@ -298,6 +301,19 @@ vector_config: |
       "snooper-",
     ]
 
+  # OTLP traces pushed by the clients (beacon -> :4317 grpc, geth -> :4318 http).
+  # use_otlp_decoding preserves the resourceSpans envelope so the sink forwards
+  # natively with the client's batching intact (1 request in = 1 request out).
+  [sources.otlp_in]
+    type = "opentelemetry"
+    use_otlp_decoding = true
+
+  [sources.otlp_in.grpc]
+    address = "[::]:4317"
+
+  [sources.otlp_in.http]
+    address = "[::]:4318"
+
   # Shape docker_logs events into an OTLP resourceLogs envelope with full metadata.
   [transforms.otel_shape]
     type = "remap"
@@ -423,3 +439,18 @@ vector_config: |
       # max_events MUST be 1 — OTLP/HTTP allows one envelope per request.
       batch.max_events = 1
       batch.timeout_secs = 5
+
+  [sinks.otlp_traces]
+    type = "opentelemetry"
+    inputs = ["otlp_in.traces"]
+    [sinks.otlp_traces.protocol]
+      type = "http"
+      uri = "{{ otlp_endpoint }}/v1/traces"
+      method = "post"
+      encoding.codec = "otlp"
+      auth.strategy = "basic"
+      auth.user = "{{ ethereum_network_name }}"
+      auth.password = "{{ secret_loki.password }}"
+      # use_otlp_decoding => one event already carries the client's full span batch.
+      batch.max_events = 1
+      batch.timeout_secs = 5

ansible/inventories/devnet-0/group_vars/bootnode.yaml

@@ -104,6 +104,10 @@ geth_container_command_extra_args:
   - --syncmode=full
   - --gcmode=archive
   - --state.scheme=hash
+  # OTLP traces → local otelcol sidecar (handles upstream auth + endpoint).
+  - --rpc.telemetry
+  - --rpc.telemetry.endpoint=http://vector:4318/v1/traces
+  - --rpc.telemetry.instance-id={{ ethereum_network_name }}-{{ inventory_hostname }}
   - >-
     --bootnodes={{
       (

ansible/inventories/devnet-0/group_vars/geth.yaml

@@ -25,6 +25,11 @@ geth_container_command_extra_args:
   - --networkid={{ ethereum_network_id }}
   - --syncmode=full
   - --bootnodes={{ ethereum_el_bootnodes | join(',') }}
+  - --rpc.telemetry
+  - --rpc.telemetry.endpoint=http://vector:4318/v1/traces
+  - --rpc.telemetry.sample-ratio=1 # Required until geth ships the fix for the SampleRatio default https://github.com/ethereum/go-ethereum/pull/34948
+  - --rpc.telemetry.instance-id={{ ethereum_network_name }}-{{ inventory_hostname }}
+  - --rpc.telemetry.tags=execution_client={{ ethereum_node_el }},consensus_client={{ ethereum_node_cl }},supernode={{ ethereum_node_cl_supernode_enabled | bool | default(false) }},network={{ ethereum_network_name }},instance-id={{ ethereum_network_name }}-{{ inventory_hostname }}
 
 geth_container_pull: true
 

ansible/inventories/devnet-0/group_vars/grandine.yaml

@@ -43,6 +43,8 @@ grandine_container_command_extra_args:
   - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }}
   - --graffiti={{ ansible_hostname }}
   - --features=LogHttpRequests
+  - --telemetry-metrics-url=http://vector:4317 # despite the name this exports tracing spans (grandine #487); gRPC only
+  - --telemetry-service-name={{ ethereum_network_name }}-{{ inventory_hostname }}
 
 grandine_validator_container_volumes:
   - "{{ grandine_validator_datadir }}:/validator-data"

ansible/inventories/devnet-0/group_vars/lighthouse.yaml

@@ -38,6 +38,8 @@ lighthouse_container_command_extra_simple_args:
   - --testnet-dir=/network-config
   - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }}
   - --allow-insecure-genesis-sync
+  - --telemetry-collector-url=http://vector:4317
+  - --telemetry-service-name={{ ethereum_network_name }}-{{ inventory_hostname }}
 lighthouse_validator_container_volumes:
   - "{{ lighthouse_validator_datadir }}:/validator-data"
   - "{{ eth_testnet_config_dir }}:/network-config:ro"

ansible/inventories/devnet-0/group_vars/prysm.yaml

@@ -49,6 +49,10 @@ prysm_container_command_extra_simple_args:
   - --min-sync-peers=1
   - --verbosity=debug
   - --subscribe-all-subnets
+  - --enable-tracing
+  - --tracing-endpoint=http://vector:4318/v1/traces # the flag default is a stale Jaeger URL; always set explicitly
+  - --tracing-process-name={{ ethereum_network_name }}-{{ inventory_hostname }}
+  - --trace-sample-fraction=1.0 # default samples only 20%
 prysm_container_command_extra_bootnode_args: >-
   {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }}
 

ansible/inventories/devnet-0/group_vars/reth.yaml

@@ -14,6 +14,8 @@ reth_container_env:
   VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}"
   LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}"
   RUST_BACKTRACE: "full"
+  # reth hardcodes service.name=reth; the instance id has to come in via resource attributes
+  OTEL_RESOURCE_ATTRIBUTES: "service.instance.id={{ ethereum_network_name }}-{{ inventory_hostname }}"
 reth_container_volumes:
   - "{{ reth_datadir }}:/data"
   - "{{ reth_auth_jwt_path }}:/execution-auth.jwt:ro"
@@ -22,6 +24,9 @@ reth_container_command_extra_args:
   - --chain=/network-config/genesis.json
   - --bootnodes={{ ethereum_el_bootnodes | join(',') }}
   - --http.api=trace,rpc,eth,net,debug,web3,admin,txpool
+  - --tracing-otlp=http://vector:4318/v1/traces
+  - --tracing-otlp.filter=info # the default 'debug' produced 50GB of spans during a sync (reth #21608)
+  - --tracing-otlp.sample-ratio=1.0
 prometheus_config: |
   global:
     scrape_interval: 30s