diff --git a/compose/DEV.env b/compose/DEV.env index 278531a..d5f3506 100644 --- a/compose/DEV.env +++ b/compose/DEV.env @@ -56,10 +56,10 @@ DATA_LAKE_KAFKA_NETWORK=14.1.4.0/24 DATA_LAKE_REDIS_NETWORK=14.1.5.0/24 DATA_LAKE_MONGO_DB_NETWORK=14.1.6.0/24 -OBSERVABILITY_ELASTICSEARCH_VERSION=6.8.2 -OBSERVABILITY_LOGSTASH_VERSION=6.8.2 -OBSERVABILITY_KIBANA_VERSION=6.8.2 -OBSERVABILITY_APM_VERSION=6.8.2 +OBSERVABILITY_ELASTICSEARCH_VERSION=9.2.3 +OBSERVABILITY_LOGSTASH_VERSION=9.2.3 +OBSERVABILITY_KIBANA_VERSION=9.2.3 +OBSERVABILITY_APM_VERSION=9.2.3 OBSERVABILITY_ELK_NETWORK=14.1.9.0/24 OBSERVABILITY_ELK_ELASTICSEARCH_PORT=9200 OBSERVABILITY_ELK_LOGSTASH_PORT=5044 diff --git a/compose/QAA.env b/compose/QAA.env index 68ddd58..53ba159 100644 --- a/compose/QAA.env +++ b/compose/QAA.env @@ -56,10 +56,10 @@ DATA_LAKE_KAFKA_NETWORK=15.1.4.0/24 DATA_LAKE_REDIS_NETWORK=15.1.5.0/24 DATA_LAKE_MONGO_DB_NETWORK=15.1.6.0/24 -OBSERVABILITY_ELASTICSEARCH_VERSION=6.8.2 -OBSERVABILITY_LOGSTASH_VERSION=6.8.2 -OBSERVABILITY_KIBANA_VERSION=6.8.2 -OBSERVABILITY_APM_VERSION=6.8.2 +OBSERVABILITY_ELASTICSEARCH_VERSION=9.2.3 +OBSERVABILITY_LOGSTASH_VERSION=9.2.3 +OBSERVABILITY_KIBANA_VERSION=9.2.3 +OBSERVABILITY_APM_VERSION=9.2.3 OBSERVABILITY_ELK_NETWORK=15.1.9.0/24 OBSERVABILITY_ELK_ELASTICSEARCH_PORT=9200 OBSERVABILITY_ELK_LOGSTASH_PORT=5044 diff --git a/compose/QAB.env b/compose/QAB.env index 57a94f7..a97afa6 100644 --- a/compose/QAB.env +++ b/compose/QAB.env @@ -56,10 +56,10 @@ DATA_LAKE_KAFKA_NETWORK=16.1.4.0/24 DATA_LAKE_REDIS_NETWORK=16.1.5.0/24 DATA_LAKE_MONGO_DB_NETWORK=16.1.6.0/24 -OBSERVABILITY_ELASTICSEARCH_VERSION=6.8.2 -OBSERVABILITY_LOGSTASH_VERSION=6.8.2 -OBSERVABILITY_KIBANA_VERSION=6.8.2 -OBSERVABILITY_APM_VERSION=6.8.2 +OBSERVABILITY_ELASTICSEARCH_VERSION=9.2.3 +OBSERVABILITY_LOGSTASH_VERSION=9.2.3 +OBSERVABILITY_KIBANA_VERSION=9.2.3 +OBSERVABILITY_APM_VERSION=9.2.3 OBSERVABILITY_ELK_NETWORK=16.1.9.0/24 OBSERVABILITY_ELK_ELASTICSEARCH_PORT=9200 OBSERVABILITY_ELK_LOGSTASH_PORT=5044 diff --git a/compose/observability/elk.yml b/compose/observability/elk.yml index 40451e5..6423780 100644 --- a/compose/observability/elk.yml +++ b/compose/observability/elk.yml @@ -2,7 +2,7 @@ version: '3.8' services: elasticsearch: image: sloopstash/elasticsearch:v${OBSERVABILITY_ELASTICSEARCH_VERSION} - entrypoint: /usr/bin/supervisord + entrypoint: /usr/local/bin/supervisord command: "-c /etc/supervisord.conf" privileged: true ports: @@ -19,7 +19,7 @@ services: - common logstash: image: sloopstash/logstash:v${OBSERVABILITY_LOGSTASH_VERSION} - entrypoint: /usr/bin/supervisord + entrypoint: /usr/local/bin/supervisord command: "-c /etc/supervisord.conf" ports: - "${OBSERVABILITY_ELK_LOGSTASH_PORT}:5044" @@ -38,7 +38,7 @@ services: - common kibana: image: sloopstash/kibana:v${OBSERVABILITY_KIBANA_VERSION} - entrypoint: /usr/bin/supervisord + entrypoint: /usr/local/bin/supervisord command: "-c /etc/supervisord.conf" ports: - "${OBSERVABILITY_ELK_KIBANA_PORT}:5601" @@ -54,7 +54,7 @@ services: - common apm: image: sloopstash/apm:v${OBSERVABILITY_APM_VERSION} - entrypoint: /usr/bin/supervisord + entrypoint: /usr/local/bin/supervisord command: "-c /etc/supervisord.conf" ports: - "${OBSERVABILITY_ELK_APM_PORT}:8200" diff --git a/image/apm/9.2.3/alma-linux-9.dockerfile b/image/apm/9.2.3/alma-linux-9.dockerfile new file mode 100644 index 0000000..14959eb --- /dev/null +++ b/image/apm/9.2.3/alma-linux-9.dockerfile @@ -0,0 +1,56 @@ +# Docker image to use. +FROM sloopstash/alma-linux-9:v1.1.1 AS install_system_packages + +# Install system packages. +RUN set -x \ + && dnf install -y perl-Digest-SHA \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +# Intermediate Docker image to use. +FROM --platform=linux/amd64 install_system_packages AS install_apm_amd64 + +# Install APM. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/apm-server/apm-server-9.2.3-linux-x86_64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/apm-server/apm-server-9.2.3-linux-x86_64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c apm-server-9.2.3-linux-x86_64.tar.gz.sha512 \ + && tar xvzf apm-server-9.2.3-linux-x86_64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/apm \ + && cp -r apm-server-9.2.3-linux-x86_64/* /usr/local/lib/apm/ \ + && rm -rf apm-server-9.2.3* + +# Intermediate Docker image to use. +FROM --platform=linux/arm64 install_system_packages AS install_apm_arm64 + +# Install APM. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/apm-server/apm-server-9.2.3-linux-arm64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/apm-server/apm-server-9.2.3-linux-arm64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c apm-server-9.2.3-linux-arm64.tar.gz.sha512 \ + && tar xvzf apm-server-9.2.3-linux-arm64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/apm \ + && cp -r apm-server-9.2.3-linux-arm64/* /usr/local/lib/apm/ \ + && rm -rf apm-server-9.2.3* + +# Intermediate Docker image to use. +FROM install_apm_${TARGETARCH} + +# Create APM directories. +RUN set -x \ + && mkdir /opt/apm \ + && mkdir /opt/apm/data \ + && mkdir /opt/apm/log \ + && mkdir /opt/apm/conf \ + && mkdir /opt/apm/script \ + && mkdir /opt/apm/system \ + && touch /opt/apm/system/server.pid \ + && touch /opt/apm/system/supervisor.ini \ + && ln -sf /opt/apm/conf/server.yml /usr/local/lib/apm/apm-server.yml \ + && ln -s /opt/apm/system/supervisor.ini /etc/supervisord.d/apm.ini \ + && history -c + +# Set default work directory. +WORKDIR /opt/apm diff --git a/image/apm/9.2.3/context/.gitignore b/image/apm/9.2.3/context/.gitignore new file mode 100644 index 0000000..409b213 --- /dev/null +++ b/image/apm/9.2.3/context/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory. +* +# Except this file. +!.gitignore diff --git a/image/elasticsearch/9.2.3/alma-linux-9.dockerfile b/image/elasticsearch/9.2.3/alma-linux-9.dockerfile new file mode 100644 index 0000000..739f5a1 --- /dev/null +++ b/image/elasticsearch/9.2.3/alma-linux-9.dockerfile @@ -0,0 +1,63 @@ +# Docker image to use. +FROM sloopstash/alma-linux-9:v1.1.1 AS install_system_packages + +# Install system packages. +RUN set -x \ + && dnf install -y perl-Digest-SHA \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +# Intermediate Docker image to use. +FROM --platform=linux/amd64 install_system_packages AS install_elasticsearch_amd64 + +# Install Elasticsearch. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-9.2.3-linux-x86_64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-9.2.3-linux-x86_64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c elasticsearch-9.2.3-linux-x86_64.tar.gz.sha512 \ + && tar -xzf elasticsearch-9.2.3-linux-x86_64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/elasticsearch \ + && cp -r elasticsearch-9.2.3/* /usr/local/lib/elasticsearch/ \ + && rm -rf elasticsearch-9.2.3* + +# Intermediate Docker image to use. +FROM --platform=linux/arm64 install_system_packages AS install_elasticsearch_arm64 + +# Install Elasticsearch. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-9.2.3-linux-aarch64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-9.2.3-linux-aarch64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c elasticsearch-9.2.3-linux-aarch64.tar.gz.sha512 \ + && tar xvzf elasticsearch-9.2.3-linux-aarch64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/elasticsearch \ + && cp -r elasticsearch-9.2.3/* /usr/local/lib/elasticsearch/ \ + && rm -rf elasticsearch-9.2.3* + +# Intermediate Docker image to use. +FROM install_elasticsearch_${TARGETARCH} + +# Create system user for Elasticsearch. +RUN useradd -m elasticsearch + +# Create Elasticsearch directories. +RUN set -x \ + && mkdir /opt/elasticsearch \ + && mkdir /opt/elasticsearch/data \ + && mkdir /opt/elasticsearch/log \ + && mkdir /opt/elasticsearch/conf \ + && mkdir /opt/elasticsearch/script \ + && mkdir /opt/elasticsearch/system \ + && touch /opt/elasticsearch/system/server.pid \ + && touch /opt/elasticsearch/system/supervisor.ini \ + && ln -sf /opt/elasticsearch/conf/server.yml /usr/local/lib/elasticsearch/config/elasticsearch.yml \ + && ln -sf /opt/elasticsearch/conf/jvm.options /usr/local/lib/elasticsearch/config/jvm.options \ + && ln -s /opt/elasticsearch/system/security-limit.conf /etc/security/limits.d/elasticsearch.conf \ + && ln -s /opt/elasticsearch/system/supervisor.ini /etc/supervisord.d/elasticsearch.ini \ + && chown -R elasticsearch:elasticsearch /usr/local/lib/elasticsearch \ + && chown -R elasticsearch:elasticsearch /opt/elasticsearch \ + && history -c + +# Set default work directory. +WORKDIR /opt/elasticsearch diff --git a/image/elasticsearch/9.2.3/context/.gitignore b/image/elasticsearch/9.2.3/context/.gitignore new file mode 100644 index 0000000..e33102e --- /dev/null +++ b/image/elasticsearch/9.2.3/context/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory. +* +# Except this file. +!.gitignore + diff --git a/image/kibana/9.2.3/alma-linux-9.dockerfile b/image/kibana/9.2.3/alma-linux-9.dockerfile new file mode 100644 index 0000000..e4e9f80 --- /dev/null +++ b/image/kibana/9.2.3/alma-linux-9.dockerfile @@ -0,0 +1,55 @@ +# Docker image to use. +FROM sloopstash/alma-linux-9:v1.1.1 AS install_system_packages + +# Install system packages. +RUN set -x \ + && dnf install -y perl-Digest-SHA \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +# Intermediate Docker image to use. +FROM --platform=linux/amd64 install_system_packages AS install_kibana_amd64 + +# Install Kibana. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/kibana/kibana-9.2.3-linux-x86_64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/kibana/kibana-9.2.3-linux-x86_64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c kibana-9.2.3-linux-x86_64.tar.gz.sha512 \ + && tar xvzf kibana-9.2.3-linux-x86_64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/kibana \ + && cp -r kibana-9.2.3/* /usr/local/lib/kibana/ \ + && rm -rf kibana-9.2.3* + +# Intermediate Docker image to use. +FROM --platform=linux/arm64 install_system_packages AS install_kibana_arm64 + +# Install Kibana. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/kibana/kibana-9.2.3-linux-aarch64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/kibana/kibana-9.2.3-linux-aarch64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c kibana-9.2.3-linux-aarch64.tar.gz.sha512 \ + && tar xvzf kibana-9.2.3-linux-aarch64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/kibana \ + && cp -r kibana-9.2.3/* /usr/local/lib/kibana/ \ + && rm -rf kibana-9.2.3* + +# Intermediate Docker image to use. +FROM install_kibana_${TARGETARCH} + +# Create Kibana directories. +RUN set -x \ + && mkdir /opt/kibana \ + && mkdir /opt/kibana/data \ + && mkdir /opt/kibana/log \ + && mkdir /opt/kibana/conf \ + && mkdir /opt/kibana/script \ + && mkdir /opt/kibana/system \ + && touch /opt/kibana/system/server.pid \ + && touch /opt/kibana/system/supervisor.ini \ + && ln -s /opt/kibana/system/supervisor.ini /etc/supervisord.d/kibana.ini \ + && history -c + +# Set default work directory. +WORKDIR /opt/kibana diff --git a/image/kibana/9.2.3/context/.gitignore b/image/kibana/9.2.3/context/.gitignore new file mode 100644 index 0000000..409b213 --- /dev/null +++ b/image/kibana/9.2.3/context/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory. +* +# Except this file. +!.gitignore diff --git a/image/logstash/9.2.3/alma-linux-9.dockerfile b/image/logstash/9.2.3/alma-linux-9.dockerfile new file mode 100644 index 0000000..f60ac84 --- /dev/null +++ b/image/logstash/9.2.3/alma-linux-9.dockerfile @@ -0,0 +1,63 @@ +# Docker image to use. +FROM sloopstash/alma-linux-9:v1.1.1 AS install_system_packages + +# Install system packages. +RUN set -x \ + && dnf install -y perl-Digest-SHA \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +# Intermediate Docker image to use. +FROM --platform=linux/amd64 install_system_packages AS install_logstash_amd64 + +# Install Logstash. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/logstash/logstash-9.2.3-linux-x86_64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/logstash/logstash-9.2.3-linux-x86_64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c logstash-9.2.3-linux-x86_64.tar.gz.sha512 \ + && tar xvzf logstash-9.2.3-linux-x86_64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/logstash \ + && cp -r logstash-9.2.3/* /usr/local/lib/logstash/ \ + && rm -rf logstash-9.2.3* + +# Intermediate Docker image to use. +FROM --platform=linux/arm64 install_system_packages AS install_logstash_arm64 + +# Install Logstash. +WORKDIR /tmp +RUN set -x \ + && wget https://artifacts.elastic.co/downloads/logstash/logstash-9.2.3-linux-aarch64.tar.gz --quiet \ + && wget https://artifacts.elastic.co/downloads/logstash/logstash-9.2.3-linux-aarch64.tar.gz.sha512 --quiet \ + && shasum -a 512 -c logstash-9.2.3-linux-aarch64.tar.gz.sha512 \ + && tar xvzf logstash-9.2.3-linux-aarch64.tar.gz > /dev/null \ + && mkdir /usr/local/lib/logstash \ + && cp -r logstash-9.2.3/* /usr/local/lib/logstash/ \ + && rm -rf logstash-9.2.3* + +# Intermediate Docker image to use. +FROM install_logstash_${TARGETARCH} + +# Create system user for Elasticsearch. +RUN useradd -m logstash + +# Create Logstash directories. +RUN set -x \ + && mkdir /opt/logstash \ + && mkdir /opt/logstash/data \ + && mkdir /opt/logstash/log \ + && mkdir /opt/logstash/conf \ + && mkdir /opt/logstash/script \ + && mkdir /opt/logstash/system \ + && touch /opt/logstash/system/server.pid \ + && touch /opt/logstash/system/supervisor.ini \ + && ln -sf /opt/logstash/conf/server.yml /usr/local/lib/logstash/config/logstash.yml \ + && ln -sf /opt/logstash/conf/jvm.options /usr/local/lib/logstash/config/jvm.options \ + && ln -sf /opt/logstash/conf/pipelines.yml /usr/local/lib/logstash/config/pipelines.yml \ + && ln -s /opt/logstash/system/supervisor.ini /etc/supervisord.d/logstash.ini \ + && chown -R logstash:logstash /usr/local/lib/logstash \ + && chown -R logstash:logstash /opt/logstash \ + && history -c + +# Set default work directory. +WORKDIR /opt/logstash diff --git a/image/logstash/9.2.3/context/.gitignore b/image/logstash/9.2.3/context/.gitignore new file mode 100644 index 0000000..409b213 --- /dev/null +++ b/image/logstash/9.2.3/context/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory. +* +# Except this file. +!.gitignore diff --git a/workload/apm/9.2.3/conf/server.yml b/workload/apm/9.2.3/conf/server.yml new file mode 100644 index 0000000..8c2db10 --- /dev/null +++ b/workload/apm/9.2.3/conf/server.yml @@ -0,0 +1,965 @@ +######################### APM Server Configuration ######################### + +################################ APM Server ################################ + +apm-server: + # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. + host: "0.0.0.0:8200" + + # Agent authorization configuration. If no methods are defined, all requests will be allowed. + #auth: + # Agent authorization using Elasticsearch API Keys. + #api_key: + #enabled: false + # + # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different + # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch. + #limit: 100 + + # Define a shared secret token for authorizing agents using the "Bearer" authorization method. + #secret_token: + + # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow + # limited access for untrusted agents, such as Real User Monitoring. + #anonymous: + # By default anonymous auth is automatically enabled when either auth.api_key or + # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is + # disabled by default. + # + # When anonymous auth is enabled, only agents matching allow_agent and services + # matching allow_service are allowed. See below for details on default values for + # allow_agent. + #enabled: + + # Allow anonymous access only for specified agents. + #allow_agent: [rum-js, js-base] + + # Allow anonymous access only for specified service names. By default, all service names are allowed. + #allow_service: [] + + # Rate-limit anonymous access by IP and number of events. + #rate_limit: + # Rate limiting is defined per unique client IP address, for a limited number of IP addresses. + # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000. + #ip_limit: 1000 + + # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall + # maximum event throughput for anonymous access is (event_limit * ip_limit). + #event_limit: 300 + + # Maximum permitted size in bytes of a request's header accepted by the server to be processed. + #max_header_size: 1048576 + + # Maximum amount of time to wait for the next incoming request before underlying connection is closed. + #idle_timeout: 45s + + # Maximum permitted duration for reading an entire request. + #read_timeout: 30s + + # Maximum permitted duration for writing a response. + #write_timeout: 30s + + # Maximum duration before releasing resources when shutting down the server. + #shutdown_timeout: 30s + + # Maximum permitted size in bytes of an event accepted by the server to be processed. + #max_event_size: 307200 + + # Maximum number of new connections to accept simultaneously (0 means unlimited). + #max_connections: 0 + + # Custom HTTP headers to add to all HTTP responses, e.g. for security policy compliance. + #response_headers: + # X-My-Header: Contents of the header + + # If true (default), APM Server captures the IP of the instrumented service + # or the IP and User Agent of the real user (RUM requests). + #capture_personal_data: true + + # If specified, APM Server will record this value in events which have no service environment + # defined, and add it to agent configuration queries to Kibana when none is specified in the + # request from the agent. + #default_service_environment: + + # All events will be recorded in this data stream namespace when not managed by fleet. + # data_streams.namespace: default + + # Enable APM Server Golang expvar support (https://golang.org/pkg/expvar/). + #expvar: + #enabled: false + + # Url to expose expvar. + #url: "/debug/vars" + + + #---------------------------- APM Server - Secure Communication with Agents ---------------------------- + + # Enable secure communication between APM agents and the server. By default ssl is disabled. + #ssl: + #enabled: false + + # Path to file containing the certificate for server authentication. + # Needs to be configured when ssl is enabled. + #certificate: '' + + # Path to file containing server certificate key. + # Needs to be configured when ssl is enabled. + #key: '' + + # Optional configuration options for ssl communication. + + # Passphrase for decrypting the Certificate Key. + # It is recommended to use the provided keystore instead of entering the passphrase in plain text. + #key_passphrase: '' + + # List of supported/valid protocol versions. By default TLS versions 1.2 up to 1.3 are enabled. + #supported_protocols: [TLSv1.2, TLSv1.3] + + # Configure cipher suites to be used for SSL connections. + # Note that cipher suites are not configurable for TLS 1.3. + #cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #curve_types: [] + + #---------------------------- APM Server - RUM Real User Monitoring ---------------------------- + + # Enable Real User Monitoring (RUM) Support. By default RUM is disabled. + # RUM does not support token based authorization. Enabled RUM endpoints will not require any authorization + # token configured for other endpoints. + #rum: + #enabled: false + + #-- General RUM settings + + # A list of permitted origins for real user monitoring. + # User-agents will send an origin header that will be validated against this list. + # An origin is made of a protocol scheme, host and port, without the url path. + # Allowed origins in this setting can have * to match anything (eg.: http://*.example.com) + # If an item in the list is a single '*', everything will be allowed. + #allow_origins: ['*'] + + # A list of Access-Control-Allow-Headers to allow RUM requests, in addition to "Content-Type", + # "Content-Encoding", and "Accept" + #allow_headers: [] + + # Custom HTTP headers to add to RUM responses, e.g. for security policy compliance. + #response_headers: + # X-My-Header: Contents of the header + + # Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes. + # If the regexp matches, the stacktrace frame is considered to be a library frame. + #library_pattern: "node_modules|bower_components|~" + + # Regexp to be matched against a stacktrace frame's `file_name`. + # If the regexp matches, the stacktrace frame is not used for calculating error groups. + # The default pattern excludes stacktrace frames that have a filename starting with '/webpack' + #exclude_from_grouping: "^/webpack" + + # If a source map has previously been uploaded, source mapping is automatically applied. + # to all error and transaction documents sent to the RUM endpoint. + #source_mapping: + + # Sourcemapping is enabled by default. + #enabled: true + + # Timeout for fetching source maps. + #timeout: 5s + + # The `cache.expiration` determines how long a source map should be cached in memory. + # Note that values configured without a time unit will be interpreted as seconds. + #cache.expiration: 5m + + # Source map retrieval location. + # + # If using an output other than Elasticsearch that is writing to Elasticsearch, you must + # set this option. If not set, the standard output elasticsearch configuration is used. + #elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + #hosts: ["localhost:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + # Index pattern in which to search for source maps, when fetching source maps from Elasticsearch. + #index_pattern: "apm-*-sourcemap*" + + #---------------------------- APM Server - Agent Configuration ---------------------------- + + # When using APM agent configuration, information fetched from Elasticsearch or Kibana will be cached in memory for some time. + #agent.config: + # Specify cache key expiration via this setting. Default is 30 seconds. + #cache.expiration: 30s + + # Agent config will be fetched from Elasticsearch using the output.elasticsearch configuration. + # Elasticsearch authentication configurations are exposed to allow fine-tuned permission control + # and is required when working with Elastic Agent standalone or Fleet. + # This will override credentials in output.elasticsearch configuration. + #elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + #hosts: ["localhost:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + #kibana: + # Required when `apm-server.agent.config.elasticsearch` is not set AND `output.elasticsearch` + # is not valid (either because it's not set or there aren't enough privileges). + #enabled: false + + # Scheme and port can be left out and will be set to the default (`http` and `5601`). + # In case you specify an additional path, the scheme is required: `http://localhost:5601/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:5601`. + #host: "localhost:5601" + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Optional authentication with an API key + #api_key: "id:api_key" + + # Optional HTTP path. + #path: "" + + # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication. + #ssl.enabled: true + + # Optional SSL configuration options. SSL is off by default, change the `protocol` option if you want to enable `https`. + # + # Control the verification of Kibana certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.2 up to + # 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications. + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + # It is recommended to use the provided keystore instead of entering the passphrase in plain text. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + #---------------------------- APM Server - tail-based sampling ---------------------------- + + #sampling.tail: + # Set to `true` to enable tail based sampling. Disabled by default. + #enabled: false + + # Synchronization interval for multiple APM Servers. Should be in the order of tens of seconds or low minutes. + #interval: 1m + + # TTL determines how long trace events are retained in the local storage while waiting for a sampling decision to be made. + # A greater TTL value increases storage space requirements. Should be at least 2 * interval. + #ttl: 30m + + # Defines the indexing behavior when trace events fail to be written to storage (e.g. when the storage limit is reached). + # When set to `false`, traces will bypass sampling and always be indexed, significantly increasing the indexing load. + # When set to `true`, traces will be discarded, there will be data loss potentially resulting in broken traces. The default is `false`. + #discard_on_write_failure: false + + # Criteria used to match a root transaction to a sample rate. + #policies: [] + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +#============================= Elastic Cloud ============================= + +# These settings simplify using APM Server with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` option. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Outputs ================================= + +# Configure the output to use when sending the data collected by apm-server. + +#-------------------------- Elasticsearch output -------------------------- +output.elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + hosts: ["http://elasticsearch:9200"] + + # Boolean flag to enable or disable the output module. + enabled: true + + # Set gzip compression level. + #compression_level: 5 + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. + #path: "/elasticsearch" + + # Custom HTTP headers to add to each request. + #headers: + # X-My-Header: Contents of the header + + # Proxy server url. + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, apm-server + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure http request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # The bulk request size threshold, in bytes, before flushing to Elasticsearch. + # If compression is enabled, this is compressed bytes. + # The value must have a suffix, e.g. `"2MB"`. The default is `1MB`. + #flush_bytes: 1MB + + # The maximum duration to accumulate events for a bulk request before being flushed to Elasticsearch. + # The value must have a duration suffix, e.g. `"5s"`. The default is `1s`. + #flush_interval: 1s + + # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication. + #ssl.enabled: true + + # Optional SSL configuration options. SSL is off by default, change the `protocol` option if you want to enable `https`. + # + # Control the verification of Elasticsearch certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.2 up to + # 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications. + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + # It is recommended to use the provided keystore instead of entering the passphrase in plain text. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + +#----------------------------- Console output ----------------------------- +#output.console: + # Boolean flag to enable or disable the output module. + #enabled: false + + # Configure JSON encoding. + #codec.json: + # Pretty-print JSON event. + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + +#---------------------------- Logstash output ----------------------------- +output.logstash: + # Boolean flag to enable or disable the output module. + enabled: false + + # The Logstash hosts. + hosts: ["logstash:5044"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Configure escaping html symbols in strings. + #escape_html: true + + # Optional maximum time to live for a connection to Logstash, after which the + # connection will be re-established. A value of `0s` (the default) will + # disable this feature. + # + # Not yet supported for async connections (i.e. with the "pipelining" option set). + #ttl: 30s + + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false + + # Number of batches to be sent asynchronously to Logstash while processing + # new batches. + #pipelining: 2 + + # If enabled only a subset of events in a batch of events is transferred per + # group. The number of events to be sent increases up to `bulk_max_size` + # if no error is encountered. + #slow_start: false + + # The number of seconds to wait before trying to reconnect to Logstash + # after a network error. After waiting backoff.init seconds, apm-server + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Logstash after a network error. The default is 60s. + #backoff.max: 60s + + # Optional index name. The default index name is set to apm + # in all lowercase. + #index: 'apm' + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. + #ssl.enabled: false + + # Optional SSL configuration options. SSL is off by default. + # + # Control the verification of Logstash certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.2 up to + # 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications. + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + # It is recommended to use the provided keystore instead of entering the passphrase in plain text. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + +#------------------------------ Kafka output ------------------------------ +#output.kafka: + # Boolean flag to enable or disable the output module. + #enabled: false + + # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The cluster metadata contain the actual Kafka brokers events are published + # to. + #hosts: ["localhost:9092"] + + # The Kafka topic used for produced events. The setting can be a format string + # using any event field. To set the topic from document type use `%{[type]}`. + #topic: beats + + # The Kafka event key setting. Use format string to create unique event key. + # By default no event key will be generated. + #key: '' + + # The Kafka event partitioning strategy. Default hashing strategy is `hash` + # using the `output.kafka.key` setting or randomly distributes events if + # `output.kafka.key` is not configured. + #partition.hash: + # If enabled, events will only be published to partitions with reachable + # leaders. Default is false. + #reachable_only: false + + # Configure alternative event field names used to compute the hash value. + # If empty `output.kafka.key` setting will be used. + # Default value is empty list. + #hash: [] + + # Authentication details. Password is required if username is set. + #username: '' + #password: '' + + # Kafka version libbeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' + + # Configure JSON encoding. + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + + # Metadata update configuration. Metadata do contain leader information + # deciding which broker to use when publishing. + #metadata: + # Max metadata request retry attempts when cluster is in middle of leader + # election. Defaults to 3 retries. + #retry.max: 3 + + # Waiting time between retries during leader elections. Default is 250ms. + #retry.backoff: 250ms + + # Refresh metadata interval. Defaults to every 10 minutes. + #refresh_frequency: 10m + + # The number of concurrent load-balanced Kafka output workers. + #worker: 1 + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Kafka request. The default + # is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Kafka brokers before + # timing out. The default is 30s. + #timeout: 30s + + # The maximum duration a broker will wait for number of required ACKs. The + # default is 10s. + #broker_timeout: 10s + + # The number of messages buffered for each Kafka broker. The default is 256. + #channel_buffer_size: 256 + + # The keep-alive period for an active network connection. If 0s, keep-alives + # are disabled. The default is 0 seconds. + #keep_alive: 0 + + # Sets the output compression codec. Must be one of none, snappy and gzip. The + # default is gzip. + #compression: gzip + + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + + # The maximum permitted size of JSON-encoded messages. Bigger messages will be + # dropped. The default value is 1000000 (bytes). This value should be equal to + # or less than the broker's message.max.bytes. + #max_message_bytes: 1000000 + + # The ACK reliability level required from broker. 0=no response, 1=wait for + # local commit, -1=wait for all replicas to commit. The default is 1. Note: + # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently + # on error. + #required_acks: 1 + + # The configurable ClientID used for logging, debugging, and auditing + # purposes. The default is "beats". + #client_id: beats + + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. + #ssl.enabled: false + + # Optional SSL configuration options. SSL is off by default. + # + # Control the verification of Kafka certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.2 up to + # 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications. + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + # It is recommended to use the provided keystore instead of entering the passphrase in plain text. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/krb5kdc/kafka.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/path/config + + # The service principal name. + #kerberos.service_name: HTTP/my-service@realm + + # Name of the Kerberos user. It is used when auth_type is set to password. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC + +#============================= Instrumentation ============================= + +# Instrumentation support for the server's HTTP endpoints and event publisher. +#instrumentation: + + # Set to true to enable instrumentation of the APM Server itself. + #enabled: false + + # Environment in which the APM Server is running on (eg: staging, production, etc.) + #environment: "" + + # Hosts to report instrumentation results to. + # For reporting to itself, leave this field commented + #hosts: + # - http://remote-apm-server:8200 + + # API Key for the remote APM Server(s). + # If api_key is set then secret_token will be ignored. + #api_key: + + # Secret token for the remote APM Server(s). + #secret_token: + +#================================= Paths ================================== + +# The home path for the apm-server installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution. +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +path.home: /usr/local/lib/apm + +# The configuration path for the apm-server installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +path.config: ${path.home} + +# The data path for the apm-server installation. This is the default base path +# for all the files in which apm-server needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +path.data: /opt/apm/data + +# The logs path for an apm-server installation. If not set by a CLI flag or in the +# configuration file, the default is a logs subdirectory inside the home path. +path.logs: /opt/apm/log + +#================================= Logging ================================= + +# There are three options for the log output: syslog, file, and stderr. +# Windows systems default to file output. All other systems default to syslog. + +# Sets the minimum log level. The default log level is info. +# Available log levels are: error, warning, info, or debug. +logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"]. +# Other available selectors are "beat", "publish", or "service". +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: true + +# If enabled, apm-server periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is false. +logging.metrics.enabled: false + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. When true, writes all logging output to files. +# The log files are automatically rotated when the log file size limit is reached. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + path: ${path.logs} + + # The name of the files where the logs are written to. + name: server.log + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + permissions: 0644 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + +#=============================== HTTP Endpoint =============================== + +# apm-server can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats. For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +http.enabled: false + +# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. +http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. +http.port: 5066 + +#============================= X-pack Monitoring ============================= + +# APM server can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires x-pack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +monitoring.enabled: false + +# Most settings from the Elasticsearch output are accepted here as well. +# Note that these settings should be configured to point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration. This means that if you have the Elasticsearch output configured, +# you can simply uncomment the following line. +#monitoring.elasticsearch: + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + #hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Custom HTTP headers to add to each request. + #headers: + # X-My-Header: Contents of the header + + # Proxy server url. + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, apm-server + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication. + #ssl.enabled: true + + # Optional SSL configuration options. SSL is off by default, change the `protocol` option if you want to enable `https`. + # + # Control the verification of Elasticsearch certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.2 up to + # 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications. + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + # It is recommended to use the provided keystore instead of entering the passphrase in plain text. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + #metrics.period: 10s + #state.period: 1m \ No newline at end of file diff --git a/workload/apm/9.2.3/conf/supervisor.ini b/workload/apm/9.2.3/conf/supervisor.ini new file mode 100644 index 0000000..0620c62 --- /dev/null +++ b/workload/apm/9.2.3/conf/supervisor.ini @@ -0,0 +1,6 @@ +[program:apm] +command=/usr/local/lib/apm/apm-server run --path.config=/usr/local/lib/apm --strict.perms=false +directory=/usr/local/lib/apm +process_name=%(program_name)s +pidfile=/opt/apm/system/server.pid +autorestart=false diff --git a/workload/elasticsearch/9.2.3/conf/jvm.options b/workload/elasticsearch/9.2.3/conf/jvm.options new file mode 100644 index 0000000..2fc24df --- /dev/null +++ b/workload/elasticsearch/9.2.3/conf/jvm.options @@ -0,0 +1,91 @@ +################################################################ +## +## JVM configuration +## +################################################################ +## +## WARNING: DO NOT EDIT THIS FILE. If you want to override the +## JVM options in this file, or set any additional options, you +## should create one or more files in the jvm.options.d +## directory containing your adjustments. +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/advanced-configuration.html#set-jvm-options +## for more information. +## +################################################################ + + + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## The heap size is automatically configured by Elasticsearch +## based on the available memory in your system and the roles +## each node is configured to fulfill. If specifying heap is +## required, it should be done through a file in jvm.options.d, +## which should be named with .options suffix, and the min and +## max should be set to the same value. For example, to set the +## heap to 4 GB, create a new file in the jvm.options.d +## directory containing these lines: +## +-Xms512m +-Xmx512m +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/heap-size.html +## for more information +## +################################################################ + + +################################################################ +## Expert settings +################################################################ +## +## All settings below here are considered expert settings. Do +## not adjust them unless you understand what you are doing. Do +## not edit them in this file; instead, create a new file in the +## jvm.options.d directory containing your adjustments. +## +################################################################ + +-XX:+UseG1GC + +## JVM temporary directory +-Djava.io.tmpdir=${ES_TMPDIR} + +# Leverages accelerated vector hardware instructions; removing this may +# result in less optimal vector performance +20-:--add-modules=jdk.incubator.vector + +# Required to workaround performance issue in JDK 23, https://github.com/elastic/elasticsearch/issues/113030 +23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache +23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached + +# Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead +-Dorg.apache.lucene.store.defaultReadAdvice=normal + +# Lucene provides a mechanism for shared mmapped arenas to be referenced between multiple threads +# this is to get around potential performance issues when closing shared arenas on many threads +# default to 1 to disable this feature +-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1 + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails; heap dumps +# are created in the working directory of the JVM unless an alternative path is +# specified +-XX:+HeapDumpOnOutOfMemoryError + +# exit right after heap dump on out of memory error +-XX:+ExitOnOutOfMemoryError + +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath=/heap/dump/path + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile=hs_err_pid%p.log + +## GC logging +-Xlog:gc*,gc+age=trace,safepoint:file=gc.log:utctime,level,pid,tags:filecount=32,filesize=64m diff --git a/workload/elasticsearch/9.2.3/conf/security-limit.conf b/workload/elasticsearch/9.2.3/conf/security-limit.conf new file mode 100644 index 0000000..26fff2b --- /dev/null +++ b/workload/elasticsearch/9.2.3/conf/security-limit.conf @@ -0,0 +1,4 @@ +elasticsearch soft nofile 65536 +elasticsearch hard nofile 65536 +elasticsearch soft memlock unlimited +elasticsearch hard memlock unlimited \ No newline at end of file diff --git a/workload/elasticsearch/9.2.3/conf/server.yml b/workload/elasticsearch/9.2.3/conf/server.yml new file mode 100644 index 0000000..7538ad0 --- /dev/null +++ b/workload/elasticsearch/9.2.3/conf/server.yml @@ -0,0 +1,118 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: elasticsearch +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: elasticsearch +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: /opt/elasticsearch/data +# +# Path to log files: +# +path.logs: /opt/elasticsearch/log +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +bootstrap.memory_lock: false +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# By default Elasticsearch is only accessible on localhost. Set a different +# address here to expose this node on the network: +# +network.host: 0.0.0.0 +# +# By default Elasticsearch listens for HTTP traffic on the first free port it +# finds starting at 9200. Set a specific HTTP port here: +# +http.port: 9200 +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +discovery.seed_hosts: [] +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +cluster.initial_master_nodes: ["elasticsearch"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Allow wildcard deletion of indices: +# +action.destructive_requires_name: false + +#----------------------- BEGIN SECURITY AUTO CONFIGURATION ----------------------- +# +# The following settings, TLS certificates, and keys have been automatically +# generated to configure Elasticsearch security features on 01-09-2025 06:15:04 +# +# -------------------------------------------------------------------------------- + +# Enable security features +xpack.security.enabled: false + +xpack.security.enrollment.enabled: false + +# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents +#xpack.security.http.ssl: + #enabled: false + #keystore.path: certs/http.p12 +# Enable encryption and mutual authentication between cluster nodes +#xpack.security.transport.ssl: + #enabled: false + #verification_mode: certificate + #keystore.path: certs/transport.p12 + #truststore.path: certs/transport.p12 +# Create a new cluster with the current node only +# Additional nodes can still join the cluster later +#cluster.initial_master_nodes: ["21ec2b263c3b"] + +# Allow HTTP API connections from anywhere +# Connections are encrypted and require user authentication +http.host: 0.0.0.0 + +# Allow other nodes to join the cluster from anywhere +# Connections are encrypted and mutually authenticated +transport.host: 0.0.0.0 + +#----------------------- END SECURITY AUTO CONFIGURATION ------------------------- diff --git a/workload/elasticsearch/9.2.3/conf/supervisor.ini b/workload/elasticsearch/9.2.3/conf/supervisor.ini new file mode 100644 index 0000000..f4522f0 --- /dev/null +++ b/workload/elasticsearch/9.2.3/conf/supervisor.ini @@ -0,0 +1,7 @@ +[program:elasticsearch] +command=/usr/local/lib/elasticsearch/bin/elasticsearch -p /opt/elasticsearch/system/server.pid +user=elasticsearch +directory=/usr/local/lib/elasticsearch +process_name=%(program_name)s +pidfile=/opt/elasticsearch/system/server.pid +autorestart=false \ No newline at end of file diff --git a/workload/kibana/9.2.3/conf/server.yml b/workload/kibana/9.2.3/conf/server.yml new file mode 100644 index 0000000..752da4a --- /dev/null +++ b/workload/kibana/9.2.3/conf/server.yml @@ -0,0 +1,179 @@ +# For more configuration options see the configuration guide for Kibana in +# https://www.elastic.co/guide/index.html + +# =================== System: Kibana Server =================== +# Kibana is served by a back end server. This setting specifies the port to use. +server.port: 5601 + +# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. +# The default is 'localhost', which usually means remote machines will not be able to connect. +# To allow connections from remote users, set this parameter to a non-loopback address. +server.host: 0.0.0.0 + +# Enables you to specify a path to mount Kibana at if you are running behind a proxy. +# Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath +# from requests it receives, and to prevent a deprecation warning at startup. +# This setting cannot end in a slash. +#server.basePath: "" + +# Specifies whether Kibana should rewrite requests that are prefixed with +# `server.basePath` or require that they are rewritten by your reverse proxy. +# Defaults to `false`. +#server.rewriteBasePath: false + +# Specifies the public URL at which Kibana is available for end users. If +# `server.basePath` is configured this URL should end with the same basePath. +#server.publicBaseUrl: "" + +# The maximum payload size in bytes for incoming server requests. +#server.maxPayload: 1048576 + +# The Kibana server's name. This is used for display purposes. +server.name: kibana + +# =================== System: Kibana Server (Optional) =================== +# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. +# These settings enable SSL for outgoing requests from the Kibana server to the browser. +#server.ssl.enabled: false +#server.ssl.certificate: /path/to/your/server.crt +#server.ssl.key: /path/to/your/server.key + +# =================== System: Elasticsearch =================== +# The URLs of the Elasticsearch instances to use for all your queries. +elasticsearch.hosts: ["http://elasticsearch:9200"] + +# If your Elasticsearch is protected with basic authentication, these settings provide +# the username and password that the Kibana server uses to perform maintenance on the Kibana +# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which +# is proxied through the Kibana server. +#elasticsearch.username: "kibana_system" +#elasticsearch.password: "pass" + +# Kibana can also authenticate to Elasticsearch via "service account tokens". +# Service account tokens are Bearer style tokens that replace the traditional username/password based configuration. +# Use this token instead of a username/password. +# elasticsearch.serviceAccountToken: "my_token" + +# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value +# must be a positive integer. +#elasticsearch.requestTimeout: 30000 + +# The maximum number of sockets that can be used for communications with elasticsearch. +# Defaults to `800`. +#elasticsearch.maxSockets: 1024 + +# Specifies whether Kibana should use compression for communications with elasticsearch +# Defaults to `false`. +#elasticsearch.compression: false + +# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side +# headers, set this value to [] (an empty list). +#elasticsearch.requestHeadersWhitelist: [ authorization ] + +# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten +# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. +#elasticsearch.customHeaders: {} + +# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. +#elasticsearch.shardTimeout: 30000 + +# =================== System: Elasticsearch (Optional) =================== +# These files are used to verify the identity of Kibana to Elasticsearch and are required when +# xpack.security.http.ssl.client_authentication in Elasticsearch is set to required. +#elasticsearch.ssl.certificate: /path/to/your/client.crt +#elasticsearch.ssl.key: /path/to/your/client.key + +# Enables you to specify a path to the PEM file for the certificate +# authority for your Elasticsearch instance. +#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ] + +# To disregard the validity of SSL certificates, change this setting's value to 'none'. +elasticsearch.ssl.verificationMode: none + +# =================== System: Logging =================== +# Set the value of this setting to off to suppress all logging output, or to debug to log everything. Defaults to 'info' +logging.root.level: info + +# Enables you to specify a file where Kibana stores log output. +logging.appenders.default: + type: file + fileName: /opt/kibana/log/stdout.log + layout: + type: json + +# Example with size based log rotation +#logging.appenders.default: +# type: rolling-file +# fileName: /var/logs/kibana.log +# policy: +# type: size-limit +# size: 256mb +# strategy: +# type: numeric +# max: 10 +# layout: +# type: json + +# Logs queries sent to Elasticsearch. +#logging.loggers: +# - name: elasticsearch.query +# level: debug + +# Logs http responses. +#logging.loggers: +# - name: http.server.response +# level: debug + +# Logs system usage information. +#logging.loggers: +# - name: metrics.ops +# level: debug + +# Enables debug logging on the browser (dev console) +#logging.browser.root: +# level: debug + +# =================== System: Other =================== +# The path where Kibana stores persistent data not saved in Elasticsearch. Defaults to data +path.data: /opt/kibana/data + +# Specifies the path where Kibana creates the process ID file. +pid.file: /opt/kibana/system/server.pid + +# Set the interval in milliseconds to sample system and process performance +# metrics. Minimum is 100ms. Defaults to 5000ms. +ops.interval: 5000 + +# Specifies locale to be used for all localizable strings, dates and number formats. +# Supported languages are the following: English (default) "en", Chinese "zh-CN", Japanese "ja-JP", French "fr-FR", German "de-DE". +#i18n.locale: "en" + +# =================== Frequently used (Optional)=================== + +# =================== Saved Objects: Migrations =================== +# Saved object migrations run at startup. If you run into migration-related issues, you might need to adjust these settings. + +# The number of documents migrated at a time. +# If Kibana can't start up or upgrade due to an Elasticsearch `circuit_breaking_exception`, +# use a smaller batchSize value to reduce the memory pressure. Defaults to 1000 objects per batch. +#migrations.batchSize: 1000 + +# The maximum payload size for indexing batches of upgraded saved objects. +# To avoid migrations failing due to a 413 Request Entity Too Large response from Elasticsearch. +# This value should be lower than or equal to your Elasticsearch cluster’s `http.max_content_length` +# configuration option. Default: 100mb +#migrations.maxBatchSizeBytes: 100mb + +# The number of times to retry temporary migration failures. Increase the setting +# if migrations fail frequently with a message such as `Unable to complete the [...] step after +# 15 attempts, terminating`. Defaults to 15 +#migrations.retryAttempts: 15 + +# =================== Search Autocomplete =================== +# Time in milliseconds to wait for autocomplete suggestions from Elasticsearch. +# This value must be a whole number greater than zero. Defaults to 1000ms +#unifiedSearch.autocomplete.valueSuggestions.timeout: 1000 + +# Maximum number of documents loaded by each shard to generate autocomplete suggestions. +# This value must be a whole number greater than zero. Defaults to 100_000 +#unifiedSearch.autocomplete.valueSuggestions.terminateAfter: 100000 \ No newline at end of file diff --git a/workload/kibana/9.2.3/conf/supervisor.ini b/workload/kibana/9.2.3/conf/supervisor.ini new file mode 100644 index 0000000..dbed8e2 --- /dev/null +++ b/workload/kibana/9.2.3/conf/supervisor.ini @@ -0,0 +1,6 @@ +[program:kibana] +command=/usr/local/lib/kibana/bin/kibana -c /opt/kibana/conf/server.yml --allow-root +directory=/usr/local/lib/kibana +process_name=%(program_name)s +pidfile=/opt/kibana/system/server.pid +autorestart=false \ No newline at end of file diff --git a/workload/logstash/6.8.2/conf/supervisor.ini b/workload/logstash/6.8.2/conf/supervisor.ini index d584343..53f690a 100644 --- a/workload/logstash/6.8.2/conf/supervisor.ini +++ b/workload/logstash/6.8.2/conf/supervisor.ini @@ -2,4 +2,4 @@ command=bash -c "/usr/local/lib/logstash/bin/logstash" process_name=%(program_name)s pidfile=/opt/logstash/system/server.pid -autorestart=false +autorestart=false \ No newline at end of file diff --git a/workload/logstash/9.2.3/conf/beats.conf b/workload/logstash/9.2.3/conf/beats.conf new file mode 100644 index 0000000..3a65a9d --- /dev/null +++ b/workload/logstash/9.2.3/conf/beats.conf @@ -0,0 +1,99 @@ +input { + beats { + port => 5044 + } +} + +filter { + if [event][module] == "nginx" { + if [fileset][name] == "access" { + grok { + match => { + "message" => "\[%{HTTPDATE:[nginx][access][datetime]}\] %{IP:[nginx][access][remote_ip]} %{HOSTNAME:[nginx][access][host]} \"%{WORD:[nginx][access][method]} %{URIPATHPARAM:[nginx][access][path]} HTTP/%{NUMBER:[nginx][access][version]}\" %{NUMBER:[nginx][access][status]} \"%{DATA:[nginx][access][referer]}\" %{INT:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][user_agent]}\" %{NUMBER:[nginx][access][request_time]}" + } + } + mutate { + remove_field => ["message"] + } + date { + match => [ "[nginx][access][datetime]", "dd/MMM/yyyy:HH:mm:ss Z" ] + target => "@timestamp" + } + mutate { + remove_field => ["[nginx][access][datetime]"] + } + } + else if [fileset][name] == "error" { + grok { + match => { + "message" => "%{DATE:[nginx][error][date]} %{TIME:[nginx][error][time]} \[%{WORD:[nginx][error][level]}\] %{INT:[nginx][error][process_id]}#%{INT:[nginx][error][thread_id]}: \*%{INT:[nginx][error][connection_id]} %{GREEDYDATA:[nginx][error][message]}" + } + } + mutate { + add_field => { "[nginx][error][datetime]" => "%{[nginx][error][date]} %{[nginx][error][time]} +0530" } + remove_field => ["message", "[nginx][error][date]", "[nginx][error][time]"] + } + date { + match => [ "[nginx][error][datetime]", "yy/MM/dd HH:mm:ss Z" ] + target => "@timestamp" + } + mutate { + remove_field => ["[nginx][error][datetime]"] + } + } + } + else if [event][module] == "docker" { + if [fileset][name] == "log" { + json { + source => "message" + target => "docker_json" + } + grok { + match => { + "[docker_json][log]" => "%{TIMESTAMP_ISO8601:[docker][log][datetime]} %{WORD:[docker][log][level]} %{GREEDYDATA:[docker][log][message]}" + } + } + date { + match => [ "[docker_json][time]", "ISO8601" ] + target => "@timestamp" + } + mutate { + add_field => { "[docker][log][type]" => "%{[docker_json][stream]}" } + remove_field => ["message", "docker_json", "prospector", "input"] + } + } + } + else if [agent][type] == "heartbeat" { + mutate { + add_tag => [ "heartbeat_data" ] + add_field => { "[@metadata][beat]" => "heartbeat" } + } + } +} + +output { + if [@metadata][beat] == "heartbeat" { + elasticsearch { + hosts => ["http://elasticsearch:9200"] + action => "create" + data_stream => "true" + data_stream_type => "logs" + data_stream_dataset => "heartbeat" + data_stream_namespace => "default" + } + } else if [@metadata][beat] == "filebeat" { + elasticsearch { + hosts => ["http://elasticsearch:9200"] + data_stream => "true" + data_stream_type => "logs" + data_stream_dataset => "filebeat" + data_stream_namespace => "default" + } + } else { + elasticsearch { + hosts => ["http://elasticsearch:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" + action => "create" + } + } +} \ No newline at end of file diff --git a/workload/logstash/9.2.3/conf/jvm.options b/workload/logstash/9.2.3/conf/jvm.options new file mode 100644 index 0000000..ed48478 --- /dev/null +++ b/workload/logstash/9.2.3/conf/jvm.options @@ -0,0 +1,76 @@ +## JVM configuration + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms512m +-Xms512m + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + + +## Locale +# Set the locale language +#-Duser.language=en + +# Set the locale country +#-Duser.country=US + +# Set the locale variant, if any +#-Duser.variant= + +## basic + +# set the I/O temp directory +#-Djava.io.tmpdir=${HOME} + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +#-Djna.nosys=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof + +## GC logging +#-Xlog:gc*,gc+age=trace,safepoint:file=${LS_GC_LOG_FILE}:utctime,pid,tags:filecount=32,filesize=64m + +# Entropy source for randomness +-Djava.security.egd=file:/dev/urandom + +# FasterXML/jackson defaults +# +# Sets the maximum string length (in chars or bytes, depending on input context). +# This limit is not exact and an exception will happen at sizes greater than this limit. +# Some text values that are a little bigger than the limit may be treated as valid but no +# text values with sizes less than or equal to this limit will be treated as invalid. +# This value should be higher than `logstash.jackson.stream-read-constraints.max-number-length`. +# The jackson library defaults to 20000000 or 20MB, whereas Logstash defaults to 200MB or 200000000 characters. +-Dlogstash.jackson.stream-read-constraints.max-string-length=200000000 +# +# Sets the maximum number length (in chars or bytes, depending on input context). +# The jackson library defaults to 1000, whereas Logstash defaults to 10000. +-Dlogstash.jackson.stream-read-constraints.max-number-length=10000 +# +# Sets the maximum nesting depth. The depth is a count of objects and arrays that have not +# been closed, `{` and `[` respectively. +-Dlogstash.jackson.stream-read-constraints.max-nesting-depth=1000 \ No newline at end of file diff --git a/workload/logstash/9.2.3/conf/pipelines.yml b/workload/logstash/9.2.3/conf/pipelines.yml new file mode 100644 index 0000000..15f3b75 --- /dev/null +++ b/workload/logstash/9.2.3/conf/pipelines.yml @@ -0,0 +1,106 @@ +# List of pipelines to be loaded by Logstash +# +# This document must be a list of dictionaries/hashes, where the keys/values are pipeline settings. +# Default values for omitted settings are read from the `logstash.yml` file. +# When declaring multiple pipelines, each MUST have its own `pipeline.id`. +# +# Example of two pipelines: +# +# - pipeline.id: test +# pipeline.workers: 1 +# pipeline.batch.size: 1 +# config.string: "input { generator {} } filter { sleep { time => 1 } } output { stdout { codec => dots } }" +- pipeline.id: beats + queue.type: persisted + path.config: /opt/logstash/conf/beats.conf +# +# Available options: +# +# # name of the pipeline +# pipeline.id: mylogs +# +# # The configuration string to be used by this pipeline +# config.string: "input { generator {} } filter { sleep { time => 1 } } output { stdout { codec => dots } }" +# +# # The path from where to read the configuration text +# path.config: "/etc/conf.d/logstash/myconfig.cfg" +# +# # How many worker threads execute the Filters+Outputs stage of the pipeline +# pipeline.workers: 1 (actually defaults to number of CPUs) +# +# # How many events to retrieve from inputs before sending to filters+workers +# pipeline.batch.size: 125 +# +# # How long to wait in milliseconds while polling for the next event +# # before dispatching an undersized batch to filters+outputs +# pipeline.batch.delay: 50 +# +# Set the pipeline event ordering. Options are "auto" (the default), "true" # # or "false". +# "auto" automatically enables ordering if the 'pipeline.workers' setting +# is also set to '1', and disables otherwise. +# "true" enforces ordering on a pipeline and prevents logstash from starting +# a pipeline with multiple workers allocated. +# "false" disable any extra processing necessary for preserving ordering. +# +# pipeline.ordered: auto +# +# # Internal queuing model, "memory" for legacy in-memory based queuing and +# # "persisted" for disk-based acked queueing. Defaults is memory +# queue.type: memory +# +# # If using queue.type: persisted, the page data files size. The queue data consists of +# # append-only data files separated into pages. Default is 64mb +# queue.page_capacity: 64mb +# +# # If using queue.type: persisted, the maximum number of unread events in the queue. +# # Default is 0 (unlimited) +# queue.max_events: 0 +# +# # If using queue.type: persisted, the total capacity of the queue in number of bytes. +# # Default is 1024mb or 1gb +# queue.max_bytes: 1024mb +# +# # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint +# # Default is 1024, 0 for unlimited +# queue.checkpoint.acks: 1024 +# +# # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint +# # Default is 1024, 0 for unlimited +# queue.checkpoint.writes: 1024 +# +# # Enable Dead Letter Queueing for this pipeline. +# dead_letter_queue.enable: false +# +# If using dead_letter_queue.enable: true, the maximum size of dead letter queue for this pipeline. Entries +# will be dropped if they would increase the size of the dead letter queue beyond this setting. +# Default is 1024mb +# dead_letter_queue.max_bytes: 1024mb +# +# If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ +# have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files +# may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and +# being available to be read by the dead_letter_queue input when items are are written infrequently. +# Default is 5000. +# +# dead_letter_queue.flush_interval: 5000 + +# If using dead_letter_queue.enable: true, controls which entries should be dropped to avoid exceeding the size limit. +# Set the value to `drop_newer` (default) to stop accepting new events that would push the DLQ size over the limit. +# Set the value to `drop_older` to remove queue pages containing the oldest events to make space for new ones. +# +# dead_letter_queue.storage_policy: drop_newer + +# If using dead_letter_queue.enable: true, the interval that events have to be considered valid. After the interval has +# expired the events could be automatically deleted from the DLQ. +# The interval could be expressed in days, hours, minutes or seconds, using as postfix notation like 5d, +# to represent a five days interval. +# The available units are respectively d, h, m, s for day, hours, minutes and seconds. +# If not specified then the DLQ doesn't use any age policy for cleaning events. +# +# dead_letter_queue.retain.age: 1d + +# +# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. +# Default is path.data/dead_letter_queue +# +# path.dead_letter_queue: \ No newline at end of file diff --git a/workload/logstash/9.2.3/conf/server.yml b/workload/logstash/9.2.3/conf/server.yml new file mode 100644 index 0000000..fc8992c --- /dev/null +++ b/workload/logstash/9.2.3/conf/server.yml @@ -0,0 +1,372 @@ +# Settings file in YAML +# +# Settings can be specified either in hierarchical form, e.g.: +# +# pipeline: +# batch: +# size: 125 +# delay: 5 +# +# Or as flat keys: +# +# pipeline.batch.size: 125 +# pipeline.batch.delay: 5 +# +# ------------ Node identity ------------ +# +# Use a descriptive name for the node: +# +node.name: logstash +# +# If omitted the node name will default to the machine's host name +# +# ------------ Data path ------------------ +# +# Which directory should be used by logstash and its plugins +# for any persistent needs. Defaults to LOGSTASH_HOME/data +# +path.data: /opt/logstash/data +# +# ------------ Pipeline Settings -------------- +# +# The ID of the pipeline. +# +pipeline.id: main +# +# Set the number of workers that will, in parallel, execute the filters+outputs +# stage of the pipeline. +# +# This defaults to the number of the host's CPU cores. +# +pipeline.workers: 2 +# +# How many events to retrieve from inputs before sending to filters+workers +# +pipeline.batch.size: 125 +# +# How long to wait in milliseconds while polling for the next event +# before dispatching an undersized batch to filters+outputs +# +pipeline.batch.delay: 50 +# +# Set the pipeline's batch metrics reporting mode. It can be "disabled" to disable it. +# "minimal" to collect only 1% of the batches metrics, "full" to collect all batches. +# Default is "minimal". +# +# pipeline.batch.metrics.sampling_mode: "minimal" +# +# Force Logstash to exit during shutdown even if there are still inflight +# events in memory. By default, logstash will refuse to quit until all +# received events have been pushed to the outputs. +# +# WARNING: Enabling this can lead to data loss during shutdown +# +pipeline.unsafe_shutdown: false +# +# Set the pipeline event ordering. Options are "auto" (the default), "true" or "false". +# "auto" automatically enables ordering if the 'pipeline.workers' setting +# is also set to '1', and disables otherwise. +# "true" enforces ordering on the pipeline and prevent logstash from starting +# if there are multiple workers. +# "false" disables any extra processing necessary for preserving ordering. +# +# pipeline.ordered: auto +# +# Sets the pipeline's default value for `ecs_compatibility`, a setting that is +# available to plugins that implement an ECS Compatibility mode for use with +# the Elastic Common Schema. +# Possible values are: +# - disabled +# - v1 +# - v8 (default) +# Pipelines defined before Logstash 8 operated without ECS in mind. To ensure a +# migrated pipeline continues to operate as it did before your upgrade, opt-OUT +# of ECS for the individual pipeline in its `pipelines.yml` definition. Setting +# it here will set the default for _all_ pipelines, including new ones. +# +# pipeline.ecs_compatibility: v8 +# +# ------------ Pipeline Configuration Settings -------------- +# +# Where to fetch the pipeline configuration for the main pipeline +# +# path.config: +# +# Pipeline configuration string for the main pipeline +# +# config.string: +# +# At startup, test if the configuration is valid and exit (dry run) +# +# config.test_and_exit: false +# +# Periodically check if the configuration has changed and reload the pipeline +# This can also be triggered manually through the SIGHUP signal +# +# config.reload.automatic: false +# +# How often to check if the pipeline configuration has changed (in seconds) +# Note that the unit value (s) is required. Values without a qualifier (e.g. 60) +# are treated as nanoseconds. +# Setting the interval this way is not recommended and might change in later versions. +# +# config.reload.interval: 3s +# +# Show fully compiled configuration as debug log message +# NOTE: --log.level must be 'debug' +# +# config.debug: false +# +# When enabled, process escaped characters such as \n and \" in strings in the +# pipeline configuration files. +# +# config.support_escapes: false +# +# ------------ API Settings ------------- +# Define settings related to the HTTP API here. +# +# The HTTP API is enabled by default. It can be disabled, but features that rely +# on it will not work as intended. +# +# api.enabled: true +# +# By default, the HTTP API is not secured and is therefore bound to only the +# host's loopback interface, ensuring that it is not accessible to the rest of +# the network. +# When secured with SSL and Basic Auth, the API is bound to _all_ interfaces +# unless configured otherwise. +# +api.http.host: 0.0.0.0 +# +# The HTTP API web server will listen on an available port from the given range. +# Values can be specified as a single port (e.g., `9600`), or an inclusive range +# of ports (e.g., `9600-9700`). +# +api.http.port: 9600-9700 +# +# The HTTP API includes a customizable "environment" value in its response, +# which can be configured here. +# +# api.environment: "production" +# +# The HTTP API can be secured with SSL (TLS). To do so, you will need to provide +# the path to a password-protected keystore in p12 or jks format, along with credentials. +# +api.ssl.enabled: false +# api.ssl.keystore.path: /path/to/keystore.jks +# api.ssl.keystore.password: "y0uRp4$$w0rD" +# +# The availability of SSL/TLS protocols depends on the JVM version. Certain protocols are +# disabled by default and need to be enabled manually by changing `jdk.tls.disabledAlgorithms` +# in the $JDK_HOME/conf/security/java.security configuration file. +# +# api.ssl.supported_protocols: [TLSv1.2,TLSv1.3] +# +# The HTTP API can be configured to require authentication. Acceptable values are +# - `none`: no auth is required (default) +# - `basic`: clients must authenticate with HTTP Basic auth, as configured +# with `api.auth.basic.*` options below +api.auth.type: none +# +# When configured with `api.auth.type` `basic`, you must provide the credentials +# that requests will be validated against. Usage of Environment or Keystore +# variable replacements is encouraged (such as the value `"${HTTP_PASS}"`, which +# resolves to the value stored in the keystore's `HTTP_PASS` variable if present +# or the same variable from the environment) +# +# api.auth.basic.username: "logstash-user" +# api.auth.basic.password: "s3cUreP4$$w0rD" +# +# When setting `api.auth.basic.password`, the password should meet +# the default password policy requirements. +# The default password policy requires non-empty minimum 8 char string that +# includes a digit, upper case letter and lower case letter. +# Policy mode sets Logstash to WARN or ERROR when HTTP authentication password doesn't +# meet the password policy requirements. +# The default is WARN. Setting to ERROR enforces stronger passwords (recommended). +# +# api.auth.basic.password_policy.mode: WARN +# +# ------------ Queuing Settings -------------- +# +# Internal queuing model, "memory" for legacy in-memory based queuing and +# "persisted" for disk-based acked queueing. Defaults is memory +# +# queue.type: memory +# +# If `queue.type: persisted`, the directory path where the pipeline data files will be stored. +# Each pipeline will group its PQ files in a subdirectory matching its `pipeline.id`. +# Default is path.data/queue. +# +# path.queue: +# +# If using queue.type: persisted, the page data files size. The queue data consists of +# append-only data files separated into pages. Default is 64mb +# +# queue.page_capacity: 64mb +# +# If using queue.type: persisted, the maximum number of unread events in the queue. +# Default is 0 (unlimited) +# +# queue.max_events: 0 +# +# If using queue.type: persisted, the total capacity of the queue in number of bytes. +# If you would like more unacked events to be buffered in Logstash, you can increase the +# capacity using this setting. Please make sure your disk drive has capacity greater than +# the size specified here. If both max_bytes and max_events are specified, Logstash will pick +# whichever criteria is reached first +# Default is 1024mb or 1gb +# +# queue.max_bytes: 1024mb +# +# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.acks: 1024 +# +# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.writes: 1024 +# +# If using queue.type: persisted, the compression goal. Valid values are `none`, `speed`, `balanced`, and `size`. +# The default `none` is able to decompress previously-written events, even if they were compressed. +# +# queue.compression: none +# +# ------------ Dead-Letter Queue Settings -------------- +# Flag to turn on dead-letter queue. +# +# dead_letter_queue.enable: false + +# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries +# will be dropped if they would increase the size of the dead letter queue beyond this setting. +# Default is 1024mb +# dead_letter_queue.max_bytes: 1024mb + +# If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ +# have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files +# may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and +# being available to be read by the dead_letter_queue input when items are written infrequently. +# Default is 5000. +# +# dead_letter_queue.flush_interval: 5000 + +# If using dead_letter_queue.enable: true, controls which entries should be dropped to avoid exceeding the size limit. +# Set the value to `drop_newer` (default) to stop accepting new events that would push the DLQ size over the limit. +# Set the value to `drop_older` to remove queue pages containing the oldest events to make space for new ones. +# +# dead_letter_queue.storage_policy: drop_newer + +# If using dead_letter_queue.enable: true, the interval that events have to be considered valid. After the interval has +# expired the events could be automatically deleted from the DLQ. +# The interval could be expressed in days, hours, minutes or seconds, using as postfix notation like 5d, +# to represent a five days interval. +# The available units are respectively d, h, m, s for day, hours, minutes and seconds. +# If not specified then the DLQ doesn't use any age policy for cleaning events. +# +# dead_letter_queue.retain.age: 1d + +# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. +# Default is path.data/dead_letter_queue +# +# path.dead_letter_queue: +# +# ------------ Debugging Settings -------------- +# +# Options for log.level: +# * fatal +# * error +# * warn +# * info (default) +# * debug +# * trace +log.level: info +# +# Options for log.format: +# * plain (default) +# * json +# +# log.format: plain +# log.format.json.fix_duplicate_message_fields: true +# +path.logs: /opt/logstash/log +# +# ------------ Other Settings -------------- +# +# Allow or block running Logstash as superuser (default: false). Windows are excluded from the checking +# allow_superuser: false +# +# Where to find custom plugins +# path.plugins: [] +# +# Flag to output log lines of each pipeline in its separate log file. Each log filename contains the pipeline.name +# Default is false +# pipeline.separate_logs: false +# +# Determine where to allocate memory buffers, for plugins that leverage them. +# Defaults to heap,but can be switched to direct if you prefer using direct memory space instead. +# pipeline.buffer.type: heap +# +# ------------ X-Pack Settings (not applicable for OSS build)-------------- +# +# X-Pack Monitoring +# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html +# Flag to allow the legacy internal monitoring (default: false) +#xpack.monitoring.allow_legacy_collection: false +#xpack.monitoring.enabled: false +#xpack.monitoring.elasticsearch.username: logstash_system +#xpack.monitoring.elasticsearch.password: password +#xpack.monitoring.elasticsearch.proxy: ["http://proxy:port"] +#xpack.monitoring.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"] +# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth +#xpack.monitoring.elasticsearch.cloud_id: monitoring_cluster_id:xxxxxxxxxx +#xpack.monitoring.elasticsearch.cloud_auth: logstash_system:password +# another authentication alternative is to use an Elasticsearch API key +#xpack.monitoring.elasticsearch.api_key: "id:api_key" +#xpack.monitoring.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt" +#xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx +#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file +#xpack.monitoring.elasticsearch.ssl.truststore.password: password +# use either keystore.path/keystore.password or certificate/key configurations +#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file +#xpack.monitoring.elasticsearch.ssl.keystore.password: password +#xpack.monitoring.elasticsearch.ssl.certificate: /path/to/file +#xpack.monitoring.elasticsearch.ssl.key: /path/to/key +#xpack.monitoring.elasticsearch.ssl.verification_mode: full +#xpack.monitoring.elasticsearch.ssl.cipher_suites: [] +#xpack.monitoring.elasticsearch.sniffing: false +#xpack.monitoring.collection.interval: 10s +#xpack.monitoring.collection.pipeline.details.enabled: true +# +# X-Pack Management +# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html +#xpack.management.enabled: false +#xpack.management.pipeline.id: ["main", "apache_logs"] +#xpack.management.elasticsearch.username: logstash_admin_user +#xpack.management.elasticsearch.password: password +#xpack.management.elasticsearch.proxy: ["http://proxy:port"] +#xpack.management.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"] +# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth +#xpack.management.elasticsearch.cloud_id: management_cluster_id:xxxxxxxxxx +#xpack.management.elasticsearch.cloud_auth: logstash_admin_user:password +# another authentication alternative is to use an Elasticsearch API key +#xpack.management.elasticsearch.api_key: "id:api_key" +#xpack.management.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx +#xpack.management.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt" +#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file +#xpack.management.elasticsearch.ssl.truststore.password: password +# use either keystore.path/keystore.password or certificate/key configurations +#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file +#xpack.management.elasticsearch.ssl.keystore.password: password +#xpack.management.elasticsearch.ssl.certificate: /path/to/file +#xpack.management.elasticsearch.ssl.key: /path/to/certificate_key_file +#xpack.management.elasticsearch.ssl.cipher_suites: [] +#xpack.management.elasticsearch.ssl.verification_mode: full +#xpack.management.elasticsearch.sniffing: false +#xpack.management.logstash.poll_interval: 5s + +# X-Pack GeoIP Database Management +# https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update +#xpack.geoip.downloader.enabled: true +#xpack.geoip.downloader.endpoint: "https://geoip.elastic.co/v1/database" \ No newline at end of file diff --git a/workload/logstash/9.2.3/conf/supervisor.ini b/workload/logstash/9.2.3/conf/supervisor.ini new file mode 100644 index 0000000..b19fc63 --- /dev/null +++ b/workload/logstash/9.2.3/conf/supervisor.ini @@ -0,0 +1,7 @@ +[program:logstash] +command=/usr/local/lib/logstash/bin/logstash --path.settings /opt/logstash/conf +user=logstash +directory=/usr/local/lib/logstash +process_name=%(program_name)s +pidfile=/opt/logstash/system/server.pid +autorestart=false \ No newline at end of file