From a1985181bd93b858b85173dc7fbcb1102c924ba0 Mon Sep 17 00:00:00 2001 From: Saeid Hassanabadi Date: Wed, 5 Aug 2026 10:38:17 +0200 Subject: [PATCH] logstash: delete rotated log files at rollover The managed log4j2 configuration rotated files but never deleted them, so /var/log/logstash grew indefinitely (one .gz per day, per appender, since first deployment). The DefaultRolloverStrategy max attribute is not enough here because filePattern combines %d{yyyy-MM-dd} with %i, which restarts the counter on every new day. Add a Delete action to every RollingFile appender (plain, json, slowlog plain, slowlog json, deprecation and the per-pipeline routing appender), filtered by the appender's own filename glob and IfLastModified. This restores the behaviour Elastic ships for the plain and json appenders and extends it to the appenders where the vendor default is also incomplete. The retention is controlled by logstash_logging_retention, default 7D, matching Elastic. Any log4j2 IfLastModified duration works, e.g. 24H or P30D. --- roles/logstash/README.md | 1 + roles/logstash/defaults/main.yml | 1 + roles/logstash/meta/argument_specs.yml | 9 +++++ roles/logstash/templates/log4j2.properties.j2 | 36 +++++++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 561f7462..2cb77ed5 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -127,6 +127,7 @@ Run only parts of the role with `--tags`: | `logstash_logging_file` | `bool` | `true` | — | Log to the log file. Only effective when logstash_manage_logging is enabled. | | `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | | `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_retention` | `str` | `"7D"` | — | Age at which rotated log files (plain, json, slowlog, deprecation and per-pipeline routing logs) are deleted by log4j2 at rollover time. Uses log4j2 IfLastModified duration syntax such as "7D", "24H" or "P30D". Only effective when logstash_manage_logging is enabled. | | `logstash_ident` | `bool` | `true` | — | Add a field identifying the node that processed an event. | | `logstash_ident_field_name` | `str` | `"[netways][instance]"` | — | Name of the field that identifies the instance. | | `logstash_pipeline_identifier` | `bool` | `true` | — | Add a field identifying which pipeline processed an event. | diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index a347085b..49b3bf6a 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -87,6 +87,7 @@ logstash_logging_console: true logstash_logging_file: true logstash_logging_slow_console: true logstash_logging_slow_file: true +logstash_logging_retention: 7D logstash_ident: true logstash_ident_field_name: "[netways][instance]" diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 7fd244f1..eb2bbbed 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -457,6 +457,15 @@ argument_specs: default: true description: Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. + logstash_logging_retention: + type: str + default: 7D + description: >- + Age at which rotated log files (plain, json, slowlog, deprecation and + per-pipeline routing logs) are deleted by log4j2 at rollover time. + Uses log4j2 IfLastModified duration syntax such as "7D", "24H" or + "P30D". Only effective when logstash_manage_logging is enabled. + # ----- Identifying fields ----- logstash_ident: type: bool diff --git a/roles/logstash/templates/log4j2.properties.j2 b/roles/logstash/templates/log4j2.properties.j2 index 10a908e0..785c1903 100644 --- a/roles/logstash/templates/log4j2.properties.j2 +++ b/roles/logstash/templates/log4j2.properties.j2 @@ -39,6 +39,12 @@ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy appender.rolling.policies.size.size = 100MB appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.max = 30 +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:ls.logs} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = logstash-plain-* +appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified +appender.rolling.strategy.action.condition.nested_condition.age = {{ logstash_logging_retention }} appender.rolling.avoid_pipelined_filter.type = PipelineRoutingFilter appender.json_rolling.type = RollingFile @@ -56,6 +62,12 @@ appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.json_rolling.policies.size.size = 100MB appender.json_rolling.strategy.type = DefaultRolloverStrategy appender.json_rolling.strategy.max = 30 +appender.json_rolling.strategy.action.type = Delete +appender.json_rolling.strategy.action.basepath = ${sys:ls.logs} +appender.json_rolling.strategy.action.condition.type = IfFileName +appender.json_rolling.strategy.action.condition.glob = logstash-json-* +appender.json_rolling.strategy.action.condition.nested_condition.type = IfLastModified +appender.json_rolling.strategy.action.condition.nested_condition.age = {{ logstash_logging_retention }} appender.json_rolling.avoid_pipelined_filter.type = PipelineRoutingFilter appender.routing.type = PipelineRouting @@ -70,6 +82,12 @@ appender.routing.pipeline.policy.type = SizeBasedTriggeringPolicy appender.routing.pipeline.policy.size = 100MB appender.routing.pipeline.strategy.type = DefaultRolloverStrategy appender.routing.pipeline.strategy.max = 30 +appender.routing.pipeline.strategy.action.type = Delete +appender.routing.pipeline.strategy.action.basepath = ${sys:ls.logs} +appender.routing.pipeline.strategy.action.condition.type = IfFileName +appender.routing.pipeline.strategy.action.condition.glob = pipeline_${ctx:pipeline.id}*.log.gz +appender.routing.pipeline.strategy.action.condition.nested_condition.type = IfLastModified +appender.routing.pipeline.strategy.action.condition.nested_condition.age = {{ logstash_logging_retention }} rootLogger.level = ${sys:ls.log.level} {% if logstash_logging_console | bool %} @@ -107,6 +125,12 @@ appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy appender.rolling_slowlog.policies.size.size = 100MB appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy appender.rolling_slowlog.strategy.max = 30 +appender.rolling_slowlog.strategy.action.type = Delete +appender.rolling_slowlog.strategy.action.basepath = ${sys:ls.logs} +appender.rolling_slowlog.strategy.action.condition.type = IfFileName +appender.rolling_slowlog.strategy.action.condition.glob = logstash-slowlog-plain-* +appender.rolling_slowlog.strategy.action.condition.nested_condition.type = IfLastModified +appender.rolling_slowlog.strategy.action.condition.nested_condition.age = {{ logstash_logging_retention }} appender.json_rolling_slowlog.type = RollingFile appender.json_rolling_slowlog.name = json_rolling_slowlog @@ -123,6 +147,12 @@ appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy appender.json_rolling_slowlog.policies.size.size = 100MB appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy appender.json_rolling_slowlog.strategy.max = 30 +appender.json_rolling_slowlog.strategy.action.type = Delete +appender.json_rolling_slowlog.strategy.action.basepath = ${sys:ls.logs} +appender.json_rolling_slowlog.strategy.action.condition.type = IfFileName +appender.json_rolling_slowlog.strategy.action.condition.glob = logstash-slowlog-json-* +appender.json_rolling_slowlog.strategy.action.condition.nested_condition.type = IfLastModified +appender.json_rolling_slowlog.strategy.action.condition.nested_condition.age = {{ logstash_logging_retention }} logger.slowlog.name = slowlog logger.slowlog.level = trace @@ -156,6 +186,12 @@ appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.deprecation_rolling.policies.size.size = 100MB appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 30 +appender.deprecation_rolling.strategy.action.type = Delete +appender.deprecation_rolling.strategy.action.basepath = ${sys:ls.logs} +appender.deprecation_rolling.strategy.action.condition.type = IfFileName +appender.deprecation_rolling.strategy.action.condition.glob = logstash-deprecation-* +appender.deprecation_rolling.strategy.action.condition.nested_condition.type = IfLastModified +appender.deprecation_rolling.strategy.action.condition.nested_condition.age = {{ logstash_logging_retention }} logger.deprecation.name = org.logstash.deprecation, deprecation logger.deprecation.level = WARN