diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 6294fd04..04c66313 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version + - name: Install systemd development headers + run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run compliance checks env: RSHELL_COMPLIANCE_TEST: "1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1d909e9b..d3ab86d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,9 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version + - name: Install systemd development headers + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run tests with race detector run: go test -race -v -timeout 10m ./... @@ -69,6 +72,8 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version + - name: Install systemd development headers + run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run bash comparison tests env: RSHELL_BASH_TEST: "1" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index e5324d09..8e5bcbd7 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -9,7 +9,9 @@ github.com/ebitengine/purego,https://github.com/ebitengine/purego,Apache-2.0,Cop github.com/go-ole/go-ole,https://github.com/go-ole/go-ole,MIT,"Copyright (c) 2013-2017 Yasuhiro Matsumoto" github.com/google/uuid,https://github.com/google/uuid,BSD-3-Clause,Copyright (c) 2018 Google Inc. github.com/inconshreveable/mousetrap,https://github.com/inconshreveable/mousetrap,Apache-2.0,Copyright 2014 Alan Shreve +github.com/klauspost/compress,https://github.com/klauspost/compress,BSD-3-Clause,"Copyright (c) 2012 The Go Authors; Copyright (c) 2019 Klaus Post" github.com/lufia/plan9stats,https://github.com/lufia/plan9stats,BSD-3-Clause,"Copyright (c) 2019, KADOTA, Kyohei" +github.com/pierrec/lz4/v4,https://github.com/pierrec/lz4,BSD-3-Clause,"Copyright (c) 2015, Pierre Curto" github.com/pmezard/go-difflib,https://github.com/pmezard/go-difflib,BSD-3-Clause,"Copyright (c) 2013, Patrick Mezard" github.com/power-devops/perfstat,https://github.com/power-devops/perfstat,MIT,Copyright (c) 2020 Power DevOps github.com/prometheus-community/pro-bing,https://github.com/prometheus-community/pro-bing,MIT,"Copyright 2022 The Prometheus Authors; Copyright 2016 Cameron Sparr and contributors" @@ -19,6 +21,7 @@ github.com/spf13/pflag,https://github.com/spf13/pflag,BSD-3-Clause,"Copyright (c github.com/stretchr/testify,https://github.com/stretchr/testify,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors" github.com/tklauser/go-sysconf,https://github.com/tklauser/go-sysconf,BSD-3-Clause,"Copyright (c) 2018-2022, Tobias Klauser" github.com/tklauser/numcpus,https://github.com/tklauser/numcpus,Apache-2.0,"Copyright (c) 2018-2022, Tobias Klauser" +github.com/ulikunitz/xz,https://github.com/ulikunitz/xz,BSD-3-Clause,"Copyright (c) 2014-2022, Ulrich Kunitz" github.com/yusufpapurcu/wmi,https://github.com/yusufpapurcu/wmi,MIT,Copyright (c) 2013 Stack Exchange go.uber.org/atomic,https://github.com/uber-go/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc." go.yaml.in/yaml/v3,https://github.com/yaml/go-yaml,MIT AND Apache-2.0,"Copyright (c) 2006-2011 Kirill Simonov, 2011-2019 Canonical Ltd" diff --git a/README.md b/README.md index 0a2c197b..cad51a70 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Every access path is default-deny: | Resource | Default | Opt-in | |----------------------|-------------------------------------|----------------------------------------------| | Command execution | All commands blocked (exit code 127)| `AllowedCommands` with namespaced command list (e.g. `rshell:cat`) | -| System services | All services and actions blocked | `AllowedSystemServices` with exact service/action grants | +| Systemd services | All services and actions blocked | `AllowedSystemServices` with exact service/action grants | | External commands | Blocked (exit code 127) | Provide an `ExecHandler` | | Filesystem access | Blocked | Configure `AllowedPaths` with `PATH[:ro|:rw]` root specs | | Environment variables| Empty (no host env inherited) | Pass variables via the `Env` option | @@ -69,22 +69,58 @@ Every access path is default-deny: **AllowedCommands** restricts which commands (builtins or external) the interpreter may execute. Commands must be specified with the `rshell:` namespace prefix (e.g. `rshell:cat`, `rshell:echo`). If not set, no commands are allowed. -**AllowedSystemServices** configures exact service/action grants for future system-service builtins. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different grants. The supported actions are `read`, `reload`, and `restart`. Grants without actions are ignored; entries with empty, whitespace-containing, path-like, or glob-pattern service names and unsupported actions are skipped with a warning. The policy defaults to denying every service, remains enforced when all commands are allowed, and requires remediation mode before any grant can be authorized. +**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact service with generic actions (`read`, `clean`, `reload`, or `restart`), using `SERVICE:ACTION[+ACTION...]` syntax. Grants without actions are ignored; invalid services and unsupported actions are skipped with warnings. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different services. Empty service names, service names containing `:`, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go -interp.AllowedSystemServices([]interp.SystemServiceControlGrant{ +interp.AllowedSystemServices([]interp.SystemdControlGrant{ { Service: "mysql.service", - Actions: []interp.SystemServiceAction{ - interp.SystemServiceRestart, - interp.SystemServiceReload, - interp.SystemServiceRead, + Actions: []interp.SystemdAction{ + interp.SystemdActionRestart, + interp.SystemdActionReload, + interp.SystemdActionRead, }, }, + { + Service: "systemd-journald.service", + Actions: []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionClean}, + }, }) ``` -The development CLI accepts the equivalent syntax through `--allowed-services mysql.service:restart+reload+read,nginx.service:read`. The policy and authorization capability are implemented, but `systemctl` and `journalctl` builtins are not yet available. +The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,systemd-journald.service:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. + +**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. + +The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Every selected journal file header is checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. + +| Operation | Required target access | +|-----------|------------------------| +| Journal query, disk usage, or vacuum | `/etc/machine-id` and one or both of `/var/log/journal`, `/run/log/journal` | +| Journal rotation | `/etc/machine-id` and `/run/systemd/journal/io.systemd.journal` | +| Future `systemctl` manager operations | `/etc/machine-id` and `/run/dbus/system_bus_socket` | + +Root targets derive those paths below `Root`; explicit targets may map them to arbitrary absolute container paths. A command fails closed when one of its required paths was omitted. + +### Restricted journalctl + +`journalctl` provides bounded investigation and disk-cleanup operations without executing the host binary or accepting user-selected files, directories, journal matches, cursors, machines, or namespaces. + +| Operation | Supported flags | Required systemd grant | +|-----------|-----------------|------------------------| +| Exact service logs | `-u SERVICE` (repeatable), `-b`, `-n COUNT`, `--since TIME`, `-o short\|cat` | Exact `SERVICE:read` grant for every service | +| Current-boot kernel logs | `-k`, `-n COUNT`, `--since TIME`, `-o short\|cat` | `systemd-journald.service:read` | +| Allocated journal usage | `--disk-usage` | `systemd-journald.service:read` | +| Synchronous active-file rotation | `--rotate` | `systemd-journald.service:clean` plus remediation mode | +| Archived-file cleanup | `--vacuum-size SIZE`, `--vacuum-time AGE`, `--dry-run` | `systemd-journald.service:clean` plus remediation mode | + +Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. + +Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. + +The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--vacuum-size` requires `--vacuum-time`; size cleanup deletes only archives at least that old and stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. + +Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. **AllowedPaths** restricts all file operations to specified directories using Go's `os.Root` API for reads and openat-based write handling for writes. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 3833ae17..85ee2497 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,6 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; size vacuum requires a time floor and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected @@ -121,7 +122,8 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c ## Execution - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed -- ✅ AllowedSystemServices policy — configures exact, case-sensitive `read`, `reload`, and `restart` grants for system service names through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]`; names are not normalized (`mysql` does not match `mysql.service`), grants without actions are ignored, invalid names and unsupported actions are skipped with warnings, all services are denied by default, and allowing all commands does not bypass this policy; the authorization capability is implemented for future consumers, but `systemctl` and `journalctl` builtins are not yet available +- ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services with generic `read`, `clean`, `reload`, and `restart` actions; actionless grants are ignored, invalid services/actions are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` +- ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command - ✅ ProcPath — overrides the proc filesystem path used by `ps` (default `/proc`; Linux-only; useful for testing/container environments); `ps` does not read `/proc//cmdline` diff --git a/analysis/symbols_builtins.go b/analysis/symbols_builtins.go index b0fb36be..6f11d710 100644 --- a/analysis/symbols_builtins.go +++ b/analysis/symbols_builtins.go @@ -194,6 +194,24 @@ var builtinPerCommandSymbols = map[string][]string{ "os.O_RDONLY", // 🟢 read-only file flag constant; cannot open files by itself. "strconv.ParseInt", // 🟢 string-to-int conversion with base/bit-size; pure function, no I/O. }, + "journalctl": { + "context.Context", // 🟢 deadline/cancellation plumbing; pure interface, no side effects. + "fmt.Sprintf", // 🟢 formats bounded journal storage counts in memory; no I/O. + "io.WriteString", // 🟠 writes formatted journal lines to callCtx.Stdout; no filesystem access, delegates to Write. + "io.Writer", // 🟢 interface type for the already-authorized output stream; no side effects. + "strconv.ParseInt", // 🟢 parses the bounded --lines value; pure function, no I/O. + "strings.Builder", // 🟢 constructs one bounded output line in memory; no I/O. + "time.Parse", // 🟢 parses an absolute RFC3339 timestamp; pure function, no I/O. + "time.ParseDuration", // 🟢 parses a bounded lookback duration; pure function, no I/O. + "time.ParseInLocation", // 🟢 parses a local timestamp using the runner's captured location; pure function, no I/O. + "time.RFC3339Nano", // 🟢 standard timestamp layout constant; no side effects. + "time.Time", // 🟢 time value type; pure data, no side effects. + "unicode.IsGraphic", // 🟢 identifies non-graphic runes that must be escaped at the output boundary; pure function, no I/O. + "unicode/utf8.DecodeRuneInString", // 🟢 decodes output text so control and malformed bytes can be escaped; pure function, no I/O. + "unicode/utf8.RuneError", // 🟢 replacement rune constant used to detect malformed UTF-8; no side effects. + // Note: builtins/internal/sizeparse symbols are exempt from this + // allowlist (internal packages are checked separately). + }, "logrotate": { "context.Context", // 🟢 deadline/cancellation plumbing; pure interface, no side effects. "errors.Is", // 🟢 error comparison; pure function, no I/O. @@ -528,15 +546,17 @@ var builtinPerCommandSymbols = map[string][]string{ }, } -// callCtxAllFields lists every function-typed field of CallContext that the -// analyzer tracks. Plain data fields (Stdin, Stdout, Stderr, Now, InLoop, +// callCtxAllFields lists every elevated-capability field of CallContext that +// the analyzer tracks. Plain data fields (Stdin, Stdout, Stderr, Now, InLoop, // LastExitCode, Proc) are not tracked; they are universally available and -// carry no elevated capability. +// carry no elevated capability. Systemd is tracked despite being a pointer +// because it provides privileged host introspection backends. // // Every entry must match a field name declared in builtins/builtins.go. var callCtxAllFields = []string{ "AccessFile", "AllowedPathsList", + "AuthorizeSystemd", "CanonicalizeRootPrefix", "ChangeDir", "CommandAllowed", @@ -556,6 +576,7 @@ var callCtxAllFields = []string{ "RunCommandWithStdin", "SetVar", "StatFile", + "Systemd", "Truncate", "TruncateToZeroIfAtLeast", "WorkDir", @@ -636,6 +657,10 @@ var builtinPerCommandCallContextFields = map[string][]string{ "ip": { "PortableErr", }, + "journalctl": { + "AuthorizeSystemd", + "Systemd", + }, "logrotate": { "PortableErr", "TruncateToZeroIfAtLeast", @@ -846,7 +871,10 @@ var builtinAllowedSymbols = []string{ "time.Hour", // 🟢 constant representing one hour; no side effects. "time.Millisecond", // 🟢 constant representing one millisecond; no side effects. "time.Minute", // 🟢 constant representing one minute; no side effects. + "time.Parse", // 🟢 parses timestamps according to a caller-supplied layout; pure function, no I/O. "time.ParseDuration", // 🟢 parses Go duration strings (e.g. "1s"); pure function, no I/O. + "time.ParseInLocation", // 🟢 parses timestamps in a caller-supplied location; pure function, no I/O. + "time.RFC3339Nano", // 🟢 standard RFC3339 timestamp layout with optional fractional seconds; pure constant. "time.Second", // 🟢 constant representing one second; no side effects. "time.Time", // 🟢 time value type; pure data, no side effects. "time.Unix", // 🟢 constructs an absolute Time at Unix-epoch + (sec, nsec); pure constructor, no I/O or side effects. diff --git a/builtins/builtins.go b/builtins/builtins.go index 2cee22b9..f536f537 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -57,14 +57,39 @@ type AllowedPath struct { Access AllowedPathAccess } -// SystemServiceAction identifies an operation that a builtin may perform on -// an explicitly configured system service. -type SystemServiceAction string +// SystemdAction identifies an operation that a builtin may perform on an +// explicitly configured systemd service. +type SystemdAction string const ( - SystemServiceRead SystemServiceAction = "read" - SystemServiceReload SystemServiceAction = "reload" - SystemServiceRestart SystemServiceAction = "restart" + SystemdActionRead SystemdAction = "read" + SystemdActionClean SystemdAction = "clean" + SystemdActionReload SystemdAction = "reload" + SystemdActionRestart SystemdAction = "restart" +) + +const ( + // SystemdJournaldService is the exact service name used for journal-wide + // operations such as kernel log reads, disk usage, rotation, and vacuuming. + SystemdJournaldService = "systemd-journald.service" +) + +// SystemdOperation is one service action that must be authorized before a +// builtin interacts with systemd. +type SystemdOperation struct { + Service string + Action SystemdAction +} + +// Deprecated compatibility aliases for the service-only policy introduced +// before the shared systemd capability model. +type SystemServiceAction = SystemdAction + +const ( + SystemServiceRead = SystemdActionRead + SystemServiceClean = SystemdActionClean + SystemServiceReload = SystemdActionReload + SystemServiceRestart = SystemdActionRestart ) // Command pairs a builtin name with its flag-declaring factory. MakeFlags @@ -255,10 +280,15 @@ type CallContext struct { // commands. CommandAllowed func(name string) bool - // AuthorizeSystemServices reports whether all named services may be used - // for action under the current shell policy. Implementations must authorize - // the complete list before a builtin performs any operation so multi-service - // requests cannot partially execute. Service names are matched exactly. + // AuthorizeSystemd reports whether every operation may be performed under + // the current shell policy. Implementations must authorize the complete + // list before a builtin performs any operation so compound requests cannot + // partially execute. + AuthorizeSystemd func(operations ...SystemdOperation) error + + // AuthorizeSystemServices is the deprecated unit-only authorization + // capability retained for compatibility with callers built against the + // original service allowlist API. AuthorizeSystemServices func(action SystemServiceAction, services ...string) error // AllowedPathsList returns the resolved absolute paths and configured @@ -331,6 +361,10 @@ type CallContext struct { // Proc provides access to the proc filesystem for the ps builtin. // The path is fixed at construction time and cannot be overridden by callers. Proc *ProcProvider + + // Systemd contains structured backends for systemd-aware builtins. Target + // paths and transports are fixed by trusted runner configuration. + Systemd *SystemdServices } // Out writes a string to stdout. diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go new file mode 100644 index 00000000..09a2c7a6 --- /dev/null +++ b/builtins/journalctl/journalctl.go @@ -0,0 +1,452 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +// Package journalctl implements a bounded systemd journal query builtin. +// +// Usage: journalctl (-u UNIT...|-k) [OPTION]... +// +// This is a deliberately restricted journalctl subset. Every log query must +// select one or more exact system units, or the current boot's kernel log. +// Arbitrary field matches, unrestricted journal reads, follow mode, alternate +// files/directories, cursors, and machine/namespace selection are not exposed. +// +// Accepted flags: +// +// -u, --unit=UNIT select an exact system unit; may be repeated +// -k, --dmesg select kernel messages from the current target boot +// -b, --boot restrict a unit query to the current target boot +// -n, --lines=COUNT show at most COUNT entries (default 100, maximum 1000) +// -S, --since=TIME show entries since an RFC3339 timestamp, local +// YYYY-MM-DD HH:MM:SS timestamp, or lookback duration +// -o, --output=FORMAT output format: short (default) or cat +// --disk-usage show allocated journal storage and exit +// --rotate archive active journal files before returning +// --vacuum-size=SIZE remove oldest eligible archives toward allocated SIZE; +// requires --vacuum-time +// --vacuum-time=AGE remove archives older than Go duration AGE and limit +// size vacuum eligibility +// --dry-run report cleanup without deleting archives +// -h, --help print usage and exit +package journalctl + +import ( + "context" + "fmt" + "io" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/DataDog/rshell/builtins" + "github.com/DataDog/rshell/builtins/internal/sizeparse" +) + +const ( + shortTime = "Jan 02 15:04:05" + localSinceTime = "2006-01-02 15:04:05" +) + +// Cmd is the journalctl builtin command descriptor. +var Cmd = builtins.Command{ + Name: "journalctl", + Description: "query and maintain bounded systemd journals", + MakeFlags: makeFlags, +} + +type flags struct { + units *[]string + kernel *bool + boot *bool + lines *string + since *string + output *string + usage *bool + rotate *bool + vacuumSize *string + vacuumTime *string + dryRun *bool + help *bool +} + +func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { + options := flags{ + units: fs.StringArrayP("unit", "u", nil, "show logs for exact UNIT (repeatable)"), + kernel: fs.BoolP("dmesg", "k", false, "show current-boot kernel messages"), + boot: fs.BoolP("boot", "b", false, "show messages from the current boot"), + lines: fs.StringP("lines", "n", "100", "show at most COUNT entries (maximum 1000)"), + since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), + output: fs.StringP("output", "o", "short", "output format: short or cat"), + usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), + rotate: fs.Bool("rotate", false, "archive active journal files and wait for completion"), + vacuumSize: fs.String("vacuum-size", "", "remove oldest eligible archives toward allocated SIZE; requires --vacuum-time"), + vacuumTime: fs.String("vacuum-time", "", "remove archives older than Go duration AGE and limit size vacuum eligibility"), + dryRun: fs.Bool("dry-run", false, "report cleanup without deleting archives"), + help: fs.BoolP("help", "h", false, "print usage and exit"), + } + return options.run(fs) +} + +func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { + return func(ctx context.Context, callCtx *builtins.CallContext, args []string) builtins.Result { + if *options.help { + callCtx.Out("Usage: journalctl (-u UNIT...|-k) [OPTION]...\n") + callCtx.Out("Show a bounded selection of logs from the configured systemd target.\n") + callCtx.Out("Queries require exact unit scopes or the current-boot kernel scope.\n\n") + fs.SetOutput(callCtx.Stdout) + fs.PrintDefaults() + return builtins.Result{} + } + if len(args) > 0 { + callCtx.Errf("journalctl: unexpected argument %q; arbitrary journal matches are not supported\n", args[0]) + return builtins.Result{Code: 1} + } + if *options.usage { + return runDiskUsage(ctx, callCtx, fs) + } + if *options.rotate || fs.Changed("vacuum-size") || fs.Changed("vacuum-time") || *options.dryRun { + return options.runMaintenance(ctx, callCtx, fs) + } + if len(*options.units) > builtins.MaxJournalQueryUnits { + callCtx.Errf("journalctl: too many unit scopes (maximum %d)\n", builtins.MaxJournalQueryUnits) + return builtins.Result{Code: 1} + } + + units := uniqueUnits(*options.units) + if *options.kernel && len(units) > 0 { + callCtx.Errf("journalctl: --dmesg cannot be combined with --unit\n") + return builtins.Result{Code: 1} + } + if !*options.kernel && len(units) == 0 { + callCtx.Errf("journalctl: an exact --unit or --dmesg scope is required\n") + return builtins.Result{Code: 1} + } + + lines, ok := parseLineCount(*options.lines) + if !ok { + callCtx.Errf("journalctl: invalid line count %q (must be between 0 and %d)\n", *options.lines, builtins.MaxJournalQueryEntries) + return builtins.Result{Code: 1} + } + if *options.output != "short" && *options.output != "cat" { + callCtx.Errf("journalctl: unsupported output format %q (supported: short, cat)\n", *options.output) + return builtins.Result{Code: 1} + } + + var since time.Time + if fs.Changed("since") { + if since, ok = parseSince(*options.since, callCtx.Now); !ok { + callCtx.Errf("journalctl: invalid --since value %q; use RFC3339, YYYY-MM-DD HH:MM:SS, or a lookback duration such as 15m\n", *options.since) + return builtins.Result{Code: 1} + } + } + + operations := make([]builtins.SystemdOperation, 0, len(units)+1) + if *options.kernel { + operations = append(operations, builtins.SystemdOperation{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, + }) + } else { + for _, unit := range units { + operations = append(operations, builtins.SystemdOperation{ + Service: unit, + Action: builtins.SystemdActionRead, + }) + } + } + if callCtx.AuthorizeSystemd == nil { + callCtx.Errf("journalctl: systemd authorization capability is not available\n") + return builtins.Result{Code: 1} + } + if err := callCtx.AuthorizeSystemd(operations...); err != nil { + callCtx.Errf("journalctl: %s\n", err) + return builtins.Result{Code: 1} + } + if callCtx.Systemd == nil || callCtx.Systemd.Journal == nil { + callCtx.Errf("journalctl: systemd journal capability is not available\n") + return builtins.Result{Code: 1} + } + + query := builtins.JournalQuery{ + Units: units, + Kernel: *options.kernel, + CurrentBoot: *options.kernel || *options.boot, + Since: since, + MaxEntries: lines, + } + err := callCtx.Systemd.Journal.ReadJournal(ctx, query, func(entry builtins.JournalEntry) error { + return writeEntry(callCtx.Stdout, entry, *options.output) + }) + if err == nil || builtins.IsBrokenPipe(err) { + return builtins.Result{} + } + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } +} + +func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { + for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "rotate", "vacuum-size", "vacuum-time", "dry-run"} { + if fs.Changed(flagName) { + callCtx.Errf("journalctl: --disk-usage cannot be combined with --%s\n", flagName) + return builtins.Result{Code: 1} + } + } + if callCtx.AuthorizeSystemd == nil { + callCtx.Errf("journalctl: systemd authorization capability is not available\n") + return builtins.Result{Code: 1} + } + err := callCtx.AuthorizeSystemd(builtins.SystemdOperation{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, + }) + if err != nil { + callCtx.Errf("journalctl: %s\n", err) + return builtins.Result{Code: 1} + } + if callCtx.Systemd == nil || callCtx.Systemd.JournalStorage == nil { + callCtx.Errf("journalctl: systemd journal storage capability is not available\n") + return builtins.Result{Code: 1} + } + usage, err := callCtx.Systemd.JournalStorage.JournalDiskUsage(ctx) + if err != nil { + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } + callCtx.Outf("Archived and active journals take up %s in the file system.\n", formatUsage(usage.Bytes)) + return builtins.Result{} +} + +func (options flags) runMaintenance(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { + for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "disk-usage"} { + if fs.Changed(flagName) { + callCtx.Errf("journalctl: journal maintenance cannot be combined with --%s\n", flagName) + return builtins.Result{Code: 1} + } + } + sizeSet := fs.Changed("vacuum-size") + timeSet := fs.Changed("vacuum-time") + vacuumSet := sizeSet || timeSet + if *options.dryRun && *options.rotate { + callCtx.Errf("journalctl: --dry-run cannot be combined with --rotate\n") + return builtins.Result{Code: 1} + } + if *options.dryRun && !vacuumSet { + callCtx.Errf("journalctl: --dry-run requires --vacuum-size or --vacuum-time\n") + return builtins.Result{Code: 1} + } + if vacuumSet && callCtx.Now.IsZero() { + callCtx.Errf("journalctl: journal vacuum requires a runner reference time\n") + return builtins.Result{Code: 1} + } + + request := builtins.JournalVacuumRequest{} + if vacuumSet { + request.Now = callCtx.Now + request.DryRun = *options.dryRun + } + if sizeSet { + size, err := sizeparse.Parse(*options.vacuumSize) + if err != nil || size <= 0 { + callCtx.Errf("journalctl: invalid --vacuum-size value %q (must be greater than zero)\n", *options.vacuumSize) + return builtins.Result{Code: 1} + } + request.MaxBytes = uint64(size) + } + if sizeSet && !timeSet { + callCtx.Errf("journalctl: --vacuum-size requires --vacuum-time to protect recently modified archives\n") + return builtins.Result{Code: 1} + } + if timeSet { + age, err := time.ParseDuration(*options.vacuumTime) + if err != nil || age <= 0 { + callCtx.Errf("journalctl: invalid --vacuum-time value %q (use a positive Go duration such as 168h)\n", *options.vacuumTime) + return builtins.Result{Code: 1} + } + request.Before = callCtx.Now.Add(-age) + } + + if callCtx.AuthorizeSystemd == nil { + callCtx.Errf("journalctl: systemd authorization capability is not available\n") + return builtins.Result{Code: 1} + } + err := callCtx.AuthorizeSystemd(builtins.SystemdOperation{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionClean, + }) + if err != nil { + callCtx.Errf("journalctl: %s\n", err) + return builtins.Result{Code: 1} + } + if *options.rotate && (callCtx.Systemd == nil || callCtx.Systemd.JournalRotator == nil) { + callCtx.Errf("journalctl: systemd journal rotation capability is not available\n") + return builtins.Result{Code: 1} + } + if vacuumSet && (callCtx.Systemd == nil || callCtx.Systemd.JournalCleaner == nil) { + callCtx.Errf("journalctl: systemd journal cleaner capability is not available\n") + return builtins.Result{Code: 1} + } + if *options.rotate { + if err := callCtx.Systemd.JournalRotator.RotateJournal(ctx); err != nil { + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } + callCtx.Out("Journal rotation completed.\n") + } + if !vacuumSet { + return builtins.Result{} + } + result, err := callCtx.Systemd.JournalCleaner.VacuumJournal(ctx, request) + if err != nil { + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } + verb := "done, freed" + if request.DryRun { + verb = "would free" + } + noun := "files" + if result.Files == 1 { + noun = "file" + } + callCtx.Outf("Vacuuming %s %s from %d archived journal %s.\n", verb, formatUsage(result.Bytes), result.Files, noun) + return builtins.Result{} +} + +func formatUsage(bytes uint64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%dB", bytes) + } + value := float64(bytes) + for _, suffix := range []string{"K", "M", "G", "T", "P", "E"} { + value /= unit + if value < unit || suffix == "E" { + return fmt.Sprintf("%.1f%s", value, suffix) + } + } + return fmt.Sprintf("%dB", bytes) +} + +func uniqueUnits(units []string) []string { + unique := make([]string, 0, len(units)) + seen := make(map[string]struct{}, len(units)) + for _, unit := range units { + if _, exists := seen[unit]; exists { + continue + } + seen[unit] = struct{}{} + unique = append(unique, unit) + } + return unique +} + +func parseLineCount(value string) (int, bool) { + if value == "" || value[0] == '+' { + return 0, false + } + parsed, err := strconv.ParseInt(value, 10, 32) + if err != nil || parsed < 0 || parsed > builtins.MaxJournalQueryEntries { + return 0, false + } + return int(parsed), true +} + +func parseSince(value string, now time.Time) (time.Time, bool) { + if value == "" { + return time.Time{}, false + } + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { + return parsed, true + } + if !now.IsZero() { + if parsed, err := time.ParseInLocation(localSinceTime, value, now.Location()); err == nil { + return parsed, true + } + if lookback, err := time.ParseDuration(value); err == nil && lookback >= 0 { + return now.Add(-lookback), true + } + } + return time.Time{}, false +} + +func writeEntry(writer io.Writer, entry builtins.JournalEntry, output string) error { + var line strings.Builder + if output == "short" { + line.WriteString(entry.Timestamp.Format(shortTime)) + line.WriteByte(' ') + if entry.Hostname == "" { + line.WriteByte('-') + } else { + appendEscaped(&line, entry.Hostname) + } + line.WriteByte(' ') + if entry.Identifier == "" { + line.WriteString("journal") + } else { + appendEscaped(&line, entry.Identifier) + } + if entry.PID != "" { + line.WriteByte('[') + appendEscaped(&line, entry.PID) + line.WriteByte(']') + } + line.WriteString(": ") + } + appendEscaped(&line, entry.Message) + line.WriteByte('\n') + _, err := io.WriteString(writer, line.String()) + return err +} + +func appendEscaped(output *strings.Builder, value string) { + for value != "" { + r, size := utf8.DecodeRuneInString(value) + if r == utf8.RuneError && size == 1 { + appendHexEscape(output, 'x', uint32(value[0]), 2) + value = value[1:] + continue + } + switch r { + case '\n': + output.WriteString("\\n") + case '\r': + output.WriteString("\\r") + case '\t': + output.WriteString("\\t") + default: + if !unicode.IsGraphic(r) { + if r <= 0xff { + appendHexEscape(output, 'x', uint32(r), 2) + } else if r <= 0xffff { + appendHexEscape(output, 'u', uint32(r), 4) + } else { + appendHexEscape(output, 'U', uint32(r), 8) + } + } else { + output.WriteString(value[:size]) + } + } + value = value[size:] + } +} + +func appendHexEscape(output *strings.Builder, prefix byte, value uint32, width int) { + const hex = "0123456789abcdef" + output.WriteByte('\\') + output.WriteByte(prefix) + for shift := (width - 1) * 4; shift >= 0; shift -= 4 { + output.WriteByte(hex[(value>>shift)&0x0f]) + } +} diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go new file mode 100644 index 00000000..7a3e1b24 --- /dev/null +++ b/builtins/journalctl/journalctl_test.go @@ -0,0 +1,619 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package journalctl + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "syscall" + "testing" + "time" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +type fakeJournalReader struct { + queries []builtins.JournalQuery + entries []builtins.JournalEntry + err error +} + +type fakeJournalStorage struct { + usage builtins.JournalUsage + err error + calls int +} + +type fakeJournalCleaner struct { + result builtins.JournalVacuumResult + err error + requests []builtins.JournalVacuumRequest + order *[]string +} + +type fakeJournalRotator struct { + err error + calls int + order *[]string +} + +func (c *fakeJournalCleaner) VacuumJournal(_ context.Context, request builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + if c.order != nil { + *c.order = append(*c.order, "vacuum") + } + c.requests = append(c.requests, request) + return c.result, c.err +} + +func (r *fakeJournalRotator) RotateJournal(context.Context) error { + if r.order != nil { + *r.order = append(*r.order, "rotate") + } + r.calls++ + return r.err +} + +func (s *fakeJournalStorage) JournalDiskUsage(context.Context) (builtins.JournalUsage, error) { + s.calls++ + return s.usage, s.err +} + +func (r *fakeJournalReader) ReadJournal(_ context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { + r.queries = append(r.queries, query) + for _, entry := range r.entries { + if err := yield(entry); err != nil { + return err + } + } + return r.err +} + +func runJournalctl(t *testing.T, args []string, callCtx *builtins.CallContext) builtins.Result { + t.Helper() + fs := pflag.NewFlagSet("journalctl", pflag.ContinueOnError) + fs.SetOutput(io.Discard) + handler := makeFlags(fs) + require.NoError(t, fs.Parse(args)) + return handler(context.Background(), callCtx, fs.Args()) +} + +func TestJournalctlBuildsBoundedUnitQuery(t *testing.T) { + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{Message: "ready"}}} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + callCtx := &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: now, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append([]builtins.SystemdOperation(nil), operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + } + + result := runJournalctl(t, []string{ + "-u", "api.service", + "-u", "worker.service", + "-u", "api.service", + "-b", + "-n", "2", + "--since", "15m", + "--output", "cat", + }, callCtx) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "ready\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{ + {Service: "api.service", Action: builtins.SystemdActionRead}, + {Service: "worker.service", Action: builtins.SystemdActionRead}, + }, authorized) + require.Len(t, reader.queries, 1) + assert.Equal(t, builtins.JournalQuery{ + Units: []string{"api.service", "worker.service"}, + CurrentBoot: true, + Since: now.Add(-15 * time.Minute), + MaxEntries: 2, + }, reader.queries[0]) +} + +func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{ + Timestamp: time.Date(2026, time.July, 14, 12, 34, 56, 0, time.UTC), + Hostname: "host\tname", + Identifier: "kernel\x1b", + PID: "\xff", + Message: "cafe\u0301\nline\x00\u202e", + }}} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"-k", "-n1"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Jul 14 12:34:56 host\\tname kernel\\x1b[\\xff]: cafe\u0301\\nline\\x00\\u202e\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, + }}, authorized) + require.Len(t, reader.queries, 1) + assert.True(t, reader.queries[0].Kernel) + assert.True(t, reader.queries[0].CurrentBoot) +} + +func TestJournalctlDiskUsageUsesStorageReadCapability(t *testing.T) { + storage := &fakeJournalStorage{usage: builtins.JournalUsage{Bytes: 5 * 1024 * 1024, Files: 3}} + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"--disk-usage"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader, JournalStorage: storage}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Archived and active journals take up 5.0M in the file system.\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, + }}, authorized) + assert.Equal(t, 1, storage.calls) + assert.Empty(t, reader.queries) +} + +func TestJournalctlDiskUsageIsExclusive(t *testing.T) { + for _, args := range [][]string{ + {"--disk-usage", "-u", "api.service"}, + {"--disk-usage", "-k"}, + {"--disk-usage", "-b"}, + {"--disk-usage", "-n", "10"}, + {"--disk-usage", "--since", "1h"}, + {"--disk-usage", "-o", "cat"}, + {"--disk-usage", "--rotate"}, + } { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + storage := &fakeJournalStorage{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Systemd: &builtins.SystemdServices{JournalStorage: storage}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), "cannot be combined") + assert.Zero(t, storage.calls) + }) + } +} + +func TestJournalctlRotateUsesStorageCleanCapability(t *testing.T) { + rotator := &fakeJournalRotator{} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"--rotate"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Journal rotation completed.\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionClean, + }}, authorized) + assert.Equal(t, 1, rotator.calls) +} + +func TestJournalctlRotateRunsBeforeVacuum(t *testing.T) { + order := []string{} + rotator := &fakeJournalRotator{order: &order} + cleaner := &fakeJournalCleaner{order: &order, result: builtins.JournalVacuumResult{Files: 1, Bytes: 1024}} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator, JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, []string{"rotate", "vacuum"}, order) + assert.Equal(t, "Journal rotation completed.\nVacuuming done, freed 1.0K from 1 archived journal file.\n", stdout.String()) +} + +func TestJournalctlRotationFailurePreventsVacuum(t *testing.T) { + rotator := &fakeJournalRotator{err: errors.New("rotation failed")} + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator, JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), "rotation failed") + assert.Equal(t, 1, rotator.calls) + assert.Empty(t, cleaner.requests) +} + +func TestJournalctlCombinedMaintenanceChecksCapabilitiesBeforeMutation(t *testing.T) { + t.Run("missing cleaner", func(t *testing.T) { + rotator := &fakeJournalRotator{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "cleaner capability is not available") + assert.Zero(t, rotator.calls) + }) + + t.Run("missing rotator", func(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "rotation capability is not available") + assert.Empty(t, cleaner.requests) + }) +} + +func TestJournalctlVacuumAuthorizesCleanAndBuildsRequest(t *testing.T) { + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) + cleaner := &fakeJournalCleaner{result: builtins.JournalVacuumResult{Files: 2, Bytes: 3 * 1024 * 1024}} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"--vacuum-size", "64M", "--vacuum-time", "168h", "--dry-run"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: now, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Vacuuming would free 3.0M from 2 archived journal files.\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionClean, + }}, authorized) + require.Len(t, cleaner.requests, 1) + assert.Equal(t, builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-168 * time.Hour), + MaxBytes: 64 * 1024 * 1024, + DryRun: true, + }, cleaner.requests[0]) +} + +func TestJournalctlVacuumFormatsSingularResult(t *testing.T) { + cleaner := &fakeJournalCleaner{result: builtins.JournalVacuumResult{Files: 1, Bytes: 1024}} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Vacuuming done, freed 1.0K from 1 archived journal file.\n", stdout.String()) +} + +func TestJournalctlVacuumSizeRequiresTimeFloorBeforeAuthorization(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + authorized := 0 + result := runJournalctl(t, []string{"--vacuum-size", "64M"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + authorized++ + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Equal(t, "journalctl: --vacuum-size requires --vacuum-time to protect recently modified archives\n", stderr.String()) + assert.Zero(t, authorized) + assert.Empty(t, cleaner.requests) +} + +func TestJournalctlMaintenanceRejectsInvalidOrMixedOptionsBeforeAuthorization(t *testing.T) { + tests := [][]string{ + {"--dry-run"}, + {"--vacuum-size", "0"}, + {"--vacuum-size", "-1"}, + {"--vacuum-time", "0s"}, + {"--vacuum-time", "-1h"}, + {"--vacuum-time", "7d"}, + {"--rotate", "--dry-run", "--vacuum-time", "1h"}, + {"--rotate", "-u", "api.service"}, + {"--vacuum-size", "1M", "-u", "api.service"}, + {"--vacuum-time", "1h", "-k"}, + {"--vacuum-size", "1M", "-n", "10"}, + } + for _, args := range tests { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + authorized := 0 + result := runJournalctl(t, args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + authorized++ + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.NotEmpty(t, stderr.String()) + assert.Zero(t, authorized) + assert.Empty(t, cleaner.requests) + }) + } +} + +func TestJournalctlRotateDenialPreventsMutation(t *testing.T) { + rotator := &fakeJournalRotator{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return errors.New("clean denied") + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "clean denied") + assert.Zero(t, rotator.calls) +} + +func TestJournalctlVacuumDenialPreventsCleanup(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return errors.New("clean denied") + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "clean denied") + assert.Empty(t, cleaner.requests) +} + +func TestFormatUsage(t *testing.T) { + assert.Equal(t, "0B", formatUsage(0)) + assert.Equal(t, "1023B", formatUsage(1023)) + assert.Equal(t, "1.0K", formatUsage(1024)) + assert.Equal(t, "1.5M", formatUsage(1536*1024)) +} + +func TestJournalctlAuthorizesEveryScopeBeforeReading(t *testing.T) { + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + authorizationErr := errors.New("worker.service is denied") + result := runJournalctl(t, []string{"-u", "api.service", "-u", "worker.service"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + require.Len(t, operations, 2) + return authorizationErr + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), authorizationErr.Error()) + assert.Empty(t, reader.queries) +} + +func TestJournalctlRejectsUnboundedInputsBeforeAuthorization(t *testing.T) { + tests := []struct { + name string + args []string + }{ + {name: "missing scope"}, + {name: "mixed scopes", args: []string{"-u", "api.service", "-k"}}, + {name: "from head", args: []string{"-u", "api.service", "-n", "+10"}}, + {name: "negative lines", args: []string{"-u", "api.service", "--lines=-1"}}, + {name: "too many lines", args: []string{"-u", "api.service", "-n", "1001"}}, + {name: "structured output", args: []string{"-u", "api.service", "-o", "json"}}, + {name: "raw match", args: []string{"-u", "api.service", "_PID=1"}}, + {name: "bad since", args: []string{"-u", "api.service", "--since", "yesterday"}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + authorizeCalls := 0 + result := runJournalctl(t, test.args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + authorizeCalls++ + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.NotEmpty(t, stderr.String()) + assert.Zero(t, authorizeCalls) + assert.Empty(t, reader.queries) + }) + } +} + +func TestJournalctlRejectsDangerousJournalctlOptions(t *testing.T) { + Cmd.Register() + handler, ok := builtins.Lookup("journalctl") + require.True(t, ok) + + for _, args := range [][]string{ + {"--follow"}, + {"--file=/tmp/system.journal"}, + {"--directory=/var/log/journal"}, + {"--root=/host"}, + {"--machine=host"}, + {"--namespace=tenant"}, + {"--cursor=cursor"}, + {"--grep=secret"}, + {"--reverse"}, + {"--all"}, + } { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + var stdout, stderr bytes.Buffer + result := handler(context.Background(), &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + }, args) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), "unrecognized option") + }) + } +} + +func TestJournalctlLimitsRepeatedUnitScopes(t *testing.T) { + args := make([]string, 0, (builtins.MaxJournalQueryUnits+1)*2) + for i := 0; i <= builtins.MaxJournalQueryUnits; i++ { + args = append(args, "-u", "api.service") + } + var stdout, stderr bytes.Buffer + result := runJournalctl(t, args, &builtins.CallContext{Stdout: &stdout, Stderr: &stderr}) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "too many unit scopes") +} + +func TestJournalctlHelpDoesNotRequireSystemdCapability(t *testing.T) { + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--help"}, &builtins.CallContext{Stdout: &stdout, Stderr: &stderr}) + assert.Equal(t, uint8(0), result.Code) + assert.Contains(t, stdout.String(), "Usage: journalctl") + assert.Contains(t, stdout.String(), "--unit") + assert.Contains(t, stdout.String(), "--rotate") + assert.Empty(t, stderr.String()) +} + +type brokenPipeWriter struct{} + +func (brokenPipeWriter) Write([]byte) (int, error) { + return 0, syscall.EPIPE +} + +func TestJournalctlTreatsBrokenPipeAsSuccess(t *testing.T) { + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{Message: "entry"}}} + var stderr bytes.Buffer + result := runJournalctl(t, []string{"-k"}, &builtins.CallContext{ + Stdout: brokenPipeWriter{}, + Stderr: &stderr, + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) +} + +func TestParseSince(t *testing.T) { + location := time.FixedZone("test", -4*60*60) + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, location) + + absolute, ok := parseSince("2026-07-14T15:00:00Z", now) + require.True(t, ok) + assert.Equal(t, time.Date(2026, time.July, 14, 15, 0, 0, 0, time.UTC), absolute) + + local, ok := parseSince("2026-07-14 10:30:00", now) + require.True(t, ok) + assert.Equal(t, time.Date(2026, time.July, 14, 10, 30, 0, 0, location), local) + + lookback, ok := parseSince("90m", now) + require.True(t, ok) + assert.Equal(t, now.Add(-90*time.Minute), lookback) + + _, ok = parseSince("-1h", now) + assert.False(t, ok) + _, ok = parseSince(strings.Repeat("9", 100), now) + assert.False(t, ok) +} diff --git a/builtins/systemd.go b/builtins/systemd.go new file mode 100644 index 00000000..c47de256 --- /dev/null +++ b/builtins/systemd.go @@ -0,0 +1,99 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package builtins + +import ( + "context" + "errors" + "time" +) + +// ErrSystemdUnsupported reports that the current platform cannot provide a +// requested systemd operation. +var ErrSystemdUnsupported = errors.New("systemd operation is not supported") + +const ( + // MaxJournalQueryEntries is the hard per-invocation entry bound shared by + // journal builtins and backends. + MaxJournalQueryEntries = 1000 + // MaxJournalQueryUnits bounds exact unit scopes before any backend work. + MaxJournalQueryUnits = 32 +) + +// JournalQuery is the bounded, structured query accepted by the trusted +// journal backend. Callers cannot provide raw journal matches or paths. +type JournalQuery struct { + Units []string + Kernel bool + CurrentBoot bool + Since time.Time + MaxEntries int +} + +// JournalEntry contains only the fields a journalctl builtin may expose. The +// backend deliberately does not return arbitrary journal fields. +type JournalEntry struct { + Timestamp time.Time + Hostname string + Identifier string + PID string + Message string +} + +// JournalReader reads a bounded journal query and yields entries oldest first. +type JournalReader interface { + ReadJournal(ctx context.Context, query JournalQuery, yield func(JournalEntry) error) error +} + +// JournalUsage is the allocated storage consumed by the selected target's +// active and archived journal files. +type JournalUsage struct { + Bytes uint64 + Files int +} + +// JournalStorageReader exposes read-only journal storage metadata. +type JournalStorageReader interface { + JournalDiskUsage(ctx context.Context) (JournalUsage, error) +} + +// JournalVacuumRequest contains only bounded cleanup predicates. Before is an +// absolute archive mtime cutoff. MaxBytes is an allocated archived-byte target +// that may only delete archives at or before the cutoff. +type JournalVacuumRequest struct { + Now time.Time + Before time.Time + MaxBytes uint64 + DryRun bool +} + +// JournalVacuumResult reports the cleanup selected or completed without +// exposing host paths or journal filenames. +type JournalVacuumResult struct { + Files int + Bytes uint64 +} + +// JournalCleaner performs policy-bounded cleanup of archived journal files. +type JournalCleaner interface { + VacuumJournal(ctx context.Context, request JournalVacuumRequest) (JournalVacuumResult, error) +} + +// JournalRotator synchronously archives the active journals for the selected +// target. Implementations return only after journald reports completion. +type JournalRotator interface { + RotateJournal(ctx context.Context) error +} + +// SystemdServices contains the trusted backends available to systemd-aware +// builtins. Additional manager and journal-maintenance interfaces can be added +// here without exposing transports to command implementations. +type SystemdServices struct { + Journal JournalReader + JournalStorage JournalStorageReader + JournalCleaner JournalCleaner + JournalRotator JournalRotator +} diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index e67830ca..fd41174d 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -43,6 +43,11 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowAllCmds bool timeout time.Duration procPath string + systemdRoot string + journalDirs string + machineIDPath string + journalSocket string + systemBusSocket string mode string ) @@ -86,18 +91,31 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. } serviceGrants := parseAllowedServices(allowedServices) - parsedMode := interp.Mode(mode) if parsedMode != interp.ModeReadOnly && parsedMode != interp.ModeRemediation { return fmt.Errorf("--mode must be one of: read-only, remediation") } + var configuredJournalDirs []string + if journalDirs != "" { + configuredJournalDirs = strings.Split(journalDirs, ",") + } + systemdTargetSet := systemdRoot != "" || journalDirs != "" || machineIDPath != "" || journalSocket != "" || systemBusSocket != "" + execOpts := executeOpts{ allowedPaths: paths, allowedCommands: cmds, allowedServices: serviceGrants, allowAllCommands: allowAllCmds, procPath: procPath, + systemdTarget: interp.SystemdTargetConfig{ + Root: systemdRoot, + JournalDirs: configuredJournalDirs, + MachineIDPath: machineIDPath, + JournalControlSocket: journalSocket, + SystemBusSocket: systemBusSocket, + }, + systemdTargetSet: systemdTargetSet, mode: parsedMode, } @@ -148,10 +166,15 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().MarkHidden("command") //nolint:errcheck // flag is guaranteed to exist cmd.Flags().StringVarP(&allowedPaths, "allowed-paths", "p", "", "comma-separated list of PATH[:ro|:rw] directories the shell is allowed to access; entries without a suffix are read-only") cmd.Flags().StringVar(&allowedCommands, "allowed-commands", "", "comma-separated list of namespaced commands (e.g. rshell:cat,rshell:find)") - cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated exact service grants in SERVICE:ACTION[+ACTION...] form; actions: read, reload, restart") + cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated systemd service grants in SERVICE:ACTION[+ACTION...] form") cmd.Flags().BoolVar(&allowAllCmds, "allow-all-commands", false, "allow execution of all commands (builtins and external)") cmd.Flags().DurationVar(&timeout, "timeout", 0, "maximum execution time for the entire shell run (e.g. 100ms, 5s, 1m)") cmd.Flags().StringVar(&procPath, "proc-path", "", "path to the proc filesystem used by ps (default \"/proc\")") + cmd.Flags().StringVar(&systemdRoot, "systemd-root", "", "mounted systemd host root used to derive all target paths; cannot be combined with explicit systemd paths") + cmd.Flags().StringVar(&journalDirs, "systemd-journal-dirs", "", "comma-separated journal root directories for an explicit systemd target") + cmd.Flags().StringVar(&machineIDPath, "systemd-machine-id-path", "", "machine-id file for an explicit systemd target") + cmd.Flags().StringVar(&journalSocket, "systemd-journal-socket", "", "journald Varlink socket for an explicit systemd target") + cmd.Flags().StringVar(&systemBusSocket, "systemd-bus-socket", "", "system D-Bus socket for an explicit systemd target") cmd.Flags().StringVar(&mode, "mode", "read-only", "shell execution mode: read-only (default) or remediation (enables file-target output redirections within :rw AllowedPaths roots)") if err := cmd.ExecuteContext(ctx); err != nil { @@ -220,9 +243,11 @@ func rejectLongCommand(rawArgs []string) error { type executeOpts struct { allowedPaths []string allowedCommands []string - allowedServices []interp.SystemServiceControlGrant + allowedServices []interp.SystemdControlGrant allowAllCommands bool procPath string + systemdTarget interp.SystemdTargetConfig + systemdTargetSet bool mode interp.Mode } @@ -253,6 +278,9 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i if opts.procPath != "" { runOpts = append(runOpts, interp.ProcPath(opts.procPath)) } + if opts.systemdTargetSet { + runOpts = append(runOpts, interp.WithSystemdTarget(opts.systemdTarget)) + } if opts.mode != "" { runOpts = append(runOpts, interp.WithMode(opts.mode)) } @@ -266,32 +294,31 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i return runner.Run(ctx, prog) } -func parseAllowedServices(value string) []interp.SystemServiceControlGrant { +func parseAllowedServices(value string) []interp.SystemdControlGrant { if value == "" { return nil } entries := strings.Split(value, ",") - grants := make([]interp.SystemServiceControlGrant, 0, len(entries)) + grants := make([]interp.SystemdControlGrant, 0, len(entries)) for _, entry := range entries { separator := strings.LastIndexByte(entry, ':') if separator < 0 { - grants = append(grants, interp.SystemServiceControlGrant{Service: entry}) + grants = append(grants, interp.SystemdControlGrant{Service: entry}) continue } - var actions []interp.SystemServiceAction - if separator < len(entry)-1 { - actionNames := strings.Split(entry[separator+1:], "+") - actions = make([]interp.SystemServiceAction, len(actionNames)) - for i, action := range actionNames { - actions[i] = interp.SystemServiceAction(action) - } + actionSpec := entry[separator+1:] + var actionNames []string + if actionSpec != "" { + actionNames = strings.Split(actionSpec, "+") + } + actions := make([]interp.SystemdAction, len(actionNames)) + for i, action := range actionNames { + actions[i] = interp.SystemdAction(action) } - grants = append(grants, interp.SystemServiceControlGrant{ - Service: entry[:separator], - Actions: actions, - }) + selector := entry[:separator] + grants = append(grants, interp.SystemdControlGrant{Service: selector, Actions: actions}) } return grants } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index 03861a7a..a06c4396 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -181,9 +181,16 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "--allowed-commands") assert.Contains(t, stdout, "--allowed-services") assert.Contains(t, stdout, "SERVICE:ACTION[+ACTION...]") + assert.NotContains(t, stdout, "--allowed-systemd") assert.Contains(t, stdout, "--allow-all-commands") assert.Contains(t, stdout, "file-target output redirections within :rw AllowedPaths roots") assert.Contains(t, stdout, "--timeout") + assert.Contains(t, stdout, "--systemd-root") + assert.Contains(t, stdout, "--systemd-journal-dirs") + assert.Contains(t, stdout, "--systemd-machine-id-path") + assert.Contains(t, stdout, "--systemd-journal-socket") + assert.Contains(t, stdout, "--systemd-bus-socket") + assert.NotContains(t, stdout, "--journal-vacuum-") assert.NotContains(t, stdout, "--command", "-c/--command should be hidden from help") } @@ -298,67 +305,119 @@ func TestAllowedServicesFlag(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedServicesFlagSkipsGrantsWithoutActions(t *testing.T) { +func TestAllowedServicesFlagIgnoresGrantsWithoutActions(t *testing.T) { + for _, grant := range []string{"mysql.service", "mysql.service:"} { + t.Run(grant, func(t *testing.T) { + code, stdout, stderr := runCLI(t, + "--allow-all-commands", + "--allowed-services", grant, + "-c", `echo hello`, + ) + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Empty(t, stderr) + }) + } +} + +func TestAllowedServicesFlagWarnsAndSkipsUnknownAction(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "mysql.service,redis.service:,,nginx.service:stop", + "--allowed-services", "mysql.service:stop", "-c", `echo hello`, ) assert.Equal(t, 0, code) assert.Equal(t, "hello\n", stdout) - assert.Equal(t, "AllowedSystemServices: skipping unsupported action \"stop\" in grant 3 for \"nginx.service\"\n", stderr) + assert.Contains(t, stderr, `skipping unsupported action "stop"`) } -func TestAllowedServicesFlagWarnsAndSkipsUnknownAction(t *testing.T) { +func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "mysql.service:stop", + "--allowed-services", "mysql*.service:read", "-c", `echo hello`, ) assert.Equal(t, 0, code) assert.Equal(t, "hello\n", stdout) - assert.Contains(t, stderr, `skipping unsupported action "stop"`) + assert.Contains(t, stderr, "AllowedSystemServices: skipping") + assert.Contains(t, stderr, "glob pattern") +} + +func TestParseAllowedServicesParsesServiceActions(t *testing.T) { + grants := parseAllowedServices("systemd-journald.service:read+clean") + require.Len(t, grants, 1) + assert.Equal(t, "systemd-journald.service", grants[0].Service) + assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemdActionClean}, grants[0].Actions) +} + +func TestParseAllowedServicesPreservesEntriesForPolicyValidation(t *testing.T) { + grants := parseAllowedServices("ignored.service,tenant:mysql.service:read,mysql.service:read+stop") + require.Len(t, grants, 3) + assert.Equal(t, interp.SystemdControlGrant{Service: "ignored.service"}, grants[0]) + assert.Equal(t, interp.SystemdControlGrant{ + Service: "tenant:mysql.service", + Actions: []interp.SystemServiceAction{interp.SystemServiceRead}, + }, grants[1]) + assert.Equal(t, interp.SystemdControlGrant{ + Service: "mysql.service", + Actions: []interp.SystemServiceAction{interp.SystemServiceRead, "stop"}, + }, grants[2]) } -func TestAllowedServicesFlagWarnsAndSkipsEmptyAction(t *testing.T) { +func TestAllowedServicesFlagAcceptsServiceGrants(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "mysql.service:read++reload", + "--allowed-services", "mysql.service:read+restart,systemd-journald.service:read+clean", + "--mode", "remediation", "-c", `echo hello`, ) assert.Equal(t, 0, code) assert.Equal(t, "hello\n", stdout) - assert.Equal(t, "AllowedSystemServices: skipping unsupported action \"\" in grant 0 for \"mysql.service\"\n", stderr) + assert.Empty(t, stderr) } -func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { +func TestAllowedServicesFlagWarnsAndSkipsColonInSelector(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", ":read,mysql*.service:read,nginx.service:read", + "--allowed-services", "tenant:mysql.service:read", "-c", `echo hello`, ) assert.Equal(t, 0, code) assert.Equal(t, "hello\n", stdout) - assert.Contains(t, stderr, "AllowedSystemServices: skipping grant 0: system service name must not be empty") - assert.Contains(t, stderr, "AllowedSystemServices: skipping grant 1") - assert.Contains(t, stderr, "glob pattern") + assert.Contains(t, stderr, "AllowedSystemServices: skipping") + assert.Contains(t, stderr, "must not contain ':'") } -func TestParseAllowedServicesUsesLastColon(t *testing.T) { - grants := parseAllowedServices("tenant:mysql.service:read+reload") - require.Len(t, grants, 1) - assert.Equal(t, "tenant:mysql.service", grants[0].Service) - assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemServiceReload}, grants[0].Actions) -} - -func TestParseAllowedServicesPreservesAPIGrantSemantics(t *testing.T) { - grants := parseAllowedServices("mysql.service,redis.service:,:read,nginx.service:read++reload") - assert.Equal(t, []interp.SystemServiceControlGrant{ - {Service: "mysql.service"}, - {Service: "redis.service"}, - {Service: "", Actions: []interp.SystemServiceAction{interp.SystemServiceRead}}, - {Service: "nginx.service", Actions: []interp.SystemServiceAction{interp.SystemServiceRead, "", interp.SystemServiceReload}}, - }, grants) +func TestSystemdRootFlag(t *testing.T) { + code, stdout, stderr := runCLI(t, + "--allow-all-commands", + "--systemd-root", "/host", + "-c", `echo hello`, + ) + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Empty(t, stderr) +} + +func TestSystemdTargetFlagsRejectMixedRootAndExplicitPaths(t *testing.T) { + code, _, stderr := runCLI(t, + "--allow-all-commands", + "--systemd-root", "/host", + "--systemd-machine-id-path", "/host/etc/machine-id", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, "cannot be combined") +} + +func TestExplicitSystemdTargetRequiresMachineIDPath(t *testing.T) { + code, _, stderr := runCLI(t, + "--allow-all-commands", + "--systemd-journal-dirs", "/host/var/log/journal,/host/run/log/journal", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, "machine ID path is required") } func TestAllowAllCommandsFlag(t *testing.T) { diff --git a/docs/RULES.md b/docs/RULES.md index 98c175d0..64449122 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -71,14 +71,48 @@ f, err := os.Open(path) Using `os` constants (`os.O_RDONLY`, `os.FileMode`) and types (`*os.File` for stdin) is fine; only the filesystem-accessing *functions* are forbidden. +#### Trusted systemd target exception + +Systemd-aware builtins MUST use the structured services on `callCtx.Systemd` and +MUST NOT open target paths themselves. The trusted `internal/systemd` backend may +read paths selected by `interp.WithSystemdTarget`; those paths intentionally +bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding +application and cannot be supplied by shell scripts. + +Journal reads and storage metadata are limited to regular, non-symlink `.journal` +files directly under the configured machine-ID directories. The pure-Go reader +verifies each file header's machine ID, snapshots the file set and identities, +retries once after a concurrent rotation or structurally inconsistent read, and +applies fixed file, index, field-size, entry-count, decompression, and cancellation +bounds. Builtins receive selected fields only and never receive a raw journal +handle, target path, or arbitrary field-match capability. + +The only deletion exception is `JournalCleaner.VacuumJournal`. It is available +only through trusted systemd target configuration and a validated operator +vacuum policy. The backend pins each configured journal directory with `os.Root`, +checks directory identity across open, accepts only strict systemd archived-file +names, excludes symlinks and hardlinks, and revalidates file identity immediately +before rooted removal. Active, malformed, recently modified, and policy-retained +files must never be deleted. Every cleanup call has both file-count and byte +ceilings in addition to the exact `systemd-journald.service:clean` authorization +and remediation-mode requirement. + +`JournalRotator.RotateJournal` is the only journal-daemon mutation exception. +It may call only the fixed `io.systemd.Journal.Rotate` Varlink method through +the configured journal control socket. The backend validates the target machine +ID before connecting, rejects symlink and non-socket endpoints, verifies socket +identity across the connection, and applies fixed response-size and execution +time bounds. A generic Varlink method or parameter interface must not be exposed +to builtins. + --- ## Implementation Rules ### File System Safety -- Commands MUST NOT write to any files on the system in any way +- Commands MUST NOT write to any files on the system except through an explicitly documented, structured remediation capability such as `TruncateToZeroIfAtLeast` or `JournalCleaner` - Commands MUST NOT execute any files or external binaries on the system in any way -- Commands MUST NOT create, modify, or delete files, directories, or symlinks +- Commands MUST NOT create, modify, or delete files, directories, or symlinks except as explicitly permitted by such a remediation capability - Commands MUST NOT follow symlinks during write operations (no writes = no risk, but verify) ### Memory Safety & Resource Limits diff --git a/go.mod b/go.mod index 71108233..40b21224 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,13 @@ toolchain go1.26.2 require ( github.com/DataDog/datadog-agent/pkg/fleet/installer v0.78.0 + github.com/klauspost/compress v1.19.0 + github.com/pierrec/lz4/v4 v4.1.27 github.com/prometheus-community/pro-bing v0.8.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 + github.com/ulikunitz/xz v0.5.15 golang.org/x/sys v0.46.0 golang.org/x/term v0.44.0 golang.org/x/tools v0.44.0 diff --git a/go.sum b/go.sum index 4ffe9e64..c7812b78 100644 --- a/go.sum +++ b/go.sum @@ -24,12 +24,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= @@ -52,6 +56,8 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/internal/systemd/client.go b/internal/systemd/client.go new file mode 100644 index 00000000..49d2d876 --- /dev/null +++ b/internal/systemd/client.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +// Client implements the trusted systemd backends for one resolved target. +// Target paths are copied at construction so runner configuration remains +// immutable while commands execute. +type Client struct { + target Target +} + +// NewClient creates a client for one resolved systemd target. +func NewClient(target Target) *Client { + target.JournalDirs = append([]string(nil), target.JournalDirs...) + return &Client{target: target} +} diff --git a/internal/systemd/journal_data.go b/internal/systemd/journal_data.go new file mode 100644 index 00000000..aff5380e --- /dev/null +++ b/internal/systemd/journal_data.go @@ -0,0 +1,270 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "math" + + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" + "github.com/ulikunitz/xz" +) + +const ( + journalDataRegularHeaderSize = 64 + journalDataCompactHeaderSize = 72 + + maxJournalDecodeWindow = 8 * 1024 * 1024 + maxJournalEncodedRead = 8 * 1024 * 1024 + maxJournalLZ4DataSize = 8 * 1024 * 1024 + maxJournalPayloadRead = maxJournalFieldSize + 512 +) + +type journalDataObject struct { + journalObject + hash uint64 + nextHashOffset uint64 + nextFieldOffset uint64 + entryOffset uint64 + entryArrayOffset uint64 + nEntries uint64 + tailEntryArrayOffset uint32 + tailEntryArrayNEntries uint32 + hasTailEntryArrayReference bool + payloadOffset uint64 + payloadSize uint64 +} + +func (f *journalFileView) dataObjectAt(offset uint64) (journalDataObject, error) { + object, err := f.objectAt(offset, journalObjectData) + if err != nil { + return journalDataObject{}, err + } + + headerSize := uint64(journalDataRegularHeaderSize) + if f.header.compact() { + headerSize = journalDataCompactHeaderSize + } + if object.size < headerSize { + return journalDataObject{}, journalCorrupt(f.name, offset, "DATA object size %d is smaller than its %d-byte header", object.size, headerSize) + } + + var raw [journalDataCompactHeaderSize]byte + if err := readJournalAt(f.name, f.reader, f.size, offset, raw[:headerSize]); err != nil { + return journalDataObject{}, err + } + data := journalDataObject{ + journalObject: object, + hash: binary.LittleEndian.Uint64(raw[16:24]), + nextHashOffset: binary.LittleEndian.Uint64(raw[24:32]), + nextFieldOffset: binary.LittleEndian.Uint64(raw[32:40]), + entryOffset: binary.LittleEndian.Uint64(raw[40:48]), + entryArrayOffset: binary.LittleEndian.Uint64(raw[48:56]), + nEntries: binary.LittleEndian.Uint64(raw[56:64]), + payloadOffset: offset + headerSize, + payloadSize: object.size - headerSize, + } + if f.header.compact() { + data.tailEntryArrayOffset = binary.LittleEndian.Uint32(raw[64:68]) + data.tailEntryArrayNEntries = binary.LittleEndian.Uint32(raw[68:72]) + data.hasTailEntryArrayReference = true + } + + for _, reference := range []struct { + offset uint64 + label string + }{ + {offset: data.nextHashOffset, label: "next DATA hash"}, + {offset: data.nextFieldOffset, label: "next DATA field"}, + {offset: data.entryOffset, label: "first DATA entry"}, + {offset: data.entryArrayOffset, label: "DATA entry array"}, + } { + if reference.offset == 0 { + continue + } + if err := validateJournalObjectOffset(f.name, f.header, reference.offset, reference.label); err != nil { + return journalDataObject{}, err + } + } + if data.hasTailEntryArrayReference { + if (data.tailEntryArrayOffset == 0) != (data.tailEntryArrayNEntries == 0) { + return journalDataObject{}, journalCorrupt(f.name, uint64(data.tailEntryArrayOffset), "DATA tail entry array offset and count are inconsistent") + } + if data.tailEntryArrayOffset != 0 { + tailOffset := uint64(data.tailEntryArrayOffset) + if err := validateJournalObjectOffset(f.name, f.header, tailOffset, "DATA tail entry array"); err != nil { + return journalDataObject{}, err + } + if data.entryArrayOffset == 0 || data.entryArrayOffset > tailOffset { + return journalDataObject{}, journalCorrupt(f.name, tailOffset, "DATA tail entry array precedes its entry array chain") + } + } + } + + return data, nil +} + +func (f *journalFileView) readDataPayload(data journalDataObject, limit int) ([]byte, bool, error) { + if limit <= 0 { + return nil, false, fmt.Errorf("journal DATA payload limit must be positive") + } + if limit == math.MaxInt { + return nil, false, fmt.Errorf("journal DATA payload limit is too large") + } + if limit > maxJournalPayloadRead { + return nil, false, fmt.Errorf("journal DATA payload limit %d exceeds maximum %d", limit, maxJournalPayloadRead) + } + + switch data.flags { + case 0: + return f.readUncompressedDataPayload(data, limit) + case journalObjectCompressedXZ: + decoder, err := (xz.ReaderConfig{ + DictCap: maxJournalDecodeWindow, + SingleStream: true, + }).NewReader(f.compressedPayloadReader(data)) + if err != nil { + if errors.Is(err, errJournalLimit) { + return nil, false, err + } + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode XZ DATA payload: %v", err) + } + return readDecodedJournalPayload(f.name, data.payloadOffset, "XZ", decoder, limit) + case journalObjectCompressedLZ4: + return f.readLZ4DataPayload(data, limit) + case journalObjectCompressedZSTD: + decoder, err := zstd.NewReader( + f.compressedPayloadReader(data), + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderLowmem(true), + zstd.WithDecoderMaxMemory(maxJournalDecodeWindow), + zstd.WithDecoderMaxWindow(maxJournalDecodeWindow), + ) + if err != nil { + if errors.Is(err, errJournalLimit) { + return nil, false, err + } + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode ZSTD DATA payload: %v", err) + } + defer decoder.Close() + return readDecodedJournalPayload(f.name, data.payloadOffset, "ZSTD", decoder, limit) + default: + return nil, false, journalUnsupported(f.name, "DATA object at offset %d uses compression flags 0x%02x", data.offset, data.flags) + } +} + +func (f *journalFileView) compressedPayloadReader(data journalDataObject) io.Reader { + section := io.NewSectionReader(f.reader, int64(data.payloadOffset), int64(data.payloadSize)) + if data.payloadSize <= maxJournalEncodedRead { + return section + } + return &journalCompressedReader{ + reader: section, + remaining: maxJournalEncodedRead, + name: f.name, + offset: data.payloadOffset, + } +} + +type journalCompressedReader struct { + reader io.Reader + remaining int + name string + offset uint64 +} + +func (r *journalCompressedReader) Read(destination []byte) (int, error) { + if len(destination) == 0 { + return 0, nil + } + if r.remaining == 0 { + return 0, journalLimit(r.name, r.offset, "compressed DATA input exceeds %d bytes", maxJournalEncodedRead) + } + if len(destination) > r.remaining { + destination = destination[:r.remaining] + } + n, err := r.reader.Read(destination) + r.remaining -= n + return n, err +} + +func (f *journalFileView) readUncompressedDataPayload(data journalDataObject, limit int) ([]byte, bool, error) { + readSize := data.payloadSize + if readSize > uint64(limit)+1 { + readSize = uint64(limit) + 1 + } + payload := make([]byte, int(readSize)) + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset, payload); err != nil { + return nil, false, err + } + if len(payload) > limit { + return payload[:limit], true, nil + } + return payload, false, nil +} + +func (f *journalFileView) readLZ4DataPayload(data journalDataObject, limit int) ([]byte, bool, error) { + if data.payloadSize <= 8 { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "LZ4 DATA payload is too small") + } + + var sizeBytes [8]byte + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset, sizeBytes[:]); err != nil { + return nil, false, err + } + decodedSize := binary.LittleEndian.Uint64(sizeBytes[:]) + if decodedSize == 0 { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "LZ4 DATA payload has an empty decoded size") + } + if decodedSize > maxJournalLZ4DataSize { + return nil, false, journalLimit(f.name, data.payloadOffset, "LZ4 DATA payload expands to %d bytes; maximum is %d", decodedSize, maxJournalLZ4DataSize) + } + if data.payloadSize-8 > maxJournalEncodedRead { + return nil, false, journalLimit(f.name, data.payloadOffset, "compressed LZ4 DATA input exceeds %d bytes", maxJournalEncodedRead) + } + if decodedSize > uint64(math.MaxInt) || data.payloadSize-8 > uint64(math.MaxInt) { + return nil, false, journalLimit(f.name, data.payloadOffset, "LZ4 DATA payload exceeds the platform allocation range") + } + + compressed := make([]byte, int(data.payloadSize-8)) + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset+8, compressed); err != nil { + return nil, false, err + } + decoded := make([]byte, int(decodedSize)) + n, err := lz4.UncompressBlock(compressed, decoded) + if err != nil { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: %v", err) + } + if n != len(decoded) { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: got %d of %d bytes", n, len(decoded)) + } + if len(decoded) > limit { + return decoded[:limit], true, nil + } + return decoded, false, nil +} + +func readDecodedJournalPayload(name string, offset uint64, algorithm string, reader io.Reader, limit int) ([]byte, bool, error) { + payload, err := io.ReadAll(io.LimitReader(reader, int64(limit)+1)) + if err != nil { + if errors.Is(err, errJournalLimit) { + return nil, false, err + } + return nil, false, journalCorrupt(name, offset, "decode %s DATA payload: %v", algorithm, err) + } + if len(payload) > limit { + return payload[:limit], true, nil + } + return payload, false, nil +} + +func journalLimit(name string, offset uint64, format string, arguments ...any) error { + return fmt.Errorf("%w %q at offset %d: %s", errJournalLimit, name, offset, fmt.Sprintf(format, arguments...)) +} diff --git a/internal/systemd/journal_data_test.go b/internal/systemd/journal_data_test.go new file mode 100644 index 00000000..e09dabca --- /dev/null +++ b/internal/systemd/journal_data_test.go @@ -0,0 +1,209 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ulikunitz/xz" +) + +func TestJournalDataObjectParsesRegularAndCompactLayouts(t *testing.T) { + for _, compact := range []bool{false, true} { + name := "regular" + if compact { + name = "compact" + } + t.Run(name, func(t *testing.T) { + contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, compact) + binary.LittleEndian.PutUint64(contents[offset+16:offset+24], 0x0102030405060708) + binary.LittleEndian.PutUint64(contents[offset+56:offset+64], 3) + + view, err := newJournalFileView("data.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + assert.Equal(t, uint64(0x0102030405060708), data.hash) + assert.Equal(t, uint64(3), data.nEntries) + assert.Equal(t, uint64(len("MESSAGE=hello")), data.payloadSize) + headerSize := uint64(journalDataRegularHeaderSize) + if compact { + headerSize = journalDataCompactHeaderSize + } + assert.Equal(t, uint64(offset)+headerSize, data.payloadOffset) + assert.Equal(t, compact, data.hasTailEntryArrayReference) + }) + } +} + +func TestJournalDataPayloadDecodesSupportedCompression(t *testing.T) { + payload := []byte("MESSAGE=" + strings.Repeat("journal payload ", 32)) + tests := []struct { + name string + objectFlag uint8 + encode func(*testing.T, []byte) []byte + }{ + {name: "uncompressed", encode: func(_ *testing.T, data []byte) []byte { return data }}, + {name: "xz", objectFlag: journalObjectCompressedXZ, encode: encodeJournalXZ}, + {name: "lz4", objectFlag: journalObjectCompressedLZ4, encode: encodeJournalLZ4}, + {name: "zstd", objectFlag: journalObjectCompressedZSTD, encode: encodeJournalZSTD}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + encoded := test.encode(t, payload) + contents, offset := testJournalDataContents(t, encoded, test.objectFlag, false) + view, err := newJournalFileView(test.name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + decoded, truncated, err := view.readDataPayload(data, len(payload)+1) + require.NoError(t, err) + assert.False(t, truncated) + assert.Equal(t, payload, decoded) + + decoded, truncated, err = view.readDataPayload(data, 17) + require.NoError(t, err) + assert.True(t, truncated) + assert.Equal(t, payload[:17], decoded) + }) + } +} + +func TestJournalDataPayloadRejectsMalformedCompression(t *testing.T) { + for _, objectFlag := range []uint8{ + journalObjectCompressedXZ, + journalObjectCompressedLZ4, + journalObjectCompressedZSTD, + } { + malformed := bytes.Repeat([]byte{0xff}, 32) + if objectFlag == journalObjectCompressedLZ4 { + binary.LittleEndian.PutUint64(malformed[:8], 64) + } + contents, offset := testJournalDataContents(t, malformed, objectFlag, false) + view, err := newJournalFileView("malformed.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + _, _, err = view.readDataPayload(data, 64) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "decode") + } +} + +func TestJournalDataPayloadBoundsLZ4Expansion(t *testing.T) { + encoded := make([]byte, 9) + binary.LittleEndian.PutUint64(encoded[:8], maxJournalLZ4DataSize+1) + contents, offset := testJournalDataContents(t, encoded, journalObjectCompressedLZ4, false) + view, err := newJournalFileView("large-lz4.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + _, _, err = view.readDataPayload(data, 64) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalLimit) + assert.Contains(t, err.Error(), "expands to 8388609 bytes") +} + +func TestJournalDataPayloadRejectsUnboundedReadLimit(t *testing.T) { + contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, false) + view, err := newJournalFileView("limit.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + _, _, err = view.readDataPayload(data, maxJournalPayloadRead+1) + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeds maximum") +} + +func TestJournalDataObjectRejectsInvalidReferences(t *testing.T) { + contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, false) + binary.LittleEndian.PutUint64(contents[offset+24:offset+32], uint64(offset+1)) + view, err := newJournalFileView("reference.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + _, err = view.dataObjectAt(uint64(offset)) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "next DATA hash offset is not 8-byte aligned") +} + +func testJournalDataContents(t *testing.T, encoded []byte, objectFlag uint8, compact bool) ([]byte, int) { + t.Helper() + headerFlags := uint32(0) + switch objectFlag { + case journalObjectCompressedXZ: + headerFlags |= journalHeaderIncompatibleCompressedXZ + case journalObjectCompressedLZ4: + headerFlags |= journalHeaderIncompatibleCompressedLZ4 + case journalObjectCompressedZSTD: + headerFlags |= journalHeaderIncompatibleCompressedZSTD + } + headerSize := journalHeaderCurrentSize + dataHeaderSize := journalDataRegularHeaderSize + if compact { + headerFlags |= journalHeaderIncompatibleCompact + dataHeaderSize = journalDataCompactHeaderSize + } + objectSize := dataHeaderSize + len(encoded) + fileSize := alignJournalTestSize(headerSize + objectSize) + contents := testJournalContents(fileSize, 0, headerFlags) + binary.LittleEndian.PutUint64(contents[136:144], uint64(headerSize)) + object := contents[headerSize:] + object[0] = journalObjectData + object[1] = objectFlag + binary.LittleEndian.PutUint64(object[8:16], uint64(objectSize)) + copy(object[dataHeaderSize:], encoded) + return contents, headerSize +} + +func alignJournalTestSize(size int) int { + return (size + 7) &^ 7 +} + +func encodeJournalXZ(t *testing.T, payload []byte) []byte { + t.Helper() + var encoded bytes.Buffer + writer, err := xz.NewWriter(&encoded) + require.NoError(t, err) + _, err = writer.Write(payload) + require.NoError(t, err) + require.NoError(t, writer.Close()) + return encoded.Bytes() +} + +func encodeJournalLZ4(t *testing.T, payload []byte) []byte { + t.Helper() + compressed := make([]byte, lz4.CompressBlockBound(len(payload))) + n, err := lz4.CompressBlock(payload, compressed, nil) + require.NoError(t, err) + require.Positive(t, n) + encoded := make([]byte, 8+n) + binary.LittleEndian.PutUint64(encoded[:8], uint64(len(payload))) + copy(encoded[8:], compressed[:n]) + return encoded +} + +func encodeJournalZSTD(t *testing.T, payload []byte) []byte { + t.Helper() + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + require.NoError(t, err) + defer encoder.Close() + return encoder.EncodeAll(payload, nil) +} diff --git a/internal/systemd/journal_entry.go b/internal/systemd/journal_entry.go new file mode 100644 index 00000000..389d2a7f --- /dev/null +++ b/internal/systemd/journal_entry.go @@ -0,0 +1,291 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "fmt" +) + +const ( + journalEntryHeaderSize = 64 + journalEntryArrayHeaderSize = 24 + journalEntryRegularItemSize = 16 + journalEntryCompactItemSize = 4 + journalEntryArrayRegularItemSize = 8 + + maxJournalEntryFields = 1024 + maxJournalEntryArrays = 128 +) + +type journalEntryItem struct { + dataOffset uint64 + hash uint64 +} + +type journalEntryObject struct { + journalObject + seqnum uint64 + realtime uint64 + monotonic uint64 + bootID journalID + xorHash uint64 + items []journalEntryItem +} + +func (f *journalFileView) entryObjectAt(offset uint64) (journalEntryObject, error) { + object, err := f.objectAt(offset, journalObjectEntry) + if err != nil { + return journalEntryObject{}, err + } + if object.size < journalEntryHeaderSize { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY object size %d is smaller than its header", object.size) + } + + itemSize := uint64(journalEntryRegularItemSize) + if f.header.compact() { + itemSize = journalEntryCompactItemSize + } + itemsSize := object.size - journalEntryHeaderSize + if itemsSize%itemSize != 0 { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY item payload size %d is not a multiple of %d", itemsSize, itemSize) + } + itemCount := itemsSize / itemSize + if itemCount == 0 { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY object has no DATA items") + } + if itemCount > maxJournalEntryFields { + return journalEntryObject{}, journalLimit(f.name, offset, "ENTRY object has %d DATA items; maximum is %d", itemCount, maxJournalEntryFields) + } + + contents := make([]byte, int(object.size)) + if err := readJournalAt(f.name, f.reader, f.size, offset, contents); err != nil { + return journalEntryObject{}, err + } + entry := journalEntryObject{ + journalObject: object, + seqnum: binary.LittleEndian.Uint64(contents[16:24]), + realtime: binary.LittleEndian.Uint64(contents[24:32]), + monotonic: binary.LittleEndian.Uint64(contents[32:40]), + xorHash: binary.LittleEndian.Uint64(contents[56:64]), + items: make([]journalEntryItem, 0, int(itemCount)), + } + copy(entry.bootID[:], contents[40:56]) + for index := uint64(0); index < itemCount; index++ { + position := uint64(journalEntryHeaderSize) + index*itemSize + item := journalEntryItem{} + if f.header.compact() { + item.dataOffset = uint64(binary.LittleEndian.Uint32(contents[position : position+4])) + } else { + item.dataOffset = binary.LittleEndian.Uint64(contents[position : position+8]) + item.hash = binary.LittleEndian.Uint64(contents[position+8 : position+16]) + } + if item.dataOffset == 0 { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY item %d has a zero DATA offset", index) + } + if err := validateJournalObjectOffset(f.name, f.header, item.dataOffset, fmt.Sprintf("ENTRY item %d DATA", index)); err != nil { + return journalEntryObject{}, err + } + entry.items = append(entry.items, item) + } + return entry, nil +} + +type journalEntryArray struct { + journalObject + nextOffset uint64 + itemsOffset uint64 + capacity uint64 +} + +func (f *journalFileView) entryArrayAt(offset uint64) (journalEntryArray, error) { + object, err := f.objectAt(offset, journalObjectEntryArray) + if err != nil { + return journalEntryArray{}, err + } + if object.size < journalEntryArrayHeaderSize { + return journalEntryArray{}, journalCorrupt(f.name, offset, "ENTRY_ARRAY object size %d is smaller than its header", object.size) + } + + itemSize := uint64(journalEntryArrayRegularItemSize) + if f.header.compact() { + itemSize = journalEntryCompactItemSize + } + itemsSize := object.size - journalEntryArrayHeaderSize + if itemsSize%itemSize != 0 { + return journalEntryArray{}, journalCorrupt(f.name, offset, "ENTRY_ARRAY item payload size %d is not a multiple of %d", itemsSize, itemSize) + } + capacity := itemsSize / itemSize + if capacity == 0 { + return journalEntryArray{}, journalCorrupt(f.name, offset, "ENTRY_ARRAY object has no item capacity") + } + + var header [journalEntryArrayHeaderSize]byte + if err := readJournalAt(f.name, f.reader, f.size, offset, header[:]); err != nil { + return journalEntryArray{}, err + } + array := journalEntryArray{ + journalObject: object, + nextOffset: binary.LittleEndian.Uint64(header[16:24]), + itemsOffset: offset + journalEntryArrayHeaderSize, + capacity: capacity, + } + if array.nextOffset != 0 { + if err := validateJournalObjectOffset(f.name, f.header, array.nextOffset, "next ENTRY_ARRAY"); err != nil { + return journalEntryArray{}, err + } + } + return array, nil +} + +func (f *journalFileView) entryArrayItem(array journalEntryArray, index uint64) (uint64, error) { + if index >= array.capacity { + return 0, journalCorrupt(f.name, array.offset, "ENTRY_ARRAY item index %d exceeds capacity %d", index, array.capacity) + } + itemSize := uint64(journalEntryArrayRegularItemSize) + if f.header.compact() { + itemSize = journalEntryCompactItemSize + } + position := array.itemsOffset + index*itemSize + var raw [8]byte + if err := readJournalAt(f.name, f.reader, f.size, position, raw[:itemSize]); err != nil { + return 0, err + } + offset := binary.LittleEndian.Uint64(raw[:]) + if f.header.compact() { + offset = uint64(binary.LittleEndian.Uint32(raw[:4])) + } + if offset == 0 { + return 0, journalCorrupt(f.name, position, "ENTRY_ARRAY item %d has a zero ENTRY offset", index) + } + if err := validateJournalObjectOffset(f.name, f.header, offset, "ENTRY_ARRAY item"); err != nil { + return 0, err + } + return offset, nil +} + +type journalEntryArraySegment struct { + array journalEntryArray + count uint64 +} + +type journalEntryOffsetIterator struct { + file *journalFileView + inlineOffset uint64 + inlinePending bool + segments []journalEntryArraySegment + segmentIndex int + itemIndex uint64 + lastOffset uint64 +} + +func (f *journalFileView) entryOffsetsForData(data journalDataObject) (*journalEntryOffsetIterator, error) { + iterator := &journalEntryOffsetIterator{file: f, segmentIndex: -1} + if data.nEntries == 0 { + if data.entryOffset != 0 || data.entryArrayOffset != 0 || data.tailEntryArrayOffset != 0 || data.tailEntryArrayNEntries != 0 { + return nil, journalCorrupt(f.name, data.offset, "unreferenced DATA object has entry offsets") + } + return iterator, nil + } + if data.entryOffset == 0 { + return nil, journalCorrupt(f.name, data.offset, "referenced DATA object has no inline ENTRY offset") + } + if data.nEntries > f.header.nEntries { + return nil, journalCorrupt(f.name, data.offset, "DATA object references %d entries but the journal has %d", data.nEntries, f.header.nEntries) + } + iterator.inlineOffset = data.entryOffset + iterator.inlinePending = true + + remaining := data.nEntries - 1 + if remaining == 0 { + if data.entryArrayOffset != 0 { + return nil, journalCorrupt(f.name, data.offset, "single-entry DATA object has an ENTRY_ARRAY") + } + return iterator, nil + } + if data.entryArrayOffset == 0 { + return nil, journalCorrupt(f.name, data.offset, "multi-entry DATA object has no ENTRY_ARRAY") + } + + seen := make(map[uint64]struct{}, maxJournalEntryArrays) + offset := data.entryArrayOffset + for remaining > 0 { + if len(iterator.segments) >= maxJournalEntryArrays { + return nil, journalLimit(f.name, offset, "ENTRY_ARRAY chain exceeds %d objects", maxJournalEntryArrays) + } + if _, exists := seen[offset]; exists { + return nil, journalCorrupt(f.name, offset, "ENTRY_ARRAY chain contains a cycle") + } + seen[offset] = struct{}{} + + array, err := f.entryArrayAt(offset) + if err != nil { + return nil, err + } + count := array.capacity + if count > remaining { + count = remaining + } + iterator.segments = append(iterator.segments, journalEntryArraySegment{array: array, count: count}) + remaining -= count + if remaining == 0 { + if array.nextOffset != 0 { + return nil, journalCorrupt(f.name, array.offset, "ENTRY_ARRAY chain continues after all referenced entries") + } + break + } + if array.nextOffset == 0 { + return nil, journalCorrupt(f.name, array.offset, "ENTRY_ARRAY chain ends before all referenced entries") + } + offset = array.nextOffset + } + + if data.hasTailEntryArrayReference { + last := iterator.segments[len(iterator.segments)-1] + if uint64(data.tailEntryArrayOffset) != last.array.offset || uint64(data.tailEntryArrayNEntries) != last.count { + return nil, journalCorrupt(f.name, data.offset, "DATA tail ENTRY_ARRAY metadata does not match its array chain") + } + } + iterator.segmentIndex = len(iterator.segments) - 1 + iterator.itemIndex = iterator.segments[iterator.segmentIndex].count + return iterator, nil +} + +func (i *journalEntryOffsetIterator) previous() (uint64, bool, error) { + for i.segmentIndex >= 0 { + if i.itemIndex > 0 { + i.itemIndex-- + offset, err := i.file.entryArrayItem(i.segments[i.segmentIndex].array, i.itemIndex) + if err != nil { + return 0, false, err + } + if err := i.validateDescending(offset); err != nil { + return 0, false, err + } + return offset, true, nil + } + i.segmentIndex-- + if i.segmentIndex >= 0 { + i.itemIndex = i.segments[i.segmentIndex].count + } + } + if i.inlinePending { + i.inlinePending = false + if err := i.validateDescending(i.inlineOffset); err != nil { + return 0, false, err + } + return i.inlineOffset, true, nil + } + return 0, false, nil +} + +func (i *journalEntryOffsetIterator) validateDescending(offset uint64) error { + if i.lastOffset != 0 && offset >= i.lastOffset { + return journalCorrupt(i.file.name, offset, "ENTRY offsets are not strictly increasing in the forward index") + } + i.lastOffset = offset + return nil +} diff --git a/internal/systemd/journal_entry_test.go b/internal/systemd/journal_entry_test.go new file mode 100644 index 00000000..d101cf30 --- /dev/null +++ b/internal/systemd/journal_entry_test.go @@ -0,0 +1,218 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalEntryObjectParsesRegularAndCompactItems(t *testing.T) { + for _, compact := range []bool{false, true} { + name := "regular" + if compact { + name = "compact" + } + t.Run(name, func(t *testing.T) { + contents, entryOffset, dataOffset := testJournalEntryContents(compact) + view, err := newJournalFileView(name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + entry, err := view.entryObjectAt(uint64(entryOffset)) + require.NoError(t, err) + assert.Equal(t, uint64(42), entry.seqnum) + assert.Equal(t, uint64(1_700_000_000_000_000), entry.realtime) + assert.Equal(t, uint64(1234), entry.monotonic) + assert.Equal(t, "11111111111111111111111111111111", entry.bootID.String()) + require.Len(t, entry.items, 1) + assert.Equal(t, uint64(dataOffset), entry.items[0].dataOffset) + if compact { + assert.Zero(t, entry.items[0].hash) + } else { + assert.Equal(t, uint64(0x0102030405060708), entry.items[0].hash) + } + }) + } +} + +func TestJournalEntryOffsetIteratorReadsNewestFirst(t *testing.T) { + for _, compact := range []bool{false, true} { + name := "regular" + if compact { + name = "compact" + } + t.Run(name, func(t *testing.T) { + contents, dataOffset, entryOffsets := testJournalEntryArrayContents(compact) + view, err := newJournalFileView(name+"-array.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(dataOffset)) + require.NoError(t, err) + iterator, err := view.entryOffsetsForData(data) + require.NoError(t, err) + + var actual []uint64 + for { + offset, found, err := iterator.previous() + require.NoError(t, err) + if !found { + break + } + actual = append(actual, offset) + } + expected := make([]uint64, len(entryOffsets)) + for index := range entryOffsets { + expected[index] = uint64(entryOffsets[len(entryOffsets)-1-index]) + } + assert.Equal(t, expected, actual) + }) + } +} + +func TestJournalEntryOffsetIteratorHandlesUnreferencedData(t *testing.T) { + contents, dataOffset := testJournalDataContents(t, []byte("MESSAGE=unused"), 0, false) + view, err := newJournalFileView("unused.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(dataOffset)) + require.NoError(t, err) + iterator, err := view.entryOffsetsForData(data) + require.NoError(t, err) + + _, found, err := iterator.previous() + require.NoError(t, err) + assert.False(t, found) +} + +func TestJournalEntryOffsetIteratorRejectsUnsortedItems(t *testing.T) { + contents, dataOffset, _ := testJournalEntryArrayContents(false) + firstArrayOffset := binary.LittleEndian.Uint64(contents[dataOffset+48 : dataOffset+56]) + firstItem := firstArrayOffset + journalEntryArrayHeaderSize + first := binary.LittleEndian.Uint64(contents[firstItem : firstItem+8]) + binary.LittleEndian.PutUint64(contents[firstItem+8:firstItem+16], first-8) + + view, err := newJournalFileView("unsorted.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(dataOffset)) + require.NoError(t, err) + iterator, err := view.entryOffsetsForData(data) + require.NoError(t, err) + + for { + _, found, err := iterator.previous() + if err != nil { + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "not strictly increasing") + break + } + require.True(t, found) + } +} + +func testJournalEntryContents(compact bool) ([]byte, int, int) { + headerFlags := uint32(0) + dataHeaderSize := journalDataRegularHeaderSize + itemSize := journalEntryRegularItemSize + if compact { + headerFlags = journalHeaderIncompatibleCompact + dataHeaderSize = journalDataCompactHeaderSize + itemSize = journalEntryCompactItemSize + } + dataOffset := journalHeaderCurrentSize + entryOffset := alignJournalTestSize(dataOffset + dataHeaderSize) + fileSize := alignJournalTestSize(entryOffset + journalEntryHeaderSize + itemSize) + contents := testJournalContents(fileSize, 0, headerFlags) + binary.LittleEndian.PutUint64(contents[136:144], uint64(entryOffset)) + binary.LittleEndian.PutUint64(contents[152:160], 1) + + data := contents[dataOffset:] + data[0] = journalObjectData + binary.LittleEndian.PutUint64(data[8:16], uint64(dataHeaderSize)) + + entry := contents[entryOffset:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], uint64(journalEntryHeaderSize+itemSize)) + binary.LittleEndian.PutUint64(entry[16:24], 42) + binary.LittleEndian.PutUint64(entry[24:32], 1_700_000_000_000_000) + binary.LittleEndian.PutUint64(entry[32:40], 1234) + copy(entry[40:56], bytes.Repeat([]byte{0x11}, 16)) + if compact { + binary.LittleEndian.PutUint32(entry[64:68], uint32(dataOffset)) + } else { + binary.LittleEndian.PutUint64(entry[64:72], uint64(dataOffset)) + binary.LittleEndian.PutUint64(entry[72:80], 0x0102030405060708) + } + return contents, entryOffset, dataOffset +} + +func testJournalEntryArrayContents(compact bool) ([]byte, int, []int) { + headerFlags := uint32(0) + dataHeaderSize := journalDataRegularHeaderSize + arrayItemSize := journalEntryArrayRegularItemSize + if compact { + headerFlags = journalHeaderIncompatibleCompact + dataHeaderSize = journalDataCompactHeaderSize + arrayItemSize = journalEntryCompactItemSize + } + dataOffset := journalHeaderCurrentSize + nextOffset := alignJournalTestSize(dataOffset + dataHeaderSize) + entryOffsets := make([]int, 6) + for index := range entryOffsets { + entryOffsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalEntryHeaderSize) + } + firstArrayOffset := nextOffset + firstArrayCapacity := 4 + nextOffset = alignJournalTestSize(nextOffset + journalEntryArrayHeaderSize + firstArrayCapacity*arrayItemSize) + secondArrayOffset := nextOffset + secondArrayCapacity := 4 + nextOffset = alignJournalTestSize(nextOffset + journalEntryArrayHeaderSize + secondArrayCapacity*arrayItemSize) + + contents := testJournalContents(nextOffset, 0, headerFlags) + binary.LittleEndian.PutUint64(contents[136:144], uint64(secondArrayOffset)) + binary.LittleEndian.PutUint64(contents[144:152], 9) + binary.LittleEndian.PutUint64(contents[152:160], uint64(len(entryOffsets))) + data := contents[dataOffset:] + data[0] = journalObjectData + binary.LittleEndian.PutUint64(data[8:16], uint64(dataHeaderSize)) + binary.LittleEndian.PutUint64(data[40:48], uint64(entryOffsets[0])) + binary.LittleEndian.PutUint64(data[48:56], uint64(firstArrayOffset)) + binary.LittleEndian.PutUint64(data[56:64], uint64(len(entryOffsets))) + if compact { + binary.LittleEndian.PutUint32(data[64:68], uint32(secondArrayOffset)) + binary.LittleEndian.PutUint32(data[68:72], 1) + } + for _, offset := range entryOffsets { + entry := contents[offset:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], journalEntryHeaderSize) + } + + firstArray := contents[firstArrayOffset:] + firstArray[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(firstArray[8:16], uint64(journalEntryArrayHeaderSize+firstArrayCapacity*arrayItemSize)) + binary.LittleEndian.PutUint64(firstArray[16:24], uint64(secondArrayOffset)) + secondArray := contents[secondArrayOffset:] + secondArray[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(secondArray[8:16], uint64(journalEntryArrayHeaderSize+secondArrayCapacity*arrayItemSize)) + for index, offset := range entryOffsets[1:] { + array := firstArray + itemIndex := index + if index >= firstArrayCapacity { + array = secondArray + itemIndex -= firstArrayCapacity + } + position := journalEntryArrayHeaderSize + itemIndex*arrayItemSize + if compact { + binary.LittleEndian.PutUint32(array[position:position+4], uint32(offset)) + } else { + binary.LittleEndian.PutUint64(array[position:position+8], uint64(offset)) + } + } + return contents, dataOffset, entryOffsets +} diff --git a/internal/systemd/journal_file_unix.go b/internal/systemd/journal_file_unix.go new file mode 100644 index 00000000..8e77a7a1 --- /dev/null +++ b/internal/systemd/journal_file_unix.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux || darwin + +package systemd + +import ( + "fmt" + "math" + "os" + "syscall" +) + +const journalBlockSize = 512 + +type journalFileStat struct { + dev uint64 + ino uint64 + nlink uint64 + blocks uint64 + allocated uint64 +} + +func journalStat(info os.FileInfo) (journalFileStat, error) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Blocks < 0 { + return journalFileStat{}, fmt.Errorf("allocation metadata is unavailable") + } + blocks := uint64(stat.Blocks) + if blocks > math.MaxUint64/journalBlockSize { + return journalFileStat{}, fmt.Errorf("allocation metadata overflows") + } + return journalFileStat{ + dev: uint64(stat.Dev), + ino: uint64(stat.Ino), + nlink: uint64(stat.Nlink), + blocks: blocks, + allocated: blocks * journalBlockSize, + }, nil +} diff --git a/internal/systemd/journal_files.go b/internal/systemd/journal_files.go new file mode 100644 index 00000000..de90ab1c --- /dev/null +++ b/internal/systemd/journal_files.go @@ -0,0 +1,118 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + maxJournalFiles = 4096 + maxMachineIDFileSize = 64 +) + +func (c *Client) journalFiles() (string, []string, error) { + if c.target.MachineIDPath == "" { + return "", nil, fmt.Errorf("systemd target machine ID path is not configured") + } + if len(c.target.JournalDirs) == 0 { + return "", nil, fmt.Errorf("systemd target journal directories are not configured") + } + + machineID, err := readMachineID(c.target.MachineIDPath) + if err != nil { + return "", nil, err + } + + files := make([]string, 0) + for _, baseDir := range c.target.JournalDirs { + dirPath := filepath.Join(baseDir, machineID) + dir, err := os.Open(dirPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return "", nil, fmt.Errorf("open journal directory %q: %w", dirPath, err) + } + + entries, readErr := dir.ReadDir(maxJournalFiles + 1) + closeErr := dir.Close() + if readErr != nil && readErr != io.EOF { + return "", nil, fmt.Errorf("read journal directory %q: %w", dirPath, readErr) + } + if closeErr != nil { + return "", nil, fmt.Errorf("close journal directory %q: %w", dirPath, closeErr) + } + if len(entries) > maxJournalFiles { + return "", nil, fmt.Errorf("journal directory %q has too many entries (maximum %d)", dirPath, maxJournalFiles) + } + + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".journal") || entry.Type()&fs.ModeSymlink != 0 { + continue + } + info, err := entry.Info() + if err != nil { + return "", nil, fmt.Errorf("inspect journal file %q: %w", filepath.Join(dirPath, entry.Name()), err) + } + if !info.Mode().IsRegular() { + continue + } + files = append(files, filepath.Join(dirPath, entry.Name())) + if len(files) > maxJournalFiles { + return "", nil, fmt.Errorf("systemd target has too many journal files (maximum %d)", maxJournalFiles) + } + } + } + + sort.Strings(files) + return machineID, files, nil +} + +func readMachineID(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open systemd machine ID %q: %w", path, err) + } + data, readErr := io.ReadAll(io.LimitReader(file, maxMachineIDFileSize+1)) + closeErr := file.Close() + if readErr != nil { + return "", fmt.Errorf("read systemd machine ID %q: %w", path, readErr) + } + if closeErr != nil { + return "", fmt.Errorf("close systemd machine ID %q: %w", path, closeErr) + } + if len(data) > maxMachineIDFileSize { + return "", fmt.Errorf("systemd machine ID file %q is too large", path) + } + + machineID := strings.TrimSpace(string(data)) + if len(machineID) != 32 { + return "", fmt.Errorf("systemd machine ID in %q must contain exactly 32 hexadecimal characters", path) + } + if !validID128(machineID) { + return "", fmt.Errorf("systemd machine ID in %q contains a non-hexadecimal character", path) + } + return strings.ToLower(machineID), nil +} + +func validID128(value string) bool { + if len(value) != 32 { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go new file mode 100644 index 00000000..ecb09111 --- /dev/null +++ b/internal/systemd/journal_files_test.go @@ -0,0 +1,77 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadMachineIDValidatesAndNormalizes(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "machine-id") + require.NoError(t, os.WriteFile(path, []byte("0123456789ABCDEF0123456789ABCDEF\n"), 0o600)) + + machineID, err := readMachineID(path) + require.NoError(t, err) + assert.Equal(t, "0123456789abcdef0123456789abcdef", machineID) + + require.NoError(t, os.WriteFile(path, []byte("not-a-machine-id\n"), 0o600)) + _, err = readMachineID(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "exactly 32 hexadecimal characters") + + require.NoError(t, os.WriteFile(path, []byte(strings.Repeat("a", maxMachineIDFileSize+1)), 0o600)) + _, err = readMachineID(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "too large") +} + +func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + + firstBase := filepath.Join(root, "run-journal") + secondBase := filepath.Join(root, "var-journal") + firstMachineDir := filepath.Join(firstBase, machineID) + secondMachineDir := filepath.Join(secondBase, machineID) + require.NoError(t, os.MkdirAll(firstMachineDir, 0o700)) + require.NoError(t, os.MkdirAll(secondMachineDir, 0o700)) + + firstJournal := filepath.Join(firstMachineDir, "system.journal") + secondJournal := filepath.Join(secondMachineDir, "system@0001.journal") + require.NoError(t, os.WriteFile(firstJournal, nil, 0o600)) + require.NoError(t, os.WriteFile(secondJournal, nil, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(firstMachineDir, "ignored.txt"), nil, 0o600)) + require.NoError(t, os.Mkdir(filepath.Join(firstMachineDir, "directory.journal"), 0o700)) + if runtime.GOOS != "windows" { + require.NoError(t, os.Symlink(firstJournal, filepath.Join(firstMachineDir, "link.journal"))) + } + + client := NewClient(Target{ + JournalDirs: []string{secondBase, firstBase, filepath.Join(root, "missing")}, + MachineIDPath: machineIDPath, + }) + gotMachineID, files, err := client.journalFiles() + require.NoError(t, err) + assert.Equal(t, machineID, gotMachineID) + assert.Equal(t, []string{firstJournal, secondJournal}, files) +} + +func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { + client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}) + _, _, err := client.journalFiles() + require.Error(t, err) + assert.Contains(t, err.Error(), "journal directories are not configured") +} diff --git a/internal/systemd/journal_fixture_test.go b/internal/systemd/journal_fixture_test.go new file mode 100644 index 00000000..8804360a --- /dev/null +++ b/internal/systemd/journal_fixture_test.go @@ -0,0 +1,213 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +const journalFixtureSize = 8 * 1024 * 1024 + +const journalFixtureDirectoryEnv = "RSHELL_JOURNAL_FIXTURE_DIR" + +func TestRealJournalFixtures(t *testing.T) { + longMessage := journalFixtureLongMessage() + bootID := repeatedJournalID(0xbb) + tests := []struct { + name string + compact bool + keyedHash bool + compression uint8 + }{ + {name: "regular-uncompressed.journal.gz"}, + { + name: "compact-keyed-zstd.journal.gz", + compact: true, + keyedHash: true, + compression: journalObjectCompressedZSTD, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := readJournalFixture(t, test.name) + view, err := newJournalFileView(test.name, bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", view.header.machineID.String()) + assert.Equal(t, uint64(4), view.header.nEntries) + assert.Equal(t, test.compact, view.header.compact()) + assert.Equal(t, test.keyedHash, view.header.keyedHash()) + + messageData, found, err := view.findDataObject([]byte("MESSAGE=" + longMessage)) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, test.compression, messageData.flags) + payload, truncated, err := view.readDataPayload(messageData, len(longMessage)+len("MESSAGE=")) + require.NoError(t, err) + assert.False(t, truncated) + assert.Equal(t, "MESSAGE="+longMessage, string(payload)) + + unitIterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + CurrentBoot: true, + MaxEntries: 10, + }, &bootID) + require.NoError(t, err) + unitEntries := collectJournalQueryEntries(t, unitIterator) + require.Len(t, unitEntries, 3) + assert.Equal(t, []string{longMessage, "manager noticed service", "service started"}, journalMessages(unitEntries)) + assert.Equal(t, "systemd", unitEntries[1].selected.Identifier) + assert.Equal(t, "1", unitEntries[1].selected.PID) + + kernelIterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Kernel: true, + CurrentBoot: true, + MaxEntries: 10, + }, &bootID) + require.NoError(t, err) + kernelEntries := collectJournalQueryEntries(t, kernelIterator) + require.Len(t, kernelEntries, 1) + assert.Equal(t, "synthetic kernel event", kernelEntries[0].selected.Message) + assert.Equal(t, "kernel", kernelEntries[0].selected.Identifier) + }) + } +} + +func TestJournalFixtureMatchesJournalctl(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("journalctl differential check requires Linux") + } + journalctl, err := exec.LookPath("journalctl") + if err != nil { + t.Skip("journalctl is not installed") + } + + path := filepath.Join(t.TempDir(), "fixture.journal") + require.NoError(t, os.WriteFile(path, readJournalFixture(t, "regular-uncompressed.journal.gz"), 0o600)) + + unitOutput := runFixtureJournalctl(t, journalctl, path, "--unit=rshell-fixture.service") + assert.Equal(t, strings.Join([]string{ + "service started", + "manager noticed service", + journalFixtureLongMessage(), + "", + }, "\n"), unitOutput) + + kernelOutput := runFixtureJournalctl(t, journalctl, path, "--dmesg") + assert.Equal(t, "synthetic kernel event\n", kernelOutput) +} + +func TestReadJournalUsesPureGoFixtureBackend(t *testing.T) { + root := t.TempDir() + machineID := repeatedJournalID(0xaa) + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID.String()+"\n"), 0o600)) + baseDir := filepath.Join(root, "journal") + journalDir := filepath.Join(baseDir, machineID.String()) + require.NoError(t, os.MkdirAll(journalDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(journalDir, "system.journal"), + readJournalFixture(t, "compact-keyed-zstd.journal.gz"), + 0o600, + )) + + client := NewClient(Target{JournalDirs: []string{baseDir}, MachineIDPath: machineIDPath}) + var entries []builtins.JournalEntry + err := client.ReadJournal(context.Background(), builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + CurrentBoot: true, + MaxEntries: 2, + }, func(entry builtins.JournalEntry) error { + entries = append(entries, entry) + return nil + }) + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, []string{"manager noticed service", journalFixtureLongMessage()}, []string{entries[0].Message, entries[1].Message}) +} + +func TestReadJournalReturnsCanceledContextBeforeFilesystemAccess(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := NewClient(Target{}).ReadJournal(ctx, builtins.JournalQuery{ + Units: []string{"api.service"}, + MaxEntries: 1, + }, func(builtins.JournalEntry) error { return nil }) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestRealJournalFixtureRejectsCorruptedCompressedData(t *testing.T) { + contents := readJournalFixture(t, "compact-keyed-zstd.journal.gz") + view, err := newJournalFileView("corrupt-compressed.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, found, err := view.findDataObject([]byte("MESSAGE=" + journalFixtureLongMessage())) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint8(journalObjectCompressedZSTD), data.flags) + contents[data.payloadOffset+data.payloadSize/2] ^= 0xff + + view, err = newJournalFileView("corrupt-compressed.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + MaxEntries: 10, + }, nil) + require.NoError(t, err) + _, _, err = iterator.previous(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) +} + +func readJournalFixture(t *testing.T, name string) []byte { + t.Helper() + directory := os.Getenv(journalFixtureDirectoryEnv) + if directory == "" { + directory = filepath.Join("testdata", "journal") + } + file, err := os.Open(filepath.Join(directory, name)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, file.Close()) }) + reader, err := gzip.NewReader(file) + require.NoError(t, err) + contents, err := io.ReadAll(io.LimitReader(reader, journalFixtureSize+1)) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Len(t, contents, journalFixtureSize) + return contents +} + +func journalFixtureLongMessage() string { + return "compressed payload " + strings.TrimSpace(strings.Repeat(strings.Repeat("a", 64)+" ", 10)) +} + +func runFixtureJournalctl(t *testing.T, executable, journalPath string, selection ...string) string { + t.Helper() + arguments := []string{"--file=" + journalPath, "--no-pager", "--quiet", "--output=cat"} + arguments = append(arguments, selection...) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, executable, arguments...) + var stderr bytes.Buffer + command.Stderr = &stderr + output, err := command.Output() + require.NoError(t, err, stderr.String()) + return string(output) +} diff --git a/internal/systemd/journal_format.go b/internal/systemd/journal_format.go new file mode 100644 index 00000000..77f63e0d --- /dev/null +++ b/internal/systemd/journal_format.go @@ -0,0 +1,377 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "math" +) + +const ( + journalHeaderMinSize = 208 + journalHeaderCurrentSize = 272 + journalObjectHeaderSize = 16 + + journalStateOffline = 0 + journalStateOnline = 1 + journalStateArchived = 2 + + journalObjectData = 1 + journalObjectField = 2 + journalObjectEntry = 3 + journalObjectDataHashTable = 4 + journalObjectFieldHashTable = 5 + journalObjectEntryArray = 6 + journalObjectTag = 7 + + journalObjectCompressedXZ = 1 << 0 + journalObjectCompressedLZ4 = 1 << 1 + journalObjectCompressedZSTD = 1 << 2 + journalObjectCompressionMask = journalObjectCompressedXZ | + journalObjectCompressedLZ4 | + journalObjectCompressedZSTD + + journalHeaderIncompatibleCompressedXZ = 1 << 0 + journalHeaderIncompatibleCompressedLZ4 = 1 << 1 + journalHeaderIncompatibleKeyedHash = 1 << 2 + journalHeaderIncompatibleCompressedZSTD = 1 << 3 + journalHeaderIncompatibleCompact = 1 << 4 + journalHeaderKnownIncompatible = journalHeaderIncompatibleCompressedXZ | + journalHeaderIncompatibleCompressedLZ4 | + journalHeaderIncompatibleKeyedHash | + journalHeaderIncompatibleCompressedZSTD | + journalHeaderIncompatibleCompact +) + +var ( + errJournalCorrupt = errors.New("corrupt systemd journal") + errJournalUnsupported = errors.New("unsupported systemd journal format") + errJournalLimit = errors.New("systemd journal resource limit exceeded") + journalSignature = [8]byte{'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H'} +) + +type journalID [16]byte + +func (id journalID) String() string { + return hex.EncodeToString(id[:]) +} + +func (id journalID) zero() bool { + return id == journalID{} +} + +type journalHeader struct { + compatibleFlags uint32 + incompatibleFlags uint32 + state uint8 + fileID journalID + machineID journalID + tailEntryBootID journalID + seqnumID journalID + headerSize uint64 + arenaSize uint64 + arenaEnd uint64 + + dataHashTableOffset uint64 + dataHashTableSize uint64 + fieldHashTableOffset uint64 + fieldHashTableSize uint64 + tailObjectOffset uint64 + nObjects uint64 + nEntries uint64 + tailEntrySeqnum uint64 + headEntrySeqnum uint64 + entryArrayOffset uint64 + headEntryRealtime uint64 + tailEntryRealtime uint64 + tailEntryMonotonic uint64 + + tailEntryArrayOffset uint32 + tailEntryArrayNEntries uint32 + hasTailEntryArray bool + tailEntryOffset uint64 + hasTailEntryOffset bool +} + +func (h journalHeader) compact() bool { + return h.incompatibleFlags&journalHeaderIncompatibleCompact != 0 +} + +func (h journalHeader) keyedHash() bool { + return h.incompatibleFlags&journalHeaderIncompatibleKeyedHash != 0 +} + +type journalObject struct { + offset uint64 + objectType uint8 + flags uint8 + size uint64 +} + +type journalFileView struct { + name string + reader io.ReaderAt + size uint64 + header journalHeader +} + +func newJournalFileView(name string, reader io.ReaderAt, size uint64) (*journalFileView, error) { + if reader == nil { + return nil, fmt.Errorf("journal reader is nil") + } + if size > math.MaxInt64 { + return nil, journalUnsupported(name, "file is larger than the supported reader offset range") + } + + header, err := readJournalHeader(name, reader, size) + if err != nil { + return nil, err + } + return &journalFileView{name: name, reader: reader, size: size, header: header}, nil +} + +func readJournalHeader(name string, reader io.ReaderAt, fileSize uint64) (journalHeader, error) { + if fileSize < journalHeaderMinSize { + return journalHeader{}, journalCorrupt(name, 0, "file is smaller than the minimum %d-byte header", journalHeaderMinSize) + } + + var raw [journalHeaderCurrentSize]byte + if err := readJournalAt(name, reader, fileSize, 0, raw[:journalHeaderMinSize]); err != nil { + return journalHeader{}, err + } + if string(raw[:len(journalSignature)]) != string(journalSignature[:]) { + return journalHeader{}, journalCorrupt(name, 0, "invalid journal signature") + } + + headerSize := binary.LittleEndian.Uint64(raw[88:96]) + if headerSize < journalHeaderMinSize { + return journalHeader{}, journalCorrupt(name, 88, "header size %d is smaller than %d", headerSize, journalHeaderMinSize) + } + if headerSize%8 != 0 { + return journalHeader{}, journalCorrupt(name, 88, "header size %d is not 8-byte aligned", headerSize) + } + if headerSize > fileSize { + return journalHeader{}, journalCorrupt(name, 88, "header size %d exceeds the %d-byte file", headerSize, fileSize) + } + + readSize := headerSize + if readSize > journalHeaderCurrentSize { + readSize = journalHeaderCurrentSize + } + if readSize > journalHeaderMinSize { + if err := readJournalAt(name, reader, fileSize, journalHeaderMinSize, raw[journalHeaderMinSize:readSize]); err != nil { + return journalHeader{}, err + } + } + + incompatibleFlags := binary.LittleEndian.Uint32(raw[12:16]) + if unknown := incompatibleFlags &^ uint32(journalHeaderKnownIncompatible); unknown != 0 { + return journalHeader{}, journalUnsupported(name, "unknown incompatible feature flags 0x%08x", unknown) + } + state := raw[16] + if state > journalStateArchived { + return journalHeader{}, journalCorrupt(name, 16, "invalid journal state %d", state) + } + + arenaSize := binary.LittleEndian.Uint64(raw[96:104]) + if arenaSize > math.MaxUint64-headerSize { + return journalHeader{}, journalCorrupt(name, 96, "header and arena sizes overflow") + } + arenaEnd := headerSize + arenaSize + if arenaEnd > fileSize { + return journalHeader{}, journalCorrupt(name, 96, "journal arena ends at %d beyond the %d-byte file", arenaEnd, fileSize) + } + + header := journalHeader{ + compatibleFlags: binary.LittleEndian.Uint32(raw[8:12]), + incompatibleFlags: incompatibleFlags, + state: state, + headerSize: headerSize, + arenaSize: arenaSize, + arenaEnd: arenaEnd, + dataHashTableOffset: binary.LittleEndian.Uint64(raw[104:112]), + dataHashTableSize: binary.LittleEndian.Uint64(raw[112:120]), + fieldHashTableOffset: binary.LittleEndian.Uint64(raw[120:128]), + fieldHashTableSize: binary.LittleEndian.Uint64(raw[128:136]), + tailObjectOffset: binary.LittleEndian.Uint64(raw[136:144]), + nObjects: binary.LittleEndian.Uint64(raw[144:152]), + nEntries: binary.LittleEndian.Uint64(raw[152:160]), + tailEntrySeqnum: binary.LittleEndian.Uint64(raw[160:168]), + headEntrySeqnum: binary.LittleEndian.Uint64(raw[168:176]), + entryArrayOffset: binary.LittleEndian.Uint64(raw[176:184]), + headEntryRealtime: binary.LittleEndian.Uint64(raw[184:192]), + tailEntryRealtime: binary.LittleEndian.Uint64(raw[192:200]), + tailEntryMonotonic: binary.LittleEndian.Uint64(raw[200:208]), + } + copy(header.fileID[:], raw[24:40]) + copy(header.machineID[:], raw[40:56]) + copy(header.tailEntryBootID[:], raw[56:72]) + copy(header.seqnumID[:], raw[72:88]) + + if headerSize >= 264 { + header.tailEntryArrayOffset = binary.LittleEndian.Uint32(raw[256:260]) + header.tailEntryArrayNEntries = binary.LittleEndian.Uint32(raw[260:264]) + header.hasTailEntryArray = true + } + if headerSize >= 272 { + header.tailEntryOffset = binary.LittleEndian.Uint64(raw[264:272]) + header.hasTailEntryOffset = true + } + if header.hasTailEntryArray { + if (header.tailEntryArrayOffset == 0) != (header.tailEntryArrayNEntries == 0) { + return journalHeader{}, journalCorrupt(name, uint64(header.tailEntryArrayOffset), "tail entry array offset and count are inconsistent") + } + if header.tailEntryArrayOffset != 0 { + if err := validateJournalObjectOffset(name, header, uint64(header.tailEntryArrayOffset), "tail entry array"); err != nil { + return journalHeader{}, err + } + if header.entryArrayOffset == 0 || header.entryArrayOffset > uint64(header.tailEntryArrayOffset) { + return journalHeader{}, journalCorrupt(name, uint64(header.tailEntryArrayOffset), "tail entry array precedes the global entry array chain") + } + } + } + + for _, reference := range []struct { + offset uint64 + label string + }{ + {offset: header.tailObjectOffset, label: "tail object"}, + {offset: header.entryArrayOffset, label: "entry array"}, + {offset: header.tailEntryOffset, label: "tail entry"}, + } { + if reference.offset == 0 { + continue + } + if err := validateJournalObjectOffset(name, header, reference.offset, reference.label); err != nil { + return journalHeader{}, err + } + } + if err := validateJournalHashTableRange(name, header, header.dataHashTableOffset, header.dataHashTableSize, "data"); err != nil { + return journalHeader{}, err + } + if err := validateJournalHashTableRange(name, header, header.fieldHashTableOffset, header.fieldHashTableSize, "field"); err != nil { + return journalHeader{}, err + } + + return header, nil +} + +func validateJournalObjectOffset(name string, header journalHeader, offset uint64, label string) error { + if offset%8 != 0 { + return journalCorrupt(name, offset, "%s offset is not 8-byte aligned", label) + } + if offset < header.headerSize { + return journalCorrupt(name, offset, "%s offset points into the journal header", label) + } + if offset > header.arenaEnd || header.arenaEnd-offset < journalObjectHeaderSize { + return journalCorrupt(name, offset, "%s offset does not contain a complete object header", label) + } + if header.tailObjectOffset != 0 && offset > header.tailObjectOffset { + return journalCorrupt(name, offset, "%s offset is after the tail object", label) + } + return nil +} + +func validateJournalHashTableRange(name string, header journalHeader, offset, size uint64, label string) error { + if (offset == 0) != (size == 0) { + return journalCorrupt(name, offset, "%s hash table offset and size are inconsistent", label) + } + if offset == 0 { + return nil + } + if offset < journalObjectHeaderSize { + return journalCorrupt(name, offset, "%s hash table payload offset is invalid", label) + } + objectOffset := offset - journalObjectHeaderSize + if err := validateJournalObjectOffset(name, header, objectOffset, label+" hash table"); err != nil { + return err + } + if size%16 != 0 { + return journalCorrupt(name, offset, "%s hash table size %d is not a multiple of 16", label, size) + } + if size > header.arenaEnd-offset { + return journalCorrupt(name, offset, "%s hash table extends beyond the journal arena", label) + } + return nil +} + +func (f *journalFileView) objectAt(offset uint64, expectedType uint8) (journalObject, error) { + if err := validateJournalObjectOffset(f.name, f.header, offset, "object"); err != nil { + return journalObject{}, err + } + + var raw [journalObjectHeaderSize]byte + if err := readJournalAt(f.name, f.reader, f.size, offset, raw[:]); err != nil { + return journalObject{}, err + } + object := journalObject{ + offset: offset, + objectType: raw[0], + flags: raw[1], + size: binary.LittleEndian.Uint64(raw[8:16]), + } + if expectedType != 0 && object.objectType != expectedType { + return journalObject{}, journalCorrupt(f.name, offset, "expected object type %d, found %d", expectedType, object.objectType) + } + if unknown := object.flags &^ uint8(journalObjectCompressionMask); unknown != 0 { + return journalObject{}, journalUnsupported(f.name, "object at offset %d uses unknown flags 0x%02x", offset, unknown) + } + if object.flags != 0 && object.objectType != journalObjectData { + return journalObject{}, journalCorrupt(f.name, offset, "compression flags are set on non-DATA object type %d", object.objectType) + } + if object.flags != 0 && object.flags&(object.flags-1) != 0 { + return journalObject{}, journalCorrupt(f.name, offset, "DATA object has multiple compression flags") + } + for _, compression := range []struct { + objectFlag uint8 + headerFlag uint32 + name string + }{ + {objectFlag: journalObjectCompressedXZ, headerFlag: journalHeaderIncompatibleCompressedXZ, name: "XZ"}, + {objectFlag: journalObjectCompressedLZ4, headerFlag: journalHeaderIncompatibleCompressedLZ4, name: "LZ4"}, + {objectFlag: journalObjectCompressedZSTD, headerFlag: journalHeaderIncompatibleCompressedZSTD, name: "ZSTD"}, + } { + if object.flags&compression.objectFlag != 0 && f.header.incompatibleFlags&compression.headerFlag == 0 { + return journalObject{}, journalCorrupt(f.name, offset, "%s-compressed DATA object is not declared in the journal header", compression.name) + } + } + if object.size < journalObjectHeaderSize { + return journalObject{}, journalCorrupt(f.name, offset, "object size %d is smaller than its header", object.size) + } + if object.size > f.header.arenaEnd-offset { + return journalObject{}, journalCorrupt(f.name, offset, "object size %d extends beyond the journal arena", object.size) + } + return object, nil +} + +func readJournalAt(name string, reader io.ReaderAt, fileSize, offset uint64, destination []byte) error { + if offset > fileSize || uint64(len(destination)) > fileSize-offset { + return journalCorrupt(name, offset, "read of %d bytes exceeds the %d-byte file", len(destination), fileSize) + } + if offset > math.MaxInt64 { + return journalUnsupported(name, "object offset %d exceeds the supported reader range", offset) + } + n, err := reader.ReadAt(destination, int64(offset)) + if n != len(destination) { + return journalCorrupt(name, offset, "short read: got %d of %d bytes", n, len(destination)) + } + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("read systemd journal %q at offset %d: %w", name, offset, err) + } + return nil +} + +func journalCorrupt(name string, offset uint64, format string, arguments ...any) error { + return fmt.Errorf("%w %q at offset %d: %s", errJournalCorrupt, name, offset, fmt.Sprintf(format, arguments...)) +} + +func journalUnsupported(name, format string, arguments ...any) error { + return fmt.Errorf("%w %q: %s", errJournalUnsupported, name, fmt.Sprintf(format, arguments...)) +} diff --git a/internal/systemd/journal_format_test.go b/internal/systemd/journal_format_test.go new file mode 100644 index 00000000..748a8d0f --- /dev/null +++ b/internal/systemd/journal_format_test.go @@ -0,0 +1,256 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalFileViewParsesCurrentHeader(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize, 0, 0) + copy(contents[24:40], bytes.Repeat([]byte{0x11}, 16)) + copy(contents[40:56], bytes.Repeat([]byte{0x22}, 16)) + copy(contents[56:72], bytes.Repeat([]byte{0x33}, 16)) + copy(contents[72:88], bytes.Repeat([]byte{0x44}, 16)) + binary.LittleEndian.PutUint64(contents[152:160], 7) + binary.LittleEndian.PutUint64(contents[160:168], 11) + binary.LittleEndian.PutUint64(contents[168:176], 5) + binary.LittleEndian.PutUint64(contents[184:192], 1_700_000_000_000_000) + binary.LittleEndian.PutUint64(contents[192:200], 1_700_000_000_000_500) + + view, err := newJournalFileView("fixture.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + assert.Equal(t, journalStateArchived, int(view.header.state)) + assert.Equal(t, "11111111111111111111111111111111", view.header.fileID.String()) + assert.Equal(t, "22222222222222222222222222222222", view.header.machineID.String()) + assert.Equal(t, "33333333333333333333333333333333", view.header.tailEntryBootID.String()) + assert.Equal(t, "44444444444444444444444444444444", view.header.seqnumID.String()) + assert.Equal(t, uint64(7), view.header.nEntries) + assert.Equal(t, uint64(11), view.header.tailEntrySeqnum) + assert.Equal(t, uint64(5), view.header.headEntrySeqnum) + assert.Equal(t, uint64(1_700_000_000_000_000), view.header.headEntryRealtime) + assert.Equal(t, uint64(1_700_000_000_000_500), view.header.tailEntryRealtime) + assert.True(t, view.header.hasTailEntryArray) + assert.True(t, view.header.hasTailEntryOffset) + assert.False(t, view.header.compact()) + assert.False(t, view.header.keyedHash()) +} + +func TestJournalFileViewAcceptsMinimumAndFutureHeaders(t *testing.T) { + minimum := testJournalContents(journalHeaderMinSize, 0, 0) + minimumView, err := newJournalFileView("old.journal", bytes.NewReader(minimum), uint64(len(minimum))) + require.NoError(t, err) + assert.False(t, minimumView.header.hasTailEntryArray) + assert.False(t, minimumView.header.hasTailEntryOffset) + + future := testJournalContentsWithHeader(280, 280, 0, journalHeaderIncompatibleCompact|journalHeaderIncompatibleKeyedHash) + futureView, err := newJournalFileView("future.journal", bytes.NewReader(future), uint64(len(future))) + require.NoError(t, err) + assert.True(t, futureView.header.compact()) + assert.True(t, futureView.header.keyedHash()) +} + +func TestJournalFileViewRejectsInvalidHeaders(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + match string + target error + }{ + { + name: "signature", + mutate: func(contents []byte) { + contents[0] = 0 + }, + match: "invalid journal signature", + target: errJournalCorrupt, + }, + { + name: "unknown incompatible flag", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint32(contents[12:16], 1<<31) + }, + match: "unknown incompatible feature flags 0x80000000", + target: errJournalUnsupported, + }, + { + name: "invalid state", + mutate: func(contents []byte) { + contents[16] = 3 + }, + match: "invalid journal state 3", + target: errJournalCorrupt, + }, + { + name: "short header size", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint64(contents[88:96], journalHeaderMinSize-8) + }, + match: "header size 200 is smaller than 208", + target: errJournalCorrupt, + }, + { + name: "unaligned header size", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint64(contents[88:96], journalHeaderCurrentSize+1) + }, + match: "header size 273 is not 8-byte aligned", + target: errJournalCorrupt, + }, + { + name: "arena beyond file", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint64(contents[96:104], 8) + }, + match: "journal arena ends at 280 beyond the 272-byte file", + target: errJournalCorrupt, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize, 0, 0) + test.mutate(contents) + _, err := newJournalFileView("bad.journal", bytes.NewReader(contents), uint64(len(contents))) + require.Error(t, err) + assert.ErrorIs(t, err, test.target) + assert.Contains(t, err.Error(), test.match) + }) + } +} + +func TestJournalFileViewValidatesHeaderOffsets(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize+64, 0, 0) + binary.LittleEndian.PutUint64(contents[136:144], journalHeaderCurrentSize) + binary.LittleEndian.PutUint64(contents[176:184], journalHeaderCurrentSize+1) + + _, err := newJournalFileView("bad-offset.journal", bytes.NewReader(contents), uint64(len(contents))) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "entry array offset is not 8-byte aligned") +} + +func TestJournalFileViewReadsBoundedObjectHeader(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize+64, 0, 0) + binary.LittleEndian.PutUint64(contents[136:144], journalHeaderCurrentSize) + contents[journalHeaderCurrentSize] = journalObjectEntry + binary.LittleEndian.PutUint64(contents[journalHeaderCurrentSize+8:journalHeaderCurrentSize+16], 64) + + view, err := newJournalFileView("entry.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + object, err := view.objectAt(journalHeaderCurrentSize, journalObjectEntry) + require.NoError(t, err) + assert.Equal(t, uint8(journalObjectEntry), object.objectType) + assert.Equal(t, uint64(64), object.size) + + _, err = view.objectAt(journalHeaderCurrentSize, journalObjectData) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected object type 1, found 3") +} + +func TestJournalFileViewRejectsInvalidObjectHeaders(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + match string + target error + }{ + { + name: "unknown flags", + mutate: func(object []byte) { + object[1] = 1 << 7 + }, + match: "unknown flags 0x80", + target: errJournalUnsupported, + }, + { + name: "compressed non-data", + mutate: func(object []byte) { + object[1] = journalObjectCompressedZSTD + }, + match: "compression flags are set on non-DATA object", + target: errJournalCorrupt, + }, + { + name: "undeclared compression", + mutate: func(object []byte) { + object[0] = journalObjectData + object[1] = journalObjectCompressedZSTD + }, + match: "ZSTD-compressed DATA object is not declared", + target: errJournalCorrupt, + }, + { + name: "multiple compression algorithms", + mutate: func(object []byte) { + object[0] = journalObjectData + object[1] = journalObjectCompressedLZ4 | journalObjectCompressedZSTD + }, + match: "multiple compression flags", + target: errJournalCorrupt, + }, + { + name: "short object size", + mutate: func(object []byte) { + binary.LittleEndian.PutUint64(object[8:16], journalObjectHeaderSize-1) + }, + match: "object size 15 is smaller than its header", + target: errJournalCorrupt, + }, + { + name: "object beyond arena", + mutate: func(object []byte) { + binary.LittleEndian.PutUint64(object[8:16], 65) + }, + match: "object size 65 extends beyond the journal arena", + target: errJournalCorrupt, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize+64, 0, 0) + binary.LittleEndian.PutUint64(contents[136:144], journalHeaderCurrentSize) + object := contents[journalHeaderCurrentSize:] + object[0] = journalObjectEntry + binary.LittleEndian.PutUint64(object[8:16], 64) + test.mutate(object) + + view, err := newJournalFileView("bad-object.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + _, err = view.objectAt(journalHeaderCurrentSize, 0) + require.Error(t, err) + assert.True(t, errors.Is(err, test.target)) + assert.Contains(t, err.Error(), test.match) + }) + } +} + +func testJournalContents(size int, compatibleFlags, incompatibleFlags uint32) []byte { + headerSize := size + if headerSize > journalHeaderCurrentSize { + headerSize = journalHeaderCurrentSize + } + return testJournalContentsWithHeader(size, headerSize, compatibleFlags, incompatibleFlags) +} + +func testJournalContentsWithHeader(size, headerSize int, compatibleFlags, incompatibleFlags uint32) []byte { + contents := make([]byte, size) + copy(contents[:8], journalSignature[:]) + binary.LittleEndian.PutUint32(contents[8:12], compatibleFlags) + binary.LittleEndian.PutUint32(contents[12:16], incompatibleFlags) + contents[16] = journalStateArchived + binary.LittleEndian.PutUint64(contents[88:96], uint64(headerSize)) + binary.LittleEndian.PutUint64(contents[96:104], uint64(size-headerSize)) + return contents +} diff --git a/internal/systemd/journal_fuzz_test.go b/internal/systemd/journal_fuzz_test.go new file mode 100644 index 00000000..3b52313e --- /dev/null +++ b/internal/systemd/journal_fuzz_test.go @@ -0,0 +1,142 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalFuzzInput = 256 * 1024 + maxJournalFuzzEncodedData = 64 * 1024 + maxJournalFuzzPayloadRead = 4 * 1024 + journalFuzzTimeout = 10 * time.Millisecond +) + +// FuzzJournalObjects drives every bounded object decoder reachable from a +// parsed header. Corrupt and unsupported inputs are expected; the contract is +// that they return without panicking or allocating from untrusted sizes. +func FuzzJournalObjects(f *testing.F) { + f.Add([]byte{}, []byte("MESSAGE=x"), uint64(0)) + f.Add(testJournalContents(journalHeaderCurrentSize, 0, 0), []byte("MESSAGE=x"), uint64(journalHeaderCurrentSize)) + f.Add(buildJournalFuzzSeed(f, false), []byte("_SYSTEMD_UNIT=api.service"), uint64(journalHeaderCurrentSize)) + f.Add(buildJournalFuzzSeed(f, true), []byte("MESSAGE=ready"), uint64(journalHeaderCurrentSize)) + + f.Fuzz(func(t *testing.T, contents, selector []byte, rawOffset uint64) { + if len(contents) > maxJournalFuzzInput || len(selector) == 0 || len(selector) > 512 { + return + } + view, err := newJournalFileView("fuzz.journal", bytes.NewReader(contents), uint64(len(contents))) + if err != nil { + return + } + + offsets := []uint64{ + rawOffset &^ 7, + view.header.tailObjectOffset, + view.header.tailEntryOffset, + } + if view.header.dataHashTableOffset >= journalObjectHeaderSize { + offsets = append(offsets, view.header.dataHashTableOffset-journalObjectHeaderSize) + } + for _, offset := range offsets { + fuzzJournalObject(view, offset) + } + + data, found, err := view.findDataObject(selector) + if err == nil && found && data.payloadSize <= maxJournalFuzzEncodedData { + _, _, _ = view.readDataPayload(data, maxJournalFuzzPayloadRead) + } + _, _, _ = newestJournalEntry(view) + }) +} + +// FuzzJournalQuery mutates structurally valid regular and compact journals and +// runs the same indexed unit/kernel query path used by the builtin. +func FuzzJournalQuery(f *testing.F) { + f.Add(buildJournalFuzzSeed(f, false), "api.service", false, byte(10)) + f.Add(buildJournalFuzzSeed(f, true), "api.service", false, byte(10)) + f.Add(buildJournalFuzzSeed(f, false), "", true, byte(10)) + + f.Fuzz(func(t *testing.T, contents []byte, unit string, kernel bool, rawLimit byte) { + if len(contents) > maxJournalFuzzInput || len(unit) > 256 { + return + } + view, err := newJournalFileView("query-fuzz.journal", bytes.NewReader(contents), uint64(len(contents))) + if err != nil { + return + } + if view.header.incompatibleFlags&(journalHeaderIncompatibleCompressedXZ|journalHeaderIncompatibleCompressedLZ4|journalHeaderIncompatibleCompressedZSTD) != 0 { + return + } + + query := builtins.JournalQuery{Kernel: kernel, MaxEntries: int(rawLimit%10) + 1} + if !kernel { + query.Units = []string{unit} + } + if err := validateJournalQuery(query); err != nil { + return + } + + iterator, err := newJournalFileQueryIterator(view, query, nil) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), journalFuzzTimeout) + defer cancel() + for count := 0; count < query.MaxEntries; count++ { + entry, found, err := iterator.previous(ctx) + if err != nil || !found { + return + } + if len(entry.selected.Hostname) > maxJournalFieldSize || + len(entry.selected.Identifier) > maxJournalFieldSize || + len(entry.selected.PID) > maxJournalFieldSize || + len(entry.selected.Message) > maxJournalFieldSize { + t.Fatal("journal query returned a selected field over its size limit") + } + } + }) +} + +func fuzzJournalObject(view *journalFileView, offset uint64) { + if offset == 0 { + return + } + object, err := view.objectAt(offset, 0) + if err != nil { + return + } + switch object.objectType { + case journalObjectData: + data, err := view.dataObjectAt(offset) + if err == nil && data.payloadSize <= maxJournalFuzzEncodedData { + _, _, _ = view.readDataPayload(data, maxJournalFuzzPayloadRead) + } + case journalObjectEntry: + _, _ = view.entryObjectAt(offset) + case journalObjectEntryArray: + array, err := view.entryArrayAt(offset) + if err == nil && array.capacity > 0 { + _, _ = view.entryArrayItem(array, 0) + } + } +} + +func buildJournalFuzzSeed(t testing.TB, compact bool) []byte { + t.Helper() + bootID := repeatedJournalID(0x22) + return buildJournalQueryFixtureWithLayout(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "_HOSTNAME=node", "SYSLOG_IDENTIFIER=api", "_PID=42", "MESSAGE=ready"}}, + {bootID: bootID, realtime: 1_700_000_000_000_001, fields: []string{"UNIT=api.service", "_PID=1", "SYSLOG_IDENTIFIER=systemd", "MESSAGE=manager"}}, + {bootID: bootID, realtime: 1_700_000_000_000_002, fields: []string{"_TRANSPORT=kernel", "_COMM=kernel", "MESSAGE=kernel"}}, + }, compact) +} diff --git a/internal/systemd/journal_hash.go b/internal/systemd/journal_hash.go new file mode 100644 index 00000000..e2ab2430 --- /dev/null +++ b/internal/systemd/journal_hash.go @@ -0,0 +1,136 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "math/bits" +) + +// journalJenkinsHash64 implements lookup3 hashlittle2 with zero seeds, which +// older journal files use for DATA and FIELD objects. +func journalJenkinsHash64(data []byte) uint64 { + a := uint32(0xdeadbeef) + uint32(len(data)) + b := a + c := a + + for len(data) > 12 { + a += binary.LittleEndian.Uint32(data[0:4]) + b += binary.LittleEndian.Uint32(data[4:8]) + c += binary.LittleEndian.Uint32(data[8:12]) + a, b, c = journalJenkinsMix(a, b, c) + data = data[12:] + } + if len(data) == 0 { + return uint64(c)<<32 | uint64(b) + } + for index, value := range data { + shift := uint(index%4) * 8 + switch index / 4 { + case 0: + a += uint32(value) << shift + case 1: + b += uint32(value) << shift + case 2: + c += uint32(value) << shift + } + } + a, b, c = journalJenkinsFinal(a, b, c) + return uint64(c)<<32 | uint64(b) +} + +func journalJenkinsMix(a, b, c uint32) (uint32, uint32, uint32) { + a -= c + a ^= bits.RotateLeft32(c, 4) + c += b + b -= a + b ^= bits.RotateLeft32(a, 6) + a += c + c -= b + c ^= bits.RotateLeft32(b, 8) + b += a + a -= c + a ^= bits.RotateLeft32(c, 16) + c += b + b -= a + b ^= bits.RotateLeft32(a, 19) + a += c + c -= b + c ^= bits.RotateLeft32(b, 4) + b += a + return a, b, c +} + +func journalJenkinsFinal(a, b, c uint32) (uint32, uint32, uint32) { + c ^= b + c -= bits.RotateLeft32(b, 14) + a ^= c + a -= bits.RotateLeft32(c, 11) + b ^= a + b -= bits.RotateLeft32(a, 25) + c ^= b + c -= bits.RotateLeft32(b, 16) + a ^= c + a -= bits.RotateLeft32(c, 4) + b ^= a + b -= bits.RotateLeft32(a, 14) + c ^= b + c -= bits.RotateLeft32(b, 24) + return a, b, c +} + +// journalSipHash24 implements SipHash-2-4 with the journal file ID as its +// 128-bit key, which modern journal files use for DATA and FIELD objects. +func journalSipHash24(data []byte, key journalID) uint64 { + k0 := binary.LittleEndian.Uint64(key[0:8]) + k1 := binary.LittleEndian.Uint64(key[8:16]) + v0 := uint64(0x736f6d6570736575) ^ k0 + v1 := uint64(0x646f72616e646f6d) ^ k1 + v2 := uint64(0x6c7967656e657261) ^ k0 + v3 := uint64(0x7465646279746573) ^ k1 + length := len(data) + + for len(data) >= 8 { + message := binary.LittleEndian.Uint64(data[:8]) + v3 ^= message + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + v0 ^= message + data = data[8:] + } + + last := uint64(length) << 56 + for index, value := range data { + last |= uint64(value) << (uint(index) * 8) + } + v3 ^= last + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + v0 ^= last + v2 ^= 0xff + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + return v0 ^ v1 ^ v2 ^ v3 +} + +func journalSipRound(v0, v1, v2, v3 *uint64) { + *v0 += *v1 + *v1 = bits.RotateLeft64(*v1, 13) + *v1 ^= *v0 + *v0 = bits.RotateLeft64(*v0, 32) + *v2 += *v3 + *v3 = bits.RotateLeft64(*v3, 16) + *v3 ^= *v2 + *v0 += *v3 + *v3 = bits.RotateLeft64(*v3, 21) + *v3 ^= *v0 + *v2 += *v1 + *v1 = bits.RotateLeft64(*v1, 17) + *v1 ^= *v2 + *v2 = bits.RotateLeft64(*v2, 32) +} diff --git a/internal/systemd/journal_hash_test.go b/internal/systemd/journal_hash_test.go new file mode 100644 index 00000000..d852d2db --- /dev/null +++ b/internal/systemd/journal_hash_test.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJournalJenkinsHash64MatchesLookup3Vectors(t *testing.T) { + assert.Equal(t, uint64(0xdeadbeefdeadbeef), journalJenkinsHash64(nil)) + assert.Equal(t, uint64(0x17770551ce7226e6), journalJenkinsHash64([]byte("Four score and seven years ago"))) +} + +func TestJournalSipHash24MatchesReferenceVectors(t *testing.T) { + var key journalID + var message [64]byte + for index := range key { + key[index] = byte(index) + } + for index := range message { + message[index] = byte(index) + } + + vectors := []struct { + length int + hash uint64 + }{ + {length: 0, hash: 0x726fdb47dd0e0e31}, + {length: 1, hash: 0x74f839c593dc67fd}, + {length: 7, hash: 0xab0200f58b01d137}, + {length: 8, hash: 0x93f5f5799a932462}, + {length: 15, hash: 0xa129ca6149be45e5}, + {length: 63, hash: 0x958a324ceb064572}, + } + for _, vector := range vectors { + assert.Equal(t, vector.hash, journalSipHash24(message[:vector.length], key), "length %d", vector.length) + } +} diff --git a/internal/systemd/journal_index.go b/internal/systemd/journal_index.go new file mode 100644 index 00000000..30f0767c --- /dev/null +++ b/internal/systemd/journal_index.go @@ -0,0 +1,144 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +const ( + journalHashItemSize = 16 + maxJournalHashChainDepth = 128 +) + +func (f *journalFileView) dataHash(payload []byte) uint64 { + if f.header.keyedHash() { + return journalSipHash24(payload, f.header.fileID) + } + return journalJenkinsHash64(payload) +} + +func (f *journalFileView) findDataObject(payload []byte) (journalDataObject, bool, error) { + if len(payload) == 0 { + return journalDataObject{}, false, fmt.Errorf("journal DATA lookup payload is empty") + } + if len(payload) > maxJournalPayloadRead { + return journalDataObject{}, false, fmt.Errorf("journal DATA lookup payload exceeds %d bytes", maxJournalPayloadRead) + } + if f.header.dataHashTableSize == 0 { + return journalDataObject{}, false, nil + } + + tableObjectOffset := f.header.dataHashTableOffset - journalObjectHeaderSize + table, err := f.objectAt(tableObjectOffset, journalObjectDataHashTable) + if err != nil { + return journalDataObject{}, false, err + } + if table.size-journalObjectHeaderSize != f.header.dataHashTableSize { + return journalDataObject{}, false, journalCorrupt( + f.name, + tableObjectOffset, + "DATA hash table object payload is %d bytes; header declares %d", + table.size-journalObjectHeaderSize, + f.header.dataHashTableSize, + ) + } + bucketCount := f.header.dataHashTableSize / journalHashItemSize + if bucketCount == 0 { + return journalDataObject{}, false, journalCorrupt(f.name, f.header.dataHashTableOffset, "DATA hash table has no buckets") + } + + hash := f.dataHash(payload) + bucket := hash % bucketCount + itemOffset := f.header.dataHashTableOffset + bucket*journalHashItemSize + var item [journalHashItemSize]byte + if err := readJournalAt(f.name, f.reader, f.size, itemOffset, item[:]); err != nil { + return journalDataObject{}, false, err + } + headOffset := binary.LittleEndian.Uint64(item[0:8]) + tailOffset := binary.LittleEndian.Uint64(item[8:16]) + if (headOffset == 0) != (tailOffset == 0) { + return journalDataObject{}, false, journalCorrupt(f.name, itemOffset, "DATA hash bucket head and tail are inconsistent") + } + if headOffset == 0 { + return journalDataObject{}, false, nil + } + if err := validateJournalObjectOffset(f.name, f.header, headOffset, "DATA hash bucket head"); err != nil { + return journalDataObject{}, false, err + } + if err := validateJournalObjectOffset(f.name, f.header, tailOffset, "DATA hash bucket tail"); err != nil { + return journalDataObject{}, false, err + } + + seen := make(map[uint64]struct{}, maxJournalHashChainDepth) + for offset, depth := headOffset, 0; offset != 0; depth++ { + if depth >= maxJournalHashChainDepth { + return journalDataObject{}, false, journalLimit(f.name, offset, "DATA hash chain exceeds %d objects", maxJournalHashChainDepth) + } + if _, exists := seen[offset]; exists { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA hash chain contains a cycle") + } + seen[offset] = struct{}{} + + data, err := f.dataObjectAt(offset) + if err != nil { + return journalDataObject{}, false, err + } + if data.hash%bucketCount != bucket { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA object hash belongs to bucket %d instead of %d", data.hash%bucketCount, bucket) + } + if data.hash == hash { + equal, err := f.dataPayloadEqual(data, payload) + if err != nil { + return journalDataObject{}, false, err + } + if equal { + return data, true, nil + } + } + + if offset == tailOffset { + if data.nextHashOffset != 0 { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA hash bucket continues after its declared tail") + } + return journalDataObject{}, false, nil + } + if data.nextHashOffset == 0 { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA hash bucket ends before its declared tail") + } + offset = data.nextHashOffset + } + + return journalDataObject{}, false, journalCorrupt(f.name, itemOffset, "DATA hash bucket did not reach its declared tail") +} + +func (f *journalFileView) dataPayloadEqual(data journalDataObject, expected []byte) (bool, error) { + switch data.flags { + case 0: + if data.payloadSize != uint64(len(expected)) { + return false, nil + } + case journalObjectCompressedLZ4: + if data.payloadSize <= 8 { + return false, journalCorrupt(f.name, data.payloadOffset, "LZ4 DATA payload is too small") + } + var sizeBytes [8]byte + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset, sizeBytes[:]); err != nil { + return false, err + } + if binary.LittleEndian.Uint64(sizeBytes[:]) != uint64(len(expected)) { + return false, nil + } + } + + payload, truncated, err := f.readDataPayload(data, len(expected)) + if err != nil { + return false, err + } + return !truncated && bytes.Equal(payload, expected), nil +} diff --git a/internal/systemd/journal_index_test.go b/internal/systemd/journal_index_test.go new file mode 100644 index 00000000..80c26fc7 --- /dev/null +++ b/internal/systemd/journal_index_test.go @@ -0,0 +1,172 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalDataIndexFindsExactPayload(t *testing.T) { + for _, keyed := range []bool{false, true} { + name := "jenkins" + if keyed { + name = "siphash" + } + t.Run(name, func(t *testing.T) { + contents, offsets := testIndexedJournal(t, [][]byte{ + []byte("_SYSTEMD_UNIT=other.service"), + []byte("_SYSTEMD_UNIT=api.service"), + }, keyed) + view, err := newJournalFileView(name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + data, found, err := view.findDataObject([]byte("_SYSTEMD_UNIT=api.service")) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(offsets[1]), data.offset) + + _, found, err = view.findDataObject([]byte("_SYSTEMD_UNIT=missing.service")) + require.NoError(t, err) + assert.False(t, found) + }) + } +} + +func TestJournalDataIndexVerifiesPayloadAfterHashMatch(t *testing.T) { + target := []byte("_SYSTEMD_UNIT=api.service") + contents, offsets := testIndexedJournal(t, [][]byte{ + []byte("_SYSTEMD_UNIT=not-api.service"), + target, + }, false) + targetHash := journalJenkinsHash64(target) + binary.LittleEndian.PutUint64(contents[offsets[0]+16:offsets[0]+24], targetHash) + + view, err := newJournalFileView("collision.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, found, err := view.findDataObject(target) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(offsets[1]), data.offset) +} + +func TestJournalDataIndexRejectsCycle(t *testing.T) { + contents, offsets := testIndexedJournal(t, [][]byte{ + []byte("_SYSTEMD_UNIT=one.service"), + []byte("_SYSTEMD_UNIT=two.service"), + []byte("_SYSTEMD_UNIT=three.service"), + }, false) + binary.LittleEndian.PutUint64(contents[offsets[1]+24:offsets[1]+32], uint64(offsets[0])) + + view, err := newJournalFileView("cycle.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + _, _, err = view.findDataObject([]byte("_SYSTEMD_UNIT=missing.service")) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "contains a cycle") +} + +func TestJournalDataIndexRejectsWrongTableObject(t *testing.T) { + contents, _ := testIndexedJournal(t, [][]byte{[]byte("MESSAGE=hello")}, false) + contents[journalHeaderCurrentSize] = journalObjectFieldHashTable + view, err := newJournalFileView("table.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + _, _, err = view.findDataObject([]byte("MESSAGE=hello")) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "expected object type 4, found 5") +} + +func TestJournalDataIndexSkipsOversizedLZ4Collision(t *testing.T) { + target := []byte("MESSAGE=hello") + contents, offsets := testIndexedJournal(t, [][]byte{[]byte("MESSAGE=other"), target}, false) + targetHash := journalJenkinsHash64(target) + first := offsets[0] + binary.LittleEndian.PutUint64(contents[first+16:first+24], targetHash) + contents[first+1] = journalObjectCompressedLZ4 + binary.LittleEndian.PutUint32(contents[12:16], journalHeaderIncompatibleCompressedLZ4) + binary.LittleEndian.PutUint64(contents[first+64:first+72], maxJournalLZ4DataSize+1) + + view, err := newJournalFileView("large-collision.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, found, err := view.findDataObject(target) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(offsets[1]), data.offset) +} + +func TestJournalDataPayloadEqualDecodesCompressedValue(t *testing.T) { + payload := []byte("_SYSTEMD_UNIT=api.service") + encoded := encodeJournalZSTD(t, payload) + contents, offset := testJournalDataContents(t, encoded, journalObjectCompressedZSTD, false) + view, err := newJournalFileView("compressed-match.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + equal, err := view.dataPayloadEqual(data, payload) + require.NoError(t, err) + assert.True(t, equal) + equal, err = view.dataPayloadEqual(data, []byte("_SYSTEMD_UNIT=other.service")) + require.NoError(t, err) + assert.False(t, equal) +} + +func testIndexedJournal(t *testing.T, payloads [][]byte, keyed bool) ([]byte, []int) { + t.Helper() + require.NotEmpty(t, payloads) + + const bucketCount = 1 + tableOffset := journalHeaderCurrentSize + tableSize := journalObjectHeaderSize + bucketCount*journalHashItemSize + nextOffset := alignJournalTestSize(tableOffset + tableSize) + offsets := make([]int, len(payloads)) + for index, payload := range payloads { + offsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalDataRegularHeaderSize + len(payload)) + } + contents := testJournalContents(nextOffset, 0, 0) + if keyed { + binary.LittleEndian.PutUint32(contents[12:16], journalHeaderIncompatibleKeyedHash) + for index := 0; index < 16; index++ { + contents[24+index] = byte(index) + } + } + binary.LittleEndian.PutUint64(contents[104:112], uint64(tableOffset+journalObjectHeaderSize)) + binary.LittleEndian.PutUint64(contents[112:120], bucketCount*journalHashItemSize) + binary.LittleEndian.PutUint64(contents[136:144], uint64(offsets[len(offsets)-1])) + binary.LittleEndian.PutUint64(contents[144:152], uint64(len(payloads)+1)) + + table := contents[tableOffset:] + table[0] = journalObjectDataHashTable + binary.LittleEndian.PutUint64(table[8:16], uint64(tableSize)) + binary.LittleEndian.PutUint64(table[16:24], uint64(offsets[0])) + binary.LittleEndian.PutUint64(table[24:32], uint64(offsets[len(offsets)-1])) + + var fileID journalID + copy(fileID[:], contents[24:40]) + for index, payload := range payloads { + offset := offsets[index] + object := contents[offset:] + object[0] = journalObjectData + binary.LittleEndian.PutUint64(object[8:16], uint64(journalDataRegularHeaderSize+len(payload))) + hash := journalJenkinsHash64(payload) + if keyed { + hash = journalSipHash24(payload, fileID) + } + binary.LittleEndian.PutUint64(object[16:24], hash) + if index+1 < len(offsets) { + binary.LittleEndian.PutUint64(object[24:32], uint64(offsets[index+1])) + } + copy(object[journalDataRegularHeaderSize:], payload) + } + return contents, offsets +} diff --git a/internal/systemd/journal_merge.go b/internal/systemd/journal_merge.go new file mode 100644 index 00000000..bbf6f671 --- /dev/null +++ b/internal/systemd/journal_merge.go @@ -0,0 +1,298 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "errors" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalResultBytes = 8 * 1024 * 1024 +) + +type journalMergeSource struct { + iterator *journalFileQueryIterator + current *journalQueryEntry + done bool +} + +func (s *journalMergeSource) load(ctx context.Context) error { + if s.current != nil || s.done { + return nil + } + entry, found, err := s.iterator.previous(ctx) + if err != nil { + return err + } + if !found { + s.done = true + return nil + } + s.current = &entry + return nil +} + +func (c *Client) queryJournalEntries(ctx context.Context, query builtins.JournalQuery) ([]journalQueryEntry, error) { + if err := validateJournalQuery(query); err != nil { + return nil, err + } + if query.MaxEntries == 0 { + return nil, nil + } + + for attempt := 0; attempt < 2; attempt++ { + entries, err := c.queryJournalEntriesOnce(ctx, query) + if err == nil { + return entries, nil + } + if attempt == 0 && (errors.Is(err, errJournalChanged) || errors.Is(err, errJournalCorrupt)) { + continue + } + return nil, err + } + return nil, journalChanged("journal did not stabilize after retry") +} + +func (c *Client) queryJournalEntriesOnce(ctx context.Context, query builtins.JournalQuery) ([]journalQueryEntry, error) { + snapshot, err := c.openJournalSnapshot() + if err != nil { + return nil, err + } + entries, queryErr := queryJournalSnapshot(ctx, snapshot, query) + if queryErr == nil { + queryErr = snapshot.stable(c) + } + closeErr := snapshot.close() + if queryErr != nil { + return nil, queryErr + } + if closeErr != nil { + return nil, closeErr + } + return entries, nil +} + +func queryJournalSnapshot(ctx context.Context, snapshot *journalSnapshot, query builtins.JournalQuery) ([]journalQueryEntry, error) { + var currentBoot *journalID + if query.CurrentBoot { + bootID, err := newestJournalBoot(snapshot) + if err != nil { + return nil, err + } + currentBoot = &bootID + } + + sources := make([]*journalMergeSource, 0, len(snapshot.files)) + for _, opened := range snapshot.files { + iterator, err := newJournalFileQueryIterator(opened.view, query, currentBoot) + if err != nil { + return nil, err + } + sources = append(sources, &journalMergeSource{iterator: iterator}) + } + + entries := make([]journalQueryEntry, 0, query.MaxEntries) + resultBytes := 0 + for len(entries) < query.MaxEntries { + if err := ctx.Err(); err != nil { + return nil, err + } + var newest *journalQueryEntry + for _, source := range sources { + if err := source.load(ctx); err != nil { + return nil, err + } + if source.current != nil && (newest == nil || compareJournalQueryEntries(*source.current, *newest) > 0) { + newest = source.current + } + } + if newest == nil { + break + } + + selected := *newest + for _, source := range sources { + if source.current == nil || !sameJournalEntryLocation(*source.current, selected) { + continue + } + if source.current.selected != selected.selected { + return nil, journalCorrupt(source.iterator.file.name, source.current.offset, "duplicate journal location has conflicting selected fields") + } + source.current = nil + } + + resultBytes += len(selected.selected.Hostname) + len(selected.selected.Identifier) + len(selected.selected.PID) + len(selected.selected.Message) + if resultBytes > maxJournalResultBytes { + return nil, fmt.Errorf("%w: selected journal fields exceed %d bytes", errJournalLimit, maxJournalResultBytes) + } + entries = append(entries, selected) + } + return entries, nil +} + +func newestJournalBoot(snapshot *journalSnapshot) (journalID, error) { + var newest *journalQueryEntry + for _, opened := range snapshot.files { + entry, found, err := newestJournalEntry(opened.view) + if err != nil { + return journalID{}, err + } + if found && (newest == nil || compareJournalQueryEntries(entry, *newest) > 0) { + copy := entry + newest = © + } + } + if newest == nil { + return journalID{}, fmt.Errorf("could not determine the current boot from the selected journal") + } + return newest.bootID, nil +} + +func newestJournalEntry(file *journalFileView) (journalQueryEntry, bool, error) { + if file.header.nEntries == 0 { + return journalQueryEntry{}, false, nil + } + + var offset uint64 + if file.header.hasTailEntryOffset { + if file.header.tailEntryOffset == 0 { + return journalQueryEntry{}, false, journalCorrupt(file.name, 264, "journal has entries but no tail ENTRY offset") + } + offset = file.header.tailEntryOffset + } else { + iterator, err := file.globalEntryOffsets() + if err != nil { + return journalQueryEntry{}, false, err + } + var found bool + offset, found, err = iterator.previous() + if err != nil { + return journalQueryEntry{}, false, err + } + if !found { + return journalQueryEntry{}, false, journalCorrupt(file.name, file.header.entryArrayOffset, "global ENTRY_ARRAY is empty") + } + } + + entry, err := file.entryObjectAt(offset) + if err != nil { + return journalQueryEntry{}, false, err + } + if entry.seqnum == 0 || entry.realtime == 0 || entry.bootID.zero() { + return journalQueryEntry{}, false, journalCorrupt(file.name, entry.offset, "tail ENTRY has invalid sequence, timestamp, or boot metadata") + } + if file.header.tailEntrySeqnum != 0 && file.header.tailEntrySeqnum != entry.seqnum { + return journalQueryEntry{}, false, journalCorrupt(file.name, entry.offset, "tail ENTRY sequence %d does not match header sequence %d", entry.seqnum, file.header.tailEntrySeqnum) + } + return journalQueryEntry{ + fileID: file.header.fileID, + seqnumID: file.header.seqnumID, + bootID: entry.bootID, + seqnum: entry.seqnum, + realtime: entry.realtime, + monotonic: entry.monotonic, + xorHash: entry.xorHash, + offset: entry.offset, + }, true, nil +} + +func (f *journalFileView) globalEntryOffsets() (*journalEntryOffsetIterator, error) { + iterator := &journalEntryOffsetIterator{file: f, segmentIndex: -1} + if f.header.nEntries == 0 { + return iterator, nil + } + if f.header.entryArrayOffset == 0 { + return nil, journalCorrupt(f.name, 176, "journal has entries but no global ENTRY_ARRAY") + } + + remaining := f.header.nEntries + offset := f.header.entryArrayOffset + seen := make(map[uint64]struct{}, maxJournalEntryArrays) + for remaining > 0 { + if len(iterator.segments) >= maxJournalEntryArrays { + return nil, journalLimit(f.name, offset, "global ENTRY_ARRAY chain exceeds %d objects", maxJournalEntryArrays) + } + if _, exists := seen[offset]; exists { + return nil, journalCorrupt(f.name, offset, "global ENTRY_ARRAY chain contains a cycle") + } + seen[offset] = struct{}{} + + array, err := f.entryArrayAt(offset) + if err != nil { + return nil, err + } + count := array.capacity + if count > remaining { + count = remaining + } + iterator.segments = append(iterator.segments, journalEntryArraySegment{array: array, count: count}) + remaining -= count + if remaining == 0 { + if array.nextOffset != 0 { + return nil, journalCorrupt(f.name, array.offset, "global ENTRY_ARRAY continues after all entries") + } + break + } + if array.nextOffset == 0 { + return nil, journalCorrupt(f.name, array.offset, "global ENTRY_ARRAY ends before all entries") + } + offset = array.nextOffset + } + + if f.header.hasTailEntryArray { + last := iterator.segments[len(iterator.segments)-1] + if uint64(f.header.tailEntryArrayOffset) != last.array.offset || uint64(f.header.tailEntryArrayNEntries) != last.count { + return nil, journalCorrupt(f.name, 256, "global tail ENTRY_ARRAY metadata does not match its array chain") + } + } + iterator.segmentIndex = len(iterator.segments) - 1 + iterator.itemIndex = iterator.segments[iterator.segmentIndex].count + return iterator, nil +} + +func compareJournalQueryEntries(left, right journalQueryEntry) int { + if !left.seqnumID.zero() && left.seqnumID == right.seqnumID && left.seqnum != right.seqnum { + return compareUint64(left.seqnum, right.seqnum) + } + if !left.bootID.zero() && left.bootID == right.bootID && left.monotonic != right.monotonic { + return compareUint64(left.monotonic, right.monotonic) + } + if left.realtime != right.realtime { + return compareUint64(left.realtime, right.realtime) + } + if left.xorHash != right.xorHash { + return compareUint64(left.xorHash, right.xorHash) + } + if compared := bytes.Compare(left.fileID[:], right.fileID[:]); compared != 0 { + return compared + } + return compareUint64(left.offset, right.offset) +} + +func compareUint64(left, right uint64) int { + if left < right { + return -1 + } + if left > right { + return 1 + } + return 0 +} + +func sameJournalEntryLocation(left, right journalQueryEntry) bool { + return !left.seqnumID.zero() && + left.seqnumID == right.seqnumID && + left.seqnum == right.seqnum && + left.bootID == right.bootID && + left.realtime == right.realtime && + left.monotonic == right.monotonic && + left.xorHash == right.xorHash +} diff --git a/internal/systemd/journal_merge_test.go b/internal/systemd/journal_merge_test.go new file mode 100644 index 00000000..162d9f1b --- /dev/null +++ b/internal/systemd/journal_merge_test.go @@ -0,0 +1,159 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +func TestQueryJournalEntriesMergesRotatedFilesAndFindsCurrentBoot(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + sequenceID := repeatedJournalID(0x44) + oldBoot := repeatedJournalID(0x11) + currentBoot := repeatedJournalID(0x22) + start := time.Unix(1_700_000_000, 0) + + writeJournalSnapshotFixture(t, journalDir, "system@old.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 1, bootID: oldBoot, realtime: uint64(start.UnixMicro()), monotonic: 900, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=old boot"}}, + {seqnum: 2, bootID: currentBoot, realtime: uint64(start.Add(time.Second).UnixMicro()), monotonic: 100, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=started"}}, + }), machineID, repeatedJournalID(0x31), sequenceID) + writeJournalSnapshotFixture(t, journalDir, "system.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 3, bootID: currentBoot, realtime: uint64(start.Add(2 * time.Second).UnixMicro()), monotonic: 200, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=ready"}}, + {seqnum: 4, bootID: currentBoot, realtime: uint64(start.Add(3 * time.Second).UnixMicro()), monotonic: 300, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=healthy"}}, + }), machineID, repeatedJournalID(0x32), sequenceID) + + entries, err := client.queryJournalEntries(context.Background(), builtins.JournalQuery{ + Units: []string{"api.service"}, + CurrentBoot: true, + MaxEntries: 10, + }) + require.NoError(t, err) + require.Len(t, entries, 3) + assert.Equal(t, []string{"healthy", "ready", "started"}, journalMessages(entries)) +} + +func TestQueryJournalEntriesHandlesDuplicateSequenceNumbers(t *testing.T) { + bootID := repeatedJournalID(0x22) + sequenceID := repeatedJournalID(0x44) + start := time.Unix(1_700_000_000, 0) + + t.Run("different locations remain visible", func(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + writeJournalSnapshotFixture(t, journalDir, "one.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 5, bootID: bootID, realtime: uint64(start.UnixMicro()), monotonic: 100, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=first"}}, + }), machineID, repeatedJournalID(0x31), sequenceID) + writeJournalSnapshotFixture(t, journalDir, "two.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 5, bootID: bootID, realtime: uint64(start.Add(time.Second).UnixMicro()), monotonic: 200, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=second"}}, + }), machineID, repeatedJournalID(0x32), sequenceID) + + entries, err := client.queryJournalEntries(context.Background(), builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 10}) + require.NoError(t, err) + assert.Equal(t, []string{"second", "first"}, journalMessages(entries)) + }) + + t.Run("same location is deduplicated", func(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + spec := []journalQueryFixtureEntry{ + {seqnum: 5, bootID: bootID, realtime: uint64(start.UnixMicro()), monotonic: 100, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=duplicate"}}, + } + writeJournalSnapshotFixture(t, journalDir, "one.journal", buildJournalQueryFixture(t, spec), machineID, repeatedJournalID(0x31), sequenceID) + writeJournalSnapshotFixture(t, journalDir, "two.journal", buildJournalQueryFixture(t, spec), machineID, repeatedJournalID(0x32), sequenceID) + + entries, err := client.queryJournalEntries(context.Background(), builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 10}) + require.NoError(t, err) + assert.Equal(t, []string{"duplicate"}, journalMessages(entries)) + }) +} + +func TestNewestJournalEntrySupportsLegacyHeader(t *testing.T) { + bootID := repeatedJournalID(0x22) + contents := buildLegacyJournalTailFixture(bootID) + view, err := newJournalFileView("legacy.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + require.False(t, view.header.hasTailEntryOffset) + + entry, found, err := newestJournalEntry(view) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(3), entry.seqnum) + assert.Equal(t, bootID, entry.bootID) +} + +func journalMessages(entries []journalQueryEntry) []string { + messages := make([]string, len(entries)) + for index, entry := range entries { + messages[index] = entry.selected.Message + } + return messages +} + +func buildLegacyJournalTailFixture(bootID journalID) []byte { + const entryCount = 3 + payload := []byte("MESSAGE=x") + hash := journalJenkinsHash64(payload) + dataOffset := journalHeaderMinSize + dataSize := journalDataRegularHeaderSize + len(payload) + arrayOffset := alignJournalTestSize(dataOffset + dataSize) + arraySize := journalEntryArrayHeaderSize + entryCount*journalEntryArrayRegularItemSize + nextOffset := alignJournalTestSize(arrayOffset + arraySize) + entryOffsets := make([]int, entryCount) + for index := range entryOffsets { + entryOffsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalEntryHeaderSize + journalEntryRegularItemSize) + } + + contents := testJournalContentsWithHeader(nextOffset, journalHeaderMinSize, 0, 0) + fileID := repeatedJournalID(0x31) + sequenceID := repeatedJournalID(0x44) + copy(contents[24:40], fileID[:]) + copy(contents[56:72], bootID[:]) + copy(contents[72:88], sequenceID[:]) + binary.LittleEndian.PutUint64(contents[136:144], uint64(entryOffsets[len(entryOffsets)-1])) + binary.LittleEndian.PutUint64(contents[144:152], 2+entryCount) + binary.LittleEndian.PutUint64(contents[152:160], entryCount) + binary.LittleEndian.PutUint64(contents[160:168], entryCount) + binary.LittleEndian.PutUint64(contents[168:176], 1) + binary.LittleEndian.PutUint64(contents[176:184], uint64(arrayOffset)) + binary.LittleEndian.PutUint64(contents[184:192], 1_700_000_000_000_001) + binary.LittleEndian.PutUint64(contents[192:200], 1_700_000_000_000_003) + + data := contents[dataOffset:] + data[0] = journalObjectData + binary.LittleEndian.PutUint64(data[8:16], uint64(dataSize)) + binary.LittleEndian.PutUint64(data[16:24], hash) + copy(data[journalDataRegularHeaderSize:], payload) + + array := contents[arrayOffset:] + array[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(array[8:16], uint64(arraySize)) + for index, offset := range entryOffsets { + position := journalEntryArrayHeaderSize + index*journalEntryArrayRegularItemSize + binary.LittleEndian.PutUint64(array[position:position+8], uint64(offset)) + } + + for index, offset := range entryOffsets { + entry := contents[offset:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], journalEntryHeaderSize+journalEntryRegularItemSize) + binary.LittleEndian.PutUint64(entry[16:24], uint64(index+1)) + binary.LittleEndian.PutUint64(entry[24:32], uint64(1_700_000_000_000_001+index)) + binary.LittleEndian.PutUint64(entry[32:40], uint64(100+index)) + copy(entry[40:56], bootID[:]) + binary.LittleEndian.PutUint64(entry[56:64], hash) + binary.LittleEndian.PutUint64(entry[64:72], uint64(dataOffset)) + binary.LittleEndian.PutUint64(entry[72:80], hash) + } + return contents +} diff --git a/internal/systemd/journal_query.go b/internal/systemd/journal_query.go new file mode 100644 index 00000000..0e00b789 --- /dev/null +++ b/internal/systemd/journal_query.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "strings" + + "github.com/DataDog/rshell/builtins" +) + +const maxJournalFieldSize = 64 * 1024 + +func validateJournalQuery(query builtins.JournalQuery) error { + if query.MaxEntries < 0 || query.MaxEntries > builtins.MaxJournalQueryEntries { + return fmt.Errorf("journal query entry limit must be between 0 and %d", builtins.MaxJournalQueryEntries) + } + if query.Kernel && len(query.Units) > 0 { + return fmt.Errorf("journal query cannot combine kernel and unit scopes") + } + if !query.Kernel && len(query.Units) == 0 { + return fmt.Errorf("journal query requires a kernel or unit scope") + } + if len(query.Units) > builtins.MaxJournalQueryUnits { + return fmt.Errorf("journal query has too many units (maximum %d)", builtins.MaxJournalQueryUnits) + } + for _, unit := range query.Units { + if unit == "" || len(unit) > 256 || strings.IndexByte(unit, 0) >= 0 { + return fmt.Errorf("journal query contains an invalid unit name") + } + } + return nil +} diff --git a/internal/systemd/journal_query_file.go b/internal/systemd/journal_query_file.go new file mode 100644 index 00000000..e1ede5c3 --- /dev/null +++ b/internal/systemd/journal_query_file.go @@ -0,0 +1,369 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "fmt" + "math" + "strings" + "time" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalCandidatesScanned = 100_000 + maxJournalFieldNameSize = 64 +) + +type journalQueryEntry struct { + fileID journalID + seqnumID journalID + bootID journalID + seqnum uint64 + realtime uint64 + monotonic uint64 + xorHash uint64 + offset uint64 + selected builtins.JournalEntry +} + +type journalDataCandidateSource struct { + iterator *journalEntryOffsetIterator + index journalDataObject + required []journalDataObject + current *journalEntryObject + done bool +} + +func (s *journalDataCandidateSource) load(ctx context.Context, scanned *int) error { + if s.current != nil || s.done { + return nil + } + for { + if err := ctx.Err(); err != nil { + return err + } + offset, found, err := s.iterator.previous() + if err != nil { + return err + } + if !found { + s.done = true + return nil + } + if *scanned >= maxJournalCandidatesScanned { + return journalLimit(s.iterator.file.name, offset, "query examines more than %d indexed candidates", maxJournalCandidatesScanned) + } + (*scanned)++ + + entry, err := s.iterator.file.entryObjectAt(offset) + if err != nil { + return err + } + indexed, err := journalEntryContainsData(s.iterator.file, entry, s.index) + if err != nil { + return err + } + if !indexed { + return journalCorrupt(s.iterator.file.name, offset, "DATA entry index points to an ENTRY that does not reference the DATA object") + } + matches := true + for _, required := range s.required { + contains, err := journalEntryContainsData(s.iterator.file, entry, required) + if err != nil { + return err + } + if !contains { + matches = false + break + } + } + if matches { + s.current = &entry + return nil + } + } +} + +func journalEntryContainsData(file *journalFileView, entry journalEntryObject, data journalDataObject) (bool, error) { + for index, item := range entry.items { + if item.dataOffset != data.offset { + continue + } + if !file.header.compact() && item.hash != data.hash { + return false, journalCorrupt(file.name, entry.offset, "ENTRY item %d hash does not match its DATA object", index) + } + return true, nil + } + return false, nil +} + +type journalFileQueryIterator struct { + file *journalFileView + sources []*journalDataCandidateSource + bootID journalID + filterBoot bool + sinceUsec uint64 + scanned int +} + +func newJournalFileQueryIterator(file *journalFileView, query builtins.JournalQuery, currentBoot *journalID) (*journalFileQueryIterator, error) { + if err := validateJournalQuery(query); err != nil { + return nil, err + } + iterator := &journalFileQueryIterator{file: file} + if query.CurrentBoot { + if currentBoot == nil { + return nil, fmt.Errorf("current journal boot is not available") + } + iterator.bootID = *currentBoot + iterator.filterBoot = true + } + if !query.Since.IsZero() && query.Since.UnixMicro() > 0 { + iterator.sinceUsec = uint64(query.Since.UnixMicro()) + } + + if query.Kernel { + data, found, err := file.findDataObject([]byte("_TRANSPORT=kernel")) + if err != nil { + return nil, err + } + if found { + if err := iterator.addSource(data); err != nil { + return nil, err + } + } + return iterator, nil + } + + pidOne, pidOneFound, err := file.findDataObject([]byte("_PID=1")) + if err != nil { + return nil, err + } + seenUnits := make(map[string]struct{}, len(query.Units)) + for _, unit := range query.Units { + if _, exists := seenUnits[unit]; exists { + continue + } + seenUnits[unit] = struct{}{} + direct, found, err := file.findDataObject([]byte("_SYSTEMD_UNIT=" + unit)) + if err != nil { + return nil, err + } + if found { + if err := iterator.addSource(direct); err != nil { + return nil, err + } + } + + manager, found, err := file.findDataObject([]byte("UNIT=" + unit)) + if err != nil { + return nil, err + } + if found && pidOneFound { + if err := iterator.addSource(manager, pidOne); err != nil { + return nil, err + } + } + } + return iterator, nil +} + +func (i *journalFileQueryIterator) addSource(index journalDataObject, required ...journalDataObject) error { + offsets, err := i.file.entryOffsetsForData(index) + if err != nil { + return err + } + i.sources = append(i.sources, &journalDataCandidateSource{iterator: offsets, index: index, required: required}) + return nil +} + +func (i *journalFileQueryIterator) previous(ctx context.Context) (journalQueryEntry, bool, error) { + for { + entry, found, err := i.previousCandidate(ctx) + if err != nil || !found { + return journalQueryEntry{}, found, err + } + if i.filterBoot && entry.bootID != i.bootID { + continue + } + if i.sinceUsec != 0 && entry.realtime < i.sinceUsec { + continue + } + if entry.seqnum == 0 { + return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY sequence number is zero") + } + if entry.bootID.zero() { + return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY boot ID is zero") + } + if entry.realtime == 0 || entry.realtime > math.MaxInt64 { + return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY realtime timestamp %d is invalid", entry.realtime) + } + + selected, err := i.file.materializeJournalEntry(ctx, entry) + if err != nil { + return journalQueryEntry{}, false, err + } + return journalQueryEntry{ + fileID: i.file.header.fileID, + seqnumID: i.file.header.seqnumID, + bootID: entry.bootID, + seqnum: entry.seqnum, + realtime: entry.realtime, + monotonic: entry.monotonic, + xorHash: entry.xorHash, + offset: entry.offset, + selected: selected, + }, true, nil + } +} + +func (i *journalFileQueryIterator) previousCandidate(ctx context.Context) (journalEntryObject, bool, error) { + var newest *journalEntryObject + for _, source := range i.sources { + if err := source.load(ctx, &i.scanned); err != nil { + return journalEntryObject{}, false, err + } + if source.current != nil && (newest == nil || source.current.offset > newest.offset) { + newest = source.current + } + } + if newest == nil { + return journalEntryObject{}, false, nil + } + + selected := *newest + for _, source := range i.sources { + if source.current != nil && source.current.offset == selected.offset { + source.current = nil + } + } + return selected, true, nil +} + +func (f *journalFileView) materializeJournalEntry(ctx context.Context, entry journalEntryObject) (builtins.JournalEntry, error) { + selected := builtins.JournalEntry{ + Timestamp: time.UnixMicro(int64(entry.realtime)), + } + var syslogIdentifier, command string + var haveHostname, haveSyslogIdentifier, haveCommand, havePID, haveMessage bool + seen := make(map[uint64]struct{}, len(entry.items)) + + for index, item := range entry.items { + if err := ctx.Err(); err != nil { + return builtins.JournalEntry{}, err + } + if _, exists := seen[item.dataOffset]; exists { + return builtins.JournalEntry{}, journalCorrupt(f.name, entry.offset, "ENTRY contains duplicate DATA offset %d", item.dataOffset) + } + seen[item.dataOffset] = struct{}{} + + data, err := f.dataObjectAt(item.dataOffset) + if err != nil { + return builtins.JournalEntry{}, err + } + if !f.header.compact() && item.hash != data.hash { + return builtins.JournalEntry{}, journalCorrupt(f.name, entry.offset, "ENTRY item %d hash does not match its DATA object", index) + } + field, value, selectedField, err := f.selectedDataField(data) + if err != nil { + return builtins.JournalEntry{}, err + } + if !selectedField { + continue + } + switch field { + case "_HOSTNAME": + if !haveHostname { + selected.Hostname = value + haveHostname = true + } + case "SYSLOG_IDENTIFIER": + if !haveSyslogIdentifier { + syslogIdentifier = value + haveSyslogIdentifier = true + } + case "_COMM": + if !haveCommand { + command = value + haveCommand = true + } + case "_PID": + if !havePID { + selected.PID = value + havePID = true + } + case "MESSAGE": + if !haveMessage { + selected.Message = value + haveMessage = true + } + } + } + selected.Identifier = syslogIdentifier + if selected.Identifier == "" { + selected.Identifier = command + } + return selected, nil +} + +func (f *journalFileView) selectedDataField(data journalDataObject) (string, string, bool, error) { + prefix, _, err := f.readDataPayload(data, maxJournalFieldNameSize+1) + if err != nil { + return "", "", false, err + } + separator := bytes.IndexByte(prefix, '=') + if separator <= 0 || separator > maxJournalFieldNameSize { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload has an invalid field name") + } + fieldBytes := prefix[:separator] + if !validJournalFieldName(fieldBytes) { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload has an invalid field name") + } + field := string(fieldBytes) + if !selectedJournalField(field) { + return field, "", false, nil + } + + limit := separator + 1 + maxJournalFieldSize + payload, truncated, err := f.readDataPayload(data, limit) + if err != nil { + return "", "", false, err + } + fieldPrefix := field + "=" + if !strings.HasPrefix(string(payload), fieldPrefix) { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload changed while being read") + } + if !truncated && f.dataHash(payload) != data.hash { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload hash does not match its object") + } + return field, string(payload[len(fieldPrefix):]), true, nil +} + +func validJournalFieldName(field []byte) bool { + if len(field) == 0 || len(field) > maxJournalFieldNameSize || field[0] >= '0' && field[0] <= '9' { + return false + } + for _, character := range field { + if character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '_' { + continue + } + return false + } + return true +} + +func selectedJournalField(field string) bool { + switch field { + case "_HOSTNAME", "SYSLOG_IDENTIFIER", "_COMM", "_PID", "MESSAGE": + return true + default: + return false + } +} diff --git a/internal/systemd/journal_query_file_test.go b/internal/systemd/journal_query_file_test.go new file mode 100644 index 00000000..e2259fd1 --- /dev/null +++ b/internal/systemd/journal_query_file_test.go @@ -0,0 +1,396 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +func TestJournalFileQuerySelectsDirectAndManagerUnitEntries(t *testing.T) { + oldBoot := repeatedJournalID(0x11) + currentBoot := repeatedJournalID(0x22) + start := time.Unix(1_700_000_000, 0) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + { + bootID: oldBoot, + realtime: uint64(start.UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=api.service", "_HOSTNAME=node", "_COMM=api", "_PID=10", "MESSAGE=old boot"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(time.Second).UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=worker.service", "MESSAGE=worker"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(2 * time.Second).UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=api.service", "_HOSTNAME=node", "SYSLOG_IDENTIFIER=api", "_PID=20", "MESSAGE=ready"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(3 * time.Second).UnixMicro()), + fields: []string{"UNIT=api.service", "_PID=1", "SYSLOG_IDENTIFIER=systemd", "MESSAGE=stopped"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(4 * time.Second).UnixMicro()), + fields: []string{"UNIT=api.service", "_PID=22", "MESSAGE=untrusted unit field"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(5 * time.Second).UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=api.service", "UNIT=api.service", "_PID=1", "_COMM=systemd", "MESSAGE=deduplicated"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(6 * time.Second).UnixMicro()), + fields: []string{"_TRANSPORT=kernel", "_HOSTNAME=node", "_COMM=kernel", "MESSAGE=kernel message"}, + }, + }) + view, err := newJournalFileView("query.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"api.service"}, + CurrentBoot: true, + Since: start.Add(2 * time.Second), + MaxEntries: 100, + }, ¤tBoot) + require.NoError(t, err) + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 3) + assert.Equal(t, []string{"deduplicated", "stopped", "ready"}, []string{ + entries[0].selected.Message, + entries[1].selected.Message, + entries[2].selected.Message, + }) + assert.Equal(t, "systemd", entries[0].selected.Identifier) + assert.Equal(t, "systemd", entries[1].selected.Identifier) + assert.Equal(t, "api", entries[2].selected.Identifier) + assert.Equal(t, "node", entries[2].selected.Hostname) + assert.Equal(t, "20", entries[2].selected.PID) + assert.Equal(t, start.Add(2*time.Second), entries[2].selected.Timestamp) +} + +func TestJournalFileQuerySelectsKernelEntries(t *testing.T) { + bootID := repeatedJournalID(0x33) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=service"}}, + {bootID: bootID, realtime: 1_700_000_000_000_001, fields: []string{"_TRANSPORT=kernel", "_COMM=kernel", "MESSAGE=kernel"}}, + }) + view, err := newJournalFileView("kernel.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Kernel: true, + CurrentBoot: true, + MaxEntries: 10, + }, &bootID) + require.NoError(t, err) + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 1) + assert.Equal(t, "kernel", entries[0].selected.Message) + assert.Equal(t, "kernel", entries[0].selected.Identifier) +} + +func TestJournalFileQuerySupportsCompactKeyedLayout(t *testing.T) { + bootID := repeatedJournalID(0x77) + contents := buildJournalQueryFixtureWithLayout(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=first"}}, + {bootID: bootID, realtime: 1_700_000_000_000_001, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=second"}}, + }, true) + view, err := newJournalFileView("compact.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 10}, nil) + require.NoError(t, err) + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 2) + assert.Equal(t, "second", entries[0].selected.Message) + assert.Equal(t, "first", entries[1].selected.Message) +} + +func TestJournalFileQueryHonorsCancellation(t *testing.T) { + bootID := repeatedJournalID(0x44) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=service"}}, + }) + view, err := newJournalFileView("cancel.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 1}, nil) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, _, err = iterator.previous(ctx) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestJournalFileQueryRejectsCandidateMissingIndexedData(t *testing.T) { + bootID := repeatedJournalID(0x66) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=service"}}, + }) + initial, err := newJournalFileView("broken-index.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + selector, found, err := initial.findDataObject([]byte("_SYSTEMD_UNIT=api.service")) + require.NoError(t, err) + require.True(t, found) + message, found, err := initial.findDataObject([]byte("MESSAGE=service")) + require.NoError(t, err) + require.True(t, found) + entry, err := initial.entryObjectAt(selector.entryOffset) + require.NoError(t, err) + for index, item := range entry.items { + if item.dataOffset != selector.offset { + continue + } + position := int(entry.offset) + journalEntryHeaderSize + index*journalEntryRegularItemSize + binary.LittleEndian.PutUint64(contents[position:position+8], message.offset) + binary.LittleEndian.PutUint64(contents[position+8:position+16], message.hash) + } + + view, err := newJournalFileView("broken-index.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 1}, nil) + require.NoError(t, err) + _, _, err = iterator.previous(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "does not reference the DATA object") +} + +func collectJournalQueryEntries(t *testing.T, iterator *journalFileQueryIterator) []journalQueryEntry { + t.Helper() + var entries []journalQueryEntry + for { + entry, found, err := iterator.previous(context.Background()) + require.NoError(t, err) + if !found { + return entries + } + entries = append(entries, entry) + } +} + +type journalQueryFixtureEntry struct { + seqnum uint64 + bootID journalID + realtime uint64 + monotonic uint64 + fields []string +} + +type journalQueryFixtureData struct { + payload []byte + hash uint64 + offset int + references []int + arrayOffset int +} + +func buildJournalQueryFixture(t testing.TB, entries []journalQueryFixtureEntry) []byte { + t.Helper() + return buildJournalQueryFixtureWithLayout(t, entries, false) +} + +func buildJournalQueryFixtureWithLayout(t testing.TB, entries []journalQueryFixtureEntry, compact bool) []byte { + t.Helper() + require.NotEmpty(t, entries) + headerFlags := uint32(0) + dataHeaderSize := journalDataRegularHeaderSize + entryItemSize := journalEntryRegularItemSize + arrayItemSize := journalEntryArrayRegularItemSize + var fileID journalID + for index := range fileID { + fileID[index] = byte(index) + } + if compact { + headerFlags = journalHeaderIncompatibleCompact | journalHeaderIncompatibleKeyedHash + dataHeaderSize = journalDataCompactHeaderSize + entryItemSize = journalEntryCompactItemSize + arrayItemSize = journalEntryCompactItemSize + } + + dataByValue := make(map[string]*journalQueryFixtureData) + var dataObjects []*journalQueryFixtureData + for entryIndex, entry := range entries { + require.NotEmpty(t, entry.fields) + for _, field := range entry.fields { + data := dataByValue[field] + if data == nil { + payload := []byte(field) + hash := journalJenkinsHash64(payload) + if compact { + hash = journalSipHash24(payload, fileID) + } + data = &journalQueryFixtureData{payload: payload, hash: hash} + dataByValue[field] = data + dataObjects = append(dataObjects, data) + } + data.references = append(data.references, entryIndex) + } + } + + const bucketCount = 64 + tableOffset := journalHeaderCurrentSize + tableObjectSize := journalObjectHeaderSize + bucketCount*journalHashItemSize + nextOffset := alignJournalTestSize(tableOffset + tableObjectSize) + for _, data := range dataObjects { + data.offset = nextOffset + nextOffset = alignJournalTestSize(nextOffset + dataHeaderSize + len(data.payload)) + } + entryOffsets := make([]int, len(entries)) + for index, entry := range entries { + entryOffsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalEntryHeaderSize + len(entry.fields)*entryItemSize) + } + arrayCount := 0 + for _, data := range dataObjects { + if len(data.references) <= 1 { + continue + } + data.arrayOffset = nextOffset + capacity := len(data.references) - 1 + if capacity < 4 { + capacity = 4 + } + nextOffset = alignJournalTestSize(nextOffset + journalEntryArrayHeaderSize + capacity*arrayItemSize) + arrayCount++ + } + + contents := testJournalContents(nextOffset, 0, headerFlags) + copy(contents[24:40], fileID[:]) + copy(contents[56:72], entries[len(entries)-1].bootID[:]) + copy(contents[72:88], bytes.Repeat([]byte{0x55}, 16)) + binary.LittleEndian.PutUint64(contents[104:112], uint64(tableOffset+journalObjectHeaderSize)) + binary.LittleEndian.PutUint64(contents[112:120], bucketCount*journalHashItemSize) + binary.LittleEndian.PutUint64(contents[136:144], uint64(lastJournalFixtureObjectOffset(dataObjects, entryOffsets))) + binary.LittleEndian.PutUint64(contents[144:152], uint64(1+len(dataObjects)+len(entries)+arrayCount)) + binary.LittleEndian.PutUint64(contents[152:160], uint64(len(entries))) + binary.LittleEndian.PutUint64(contents[160:168], journalQueryFixtureSeqnum(entries[len(entries)-1], len(entries)-1)) + binary.LittleEndian.PutUint64(contents[168:176], journalQueryFixtureSeqnum(entries[0], 0)) + binary.LittleEndian.PutUint64(contents[184:192], entries[0].realtime) + binary.LittleEndian.PutUint64(contents[192:200], entries[len(entries)-1].realtime) + binary.LittleEndian.PutUint64(contents[264:272], uint64(entryOffsets[len(entryOffsets)-1])) + + table := contents[tableOffset:] + table[0] = journalObjectDataHashTable + binary.LittleEndian.PutUint64(table[8:16], uint64(tableObjectSize)) + buckets := make([][]*journalQueryFixtureData, bucketCount) + for _, data := range dataObjects { + bucket := data.hash % bucketCount + buckets[bucket] = append(buckets[bucket], data) + } + for bucket, chain := range buckets { + if len(chain) == 0 { + continue + } + position := journalObjectHeaderSize + bucket*journalHashItemSize + binary.LittleEndian.PutUint64(table[position:position+8], uint64(chain[0].offset)) + binary.LittleEndian.PutUint64(table[position+8:position+16], uint64(chain[len(chain)-1].offset)) + for index := 0; index+1 < len(chain); index++ { + next := chain[index+1].offset + binary.LittleEndian.PutUint64(contents[chain[index].offset+24:chain[index].offset+32], uint64(next)) + } + } + + for _, data := range dataObjects { + object := contents[data.offset:] + object[0] = journalObjectData + binary.LittleEndian.PutUint64(object[8:16], uint64(dataHeaderSize+len(data.payload))) + binary.LittleEndian.PutUint64(object[16:24], data.hash) + binary.LittleEndian.PutUint64(object[40:48], uint64(entryOffsets[data.references[0]])) + binary.LittleEndian.PutUint64(object[48:56], uint64(data.arrayOffset)) + binary.LittleEndian.PutUint64(object[56:64], uint64(len(data.references))) + copy(object[dataHeaderSize:], data.payload) + + if data.arrayOffset != 0 { + capacity := len(data.references) - 1 + if capacity < 4 { + capacity = 4 + } + array := contents[data.arrayOffset:] + array[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(array[8:16], uint64(journalEntryArrayHeaderSize+capacity*arrayItemSize)) + if compact { + binary.LittleEndian.PutUint32(object[64:68], uint32(data.arrayOffset)) + binary.LittleEndian.PutUint32(object[68:72], uint32(len(data.references)-1)) + } + for index, entryIndex := range data.references[1:] { + position := journalEntryArrayHeaderSize + index*arrayItemSize + if compact { + binary.LittleEndian.PutUint32(array[position:position+4], uint32(entryOffsets[entryIndex])) + } else { + binary.LittleEndian.PutUint64(array[position:position+8], uint64(entryOffsets[entryIndex])) + } + } + } + } + + for index, entrySpec := range entries { + entry := contents[entryOffsets[index]:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], uint64(journalEntryHeaderSize+len(entrySpec.fields)*entryItemSize)) + seqnum := journalQueryFixtureSeqnum(entrySpec, index) + monotonic := entrySpec.monotonic + if monotonic == 0 { + monotonic = seqnum * 100 + } + binary.LittleEndian.PutUint64(entry[16:24], seqnum) + binary.LittleEndian.PutUint64(entry[24:32], entrySpec.realtime) + binary.LittleEndian.PutUint64(entry[32:40], monotonic) + copy(entry[40:56], entrySpec.bootID[:]) + var xorHash uint64 + for itemIndex, field := range entrySpec.fields { + data := dataByValue[field] + position := journalEntryHeaderSize + itemIndex*entryItemSize + if compact { + binary.LittleEndian.PutUint32(entry[position:position+4], uint32(data.offset)) + } else { + binary.LittleEndian.PutUint64(entry[position:position+8], uint64(data.offset)) + binary.LittleEndian.PutUint64(entry[position+8:position+16], data.hash) + } + xorHash ^= journalJenkinsHash64(data.payload) + } + binary.LittleEndian.PutUint64(entry[56:64], xorHash) + } + return contents +} + +func journalQueryFixtureSeqnum(entry journalQueryFixtureEntry, index int) uint64 { + if entry.seqnum != 0 { + return entry.seqnum + } + return uint64(index + 1) +} + +func lastJournalFixtureObjectOffset(data []*journalQueryFixtureData, entryOffsets []int) int { + last := entryOffsets[len(entryOffsets)-1] + for _, object := range data { + if object.arrayOffset > last { + last = object.arrayOffset + } + } + return last +} + +func repeatedJournalID(value byte) journalID { + var id journalID + for index := range id { + id[index] = value + } + return id +} diff --git a/internal/systemd/journal_query_test.go b/internal/systemd/journal_query_test.go new file mode 100644 index 00000000..dbcb80d6 --- /dev/null +++ b/internal/systemd/journal_query_test.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +func TestValidateJournalQueryAcceptsBoundedScopes(t *testing.T) { + require.NoError(t, validateJournalQuery(builtins.JournalQuery{Kernel: true, MaxEntries: builtins.MaxJournalQueryEntries})) + require.NoError(t, validateJournalQuery(builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 1})) +} + +func TestValidateJournalQueryRejectsInvalidScopes(t *testing.T) { + tooManyUnits := make([]string, builtins.MaxJournalQueryUnits+1) + for index := range tooManyUnits { + tooManyUnits[index] = "api.service" + } + tests := []struct { + name string + query builtins.JournalQuery + match string + }{ + {name: "negative limit", query: builtins.JournalQuery{Kernel: true, MaxEntries: -1}, match: "entry limit"}, + {name: "large limit", query: builtins.JournalQuery{Kernel: true, MaxEntries: builtins.MaxJournalQueryEntries + 1}, match: "entry limit"}, + {name: "missing scope", query: builtins.JournalQuery{MaxEntries: 1}, match: "requires a kernel or unit scope"}, + {name: "mixed scopes", query: builtins.JournalQuery{Kernel: true, Units: []string{"api.service"}, MaxEntries: 1}, match: "cannot combine"}, + {name: "too many units", query: builtins.JournalQuery{Units: tooManyUnits, MaxEntries: 1}, match: "too many units"}, + {name: "empty unit", query: builtins.JournalQuery{Units: []string{""}, MaxEntries: 1}, match: "invalid unit name"}, + {name: "long unit", query: builtins.JournalQuery{Units: []string{strings.Repeat("a", 257)}, MaxEntries: 1}, match: "invalid unit name"}, + {name: "nul unit", query: builtins.JournalQuery{Units: []string{"api\x00.service"}, MaxEntries: 1}, match: "invalid unit name"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateJournalQuery(test.query) + require.Error(t, err) + assert.Contains(t, err.Error(), test.match) + }) + } +} diff --git a/internal/systemd/journal_read.go b/internal/systemd/journal_read.go new file mode 100644 index 00000000..498e6a97 --- /dev/null +++ b/internal/systemd/journal_read.go @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// ReadJournal reads a bounded structured query from the configured journal +// files. Results are buffered until the file snapshot is stable, then yielded +// oldest first as required by builtins.JournalReader. +func (c *Client) ReadJournal(ctx context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { + if err := ctx.Err(); err != nil { + return err + } + if yield == nil { + return fmt.Errorf("journal entry yield function is nil") + } + entries, err := c.queryJournalEntries(ctx, query) + if err != nil { + return err + } + for index := len(entries) - 1; index >= 0; index-- { + if err := ctx.Err(); err != nil { + return err + } + if err := yield(entries[index].selected); err != nil { + return err + } + } + return nil +} diff --git a/internal/systemd/journal_rotate_linux.go b/internal/systemd/journal_rotate_linux.go new file mode 100644 index 00000000..30660f4c --- /dev/null +++ b/internal/systemd/journal_rotate_linux.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux + +package systemd + +import ( + "context" + "errors" + "fmt" + "time" +) + +const journalRotationTimeout = 30 * time.Second + +// RotateJournal asks journald to rotate active journals and waits for the +// daemon to report completion. +func (c *Client) RotateJournal(ctx context.Context) error { + if c.target.MachineIDPath == "" { + return fmt.Errorf("systemd target machine ID path is unavailable") + } + if _, err := readMachineID(c.target.MachineIDPath); err != nil { + return fmt.Errorf("validate systemd target machine ID: %w", err) + } + if c.target.JournalControlSocket == "" { + return fmt.Errorf("systemd target journal control socket is unavailable") + } + + rotationCtx, cancel := context.WithTimeout(ctx, journalRotationTimeout) + defer cancel() + if err := rotateJournalControl(rotationCtx, c.target.JournalControlSocket); err != nil { + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + return fmt.Errorf("journal rotation timed out after %s", journalRotationTimeout) + } + return fmt.Errorf("rotate journal: %w", err) + } + return nil +} diff --git a/internal/systemd/journal_rotate_unsupported.go b/internal/systemd/journal_rotate_unsupported.go new file mode 100644 index 00000000..fedca45c --- /dev/null +++ b/internal/systemd/journal_rotate_unsupported.go @@ -0,0 +1,20 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !linux + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// RotateJournal requires a Linux systemd-journald control endpoint. +func (c *Client) RotateJournal(context.Context) error { + return fmt.Errorf("%w: journal rotation requires Linux", builtins.ErrSystemdUnsupported) +} diff --git a/internal/systemd/journal_snapshot.go b/internal/systemd/journal_snapshot.go new file mode 100644 index 00000000..13eedf55 --- /dev/null +++ b/internal/systemd/journal_snapshot.go @@ -0,0 +1,207 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" +) + +const maxJournalQueryFiles = 128 + +var errJournalChanged = errors.New("systemd journal changed while being read") + +type journalSnapshotFile struct { + path string + file *os.File + info fs.FileInfo + view *journalFileView +} + +type journalSnapshot struct { + machineID journalID + machineIDText string + paths []string + files []*journalSnapshotFile +} + +func (c *Client) openJournalSnapshot() (*journalSnapshot, error) { + machineIDText, paths, err := c.journalFiles() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal files changed during discovery") + } + return nil, err + } + if len(paths) == 0 { + return nil, fmt.Errorf("no journal files found for machine %s", machineIDText) + } + if len(paths) > maxJournalQueryFiles { + return nil, fmt.Errorf("%w: journal query opens %d files; maximum is %d", errJournalLimit, len(paths), maxJournalQueryFiles) + } + machineID, err := parseJournalID(machineIDText) + if err != nil { + return nil, err + } + + snapshot := &journalSnapshot{ + machineID: machineID, + machineIDText: machineIDText, + paths: append([]string(nil), paths...), + files: make([]*journalSnapshotFile, 0, len(paths)), + } + fileIDs := make(map[journalID]string, len(paths)) + for _, path := range paths { + opened, err := openJournalSnapshotFile(path) + if err != nil { + snapshot.close() + return nil, err + } + if opened.view.header.machineID != machineID { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 40, "journal machine ID %s does not match configured machine %s", opened.view.header.machineID, machineID) + } + if opened.view.header.fileID.zero() { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 24, "journal file ID is zero") + } + if opened.view.header.seqnumID.zero() { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 72, "journal sequence ID is zero") + } + if previous, exists := fileIDs[opened.view.header.fileID]; exists { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 24, "journal file ID duplicates %q", previous) + } + fileIDs[opened.view.header.fileID] = path + snapshot.files = append(snapshot.files, opened) + } + if err := snapshot.stable(c); err != nil { + snapshot.close() + return nil, err + } + return snapshot, nil +} + +func openJournalSnapshotFile(path string) (*journalSnapshotFile, error) { + before, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal file %q disappeared before open", path) + } + return nil, fmt.Errorf("inspect journal file %q before open: %w", path, err) + } + if before.Mode()&fs.ModeSymlink != 0 || !before.Mode().IsRegular() { + return nil, journalChanged("journal file %q is no longer a regular non-symlink file", path) + } + + file, err := os.Open(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal file %q disappeared during open", path) + } + return nil, fmt.Errorf("open journal file %q: %w", path, err) + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, fmt.Errorf("inspect open journal file %q: %w", path, err) + } + after, err := os.Lstat(path) + if err != nil { + file.Close() + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal file %q disappeared during open", path) + } + return nil, fmt.Errorf("inspect journal file %q after open: %w", path, err) + } + if !info.Mode().IsRegular() || after.Mode()&fs.ModeSymlink != 0 || !after.Mode().IsRegular() || !os.SameFile(before, info) || !os.SameFile(after, info) { + file.Close() + return nil, journalChanged("journal file %q was replaced during open", path) + } + if info.Size() < 0 { + file.Close() + return nil, journalCorrupt(path, 0, "journal file has a negative size") + } + view, err := newJournalFileView(path, file, uint64(info.Size())) + if err != nil { + file.Close() + return nil, err + } + return &journalSnapshotFile{path: path, file: file, info: info, view: view}, nil +} + +func (s *journalSnapshot) stable(client *Client) error { + machineID, paths, err := client.journalFiles() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return journalChanged("journal files changed during snapshot verification") + } + return err + } + if machineID != s.machineIDText || !sameJournalPaths(paths, s.paths) { + return journalChanged("journal file set changed during snapshot") + } + for _, opened := range s.files { + current, err := os.Lstat(opened.path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return journalChanged("journal file %q disappeared during snapshot", opened.path) + } + return fmt.Errorf("verify journal file %q: %w", opened.path, err) + } + if current.Mode()&fs.ModeSymlink != 0 || !current.Mode().IsRegular() || !os.SameFile(current, opened.info) { + return journalChanged("journal file %q was replaced during snapshot", opened.path) + } + if current.Size() < opened.info.Size() { + return journalChanged("journal file %q shrank during snapshot", opened.path) + } + } + return nil +} + +func (s *journalSnapshot) close() error { + var result error + for _, opened := range s.files { + if err := opened.file.Close(); err != nil { + result = errors.Join(result, fmt.Errorf("close journal file %q: %w", opened.path, err)) + } + } + return result +} + +func sameJournalPaths(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +func parseJournalID(value string) (journalID, error) { + var id journalID + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != len(id) { + return journalID{}, fmt.Errorf("invalid 128-bit journal ID %q", value) + } + copy(id[:], decoded) + return id, nil +} + +func journalChanged(format string, arguments ...any) error { + return fmt.Errorf("%w: %s", errJournalChanged, fmt.Sprintf(format, arguments...)) +} diff --git a/internal/systemd/journal_snapshot_test.go b/internal/systemd/journal_snapshot_test.go new file mode 100644 index 00000000..b0b31d66 --- /dev/null +++ b/internal/systemd/journal_snapshot_test.go @@ -0,0 +1,78 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalSnapshotDetectsFileReplacement(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not permit replacing this open fixture consistently") + } + + client, journalDir, machineID := newJournalSnapshotTestClient(t) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: repeatedJournalID(0x22), realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=ready"}}, + }) + path := writeJournalSnapshotFixture(t, journalDir, "system.journal", contents, machineID, repeatedJournalID(0x31), repeatedJournalID(0x44)) + replacement, err := os.ReadFile(path) + require.NoError(t, err) + + snapshot, err := client.openJournalSnapshot() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, snapshot.close()) }) + require.NoError(t, os.Rename(path, path+".replaced")) + require.NoError(t, os.WriteFile(path, replacement, 0o600)) + + err = snapshot.stable(client) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalChanged) + assert.Contains(t, err.Error(), "replaced during snapshot") +} + +func TestOpenJournalSnapshotRejectsTooManyFilesBeforeParsing(t *testing.T) { + client, journalDir, _ := newJournalSnapshotTestClient(t) + for index := 0; index <= maxJournalQueryFiles; index++ { + path := filepath.Join(journalDir, fmt.Sprintf("%03d.journal", index)) + require.NoError(t, os.WriteFile(path, []byte("not a journal"), 0o600)) + } + + _, err := client.openJournalSnapshot() + require.Error(t, err) + assert.ErrorIs(t, err, errJournalLimit) + assert.Contains(t, err.Error(), "maximum is 128") +} + +func newJournalSnapshotTestClient(t *testing.T) (*Client, string, journalID) { + t.Helper() + root := t.TempDir() + machineID := repeatedJournalID(0xa1) + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID.String()+"\n"), 0o600)) + baseDir := filepath.Join(root, "journal") + journalDir := filepath.Join(baseDir, machineID.String()) + require.NoError(t, os.MkdirAll(journalDir, 0o755)) + return NewClient(Target{JournalDirs: []string{baseDir}, MachineIDPath: machineIDPath}), journalDir, machineID +} + +func writeJournalSnapshotFixture(t *testing.T, journalDir, name string, contents []byte, machineID, fileID, sequenceID journalID) string { + t.Helper() + contents = append([]byte(nil), contents...) + copy(contents[24:40], fileID[:]) + copy(contents[40:56], machineID[:]) + copy(contents[72:88], sequenceID[:]) + path := filepath.Join(journalDir, name) + require.NoError(t, os.WriteFile(path, contents, 0o600)) + return path +} diff --git a/internal/systemd/journal_usage_unix.go b/internal/systemd/journal_usage_unix.go new file mode 100644 index 00000000..ffbdad76 --- /dev/null +++ b/internal/systemd/journal_usage_unix.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux || darwin + +package systemd + +import ( + "context" + "fmt" + "math" + "os" + + "github.com/DataDog/rshell/builtins" +) + +// JournalDiskUsage reports allocated blocks, matching journalctl's disk-usage +// semantics more closely than summing logical file lengths. +func (c *Client) JournalDiskUsage(ctx context.Context) (builtins.JournalUsage, error) { + _, files, err := c.journalFiles() + if err != nil { + return builtins.JournalUsage{}, err + } + usage := builtins.JournalUsage{Files: len(files)} + for _, path := range files { + if err := ctx.Err(); err != nil { + return builtins.JournalUsage{}, err + } + info, err := os.Lstat(path) + if err != nil { + return builtins.JournalUsage{}, fmt.Errorf("inspect journal file %q: %w", path, err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return builtins.JournalUsage{}, fmt.Errorf("journal file %q changed during usage scan", path) + } + stat, err := journalStat(info) + if err != nil { + return builtins.JournalUsage{}, fmt.Errorf("journal file %q: %w", path, err) + } + if usage.Bytes > math.MaxUint64-stat.allocated { + return builtins.JournalUsage{}, fmt.Errorf("journal allocation total overflow") + } + usage.Bytes += stat.allocated + } + return usage, nil +} diff --git a/internal/systemd/journal_usage_unix_test.go b/internal/systemd/journal_usage_unix_test.go new file mode 100644 index 00000000..ce71fc46 --- /dev/null +++ b/internal/systemd/journal_usage_unix_test.go @@ -0,0 +1,67 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux || darwin + +package systemd + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalDiskUsageCountsAllocatedJournalFiles(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + journalDir := filepath.Join(root, "journal") + machineDir := filepath.Join(journalDir, machineID) + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), make([]byte, 8192), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(machineDir, "ignored.txt"), make([]byte, 8192), 0o600)) + + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) + usage, err := client.JournalDiskUsage(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, usage.Files) + assert.Greater(t, usage.Bytes, uint64(0)) +} + +func TestJournalDiskUsageReturnsZeroForEmptyTarget(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + + client := NewClient(Target{JournalDirs: []string{filepath.Join(root, "missing")}, MachineIDPath: machineIDPath}) + usage, err := client.JournalDiskUsage(context.Background()) + require.NoError(t, err) + assert.Zero(t, usage.Files) + assert.Zero(t, usage.Bytes) +} + +func TestJournalDiskUsageHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + journalDir := filepath.Join(root, "journal") + machineDir := filepath.Join(journalDir, machineID) + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), []byte("data"), 0o600)) + + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) + _, err := client.JournalDiskUsage(ctx) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/internal/systemd/journal_usage_unsupported.go b/internal/systemd/journal_usage_unsupported.go new file mode 100644 index 00000000..e93ff990 --- /dev/null +++ b/internal/systemd/journal_usage_unsupported.go @@ -0,0 +1,20 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !linux && !darwin + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// JournalDiskUsage requires Unix allocation metadata. +func (c *Client) JournalDiskUsage(context.Context) (builtins.JournalUsage, error) { + return builtins.JournalUsage{}, fmt.Errorf("%w: journal disk usage requires Linux or macOS", builtins.ErrSystemdUnsupported) +} diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go new file mode 100644 index 00000000..88513822 --- /dev/null +++ b/internal/systemd/journal_vacuum_unix.go @@ -0,0 +1,274 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux || darwin + +package systemd + +import ( + "context" + "fmt" + "io" + "io/fs" + "math" + "os" + "sort" + "strings" + "time" + + "github.com/DataDog/rshell/builtins" +) + +type vacuumDirectory struct { + path string + root *os.Root +} + +type vacuumCandidate struct { + directory *vacuumDirectory + name string + modTime time.Time + size int64 + stat journalFileStat +} + +// VacuumJournal deletes only strictly recognized archived journals from the +// configured target while honoring the requested size and time thresholds. +func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + if request.Now.IsZero() { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum reference time is required") + } + if request.MaxBytes == 0 && request.Before.IsZero() { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum requires a size or time limit") + } + if request.MaxBytes > 0 && request.Before.IsZero() { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal size vacuum requires a time cutoff") + } + if !request.Before.IsZero() && request.Before.After(request.Now) { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum time cutoff cannot be in the future") + } + + directories, err := c.openVacuumDirectories() + if err != nil { + return builtins.JournalVacuumResult{}, err + } + defer closeVacuumDirectories(directories) + + candidates, archivedBytes, err := collectVacuumCandidates(directories) + if err != nil { + return builtins.JournalVacuumResult{}, err + } + remainingBytes := archivedBytes + result := builtins.JournalVacuumResult{} + + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return result, vacuumPartialError(result, err) + } + oldEnough := !candidate.modTime.After(request.Before) + overSize := request.MaxBytes > 0 && remainingBytes > request.MaxBytes + if !oldEnough || (request.MaxBytes > 0 && !overSize) { + break + } + if err := revalidateVacuumCandidate(candidate); err != nil { + return result, vacuumPartialError(result, err) + } + if !request.DryRun { + if err := candidate.directory.root.Remove(candidate.name); err != nil { + return result, vacuumPartialError(result, fmt.Errorf("remove archived journal: %w", err)) + } + } + result.Files++ + result.Bytes += candidate.stat.allocated + remainingBytes -= candidate.stat.allocated + } + return result, nil +} + +func (c *Client) openVacuumDirectories() ([]*vacuumDirectory, error) { + if c.target.MachineIDPath == "" { + return nil, fmt.Errorf("systemd target machine ID path is not configured") + } + if len(c.target.JournalDirs) == 0 { + return nil, fmt.Errorf("systemd target journal directories are not configured") + } + machineID, err := readMachineID(c.target.MachineIDPath) + if err != nil { + return nil, err + } + + directories := make([]*vacuumDirectory, 0, len(c.target.JournalDirs)) + for _, basePath := range c.target.JournalDirs { + baseRoot, err := os.OpenRoot(basePath) + if err != nil { + if os.IsNotExist(err) { + continue + } + closeVacuumDirectories(directories) + return nil, fmt.Errorf("open journal root %q: %w", basePath, err) + } + before, err := baseRoot.Lstat(machineID) + if err != nil { + baseRoot.Close() + if os.IsNotExist(err) { + continue + } + closeVacuumDirectories(directories) + return nil, fmt.Errorf("inspect journal machine directory: %w", err) + } + if !before.IsDir() || before.Mode()&fs.ModeSymlink != 0 { + baseRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("journal machine directory is not a real directory") + } + beforeStat, err := journalStat(before) + if err != nil { + baseRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("inspect journal machine directory identity: %w", err) + } + machineRoot, err := baseRoot.OpenRoot(machineID) + baseRoot.Close() + if err != nil { + closeVacuumDirectories(directories) + return nil, fmt.Errorf("open journal machine directory: %w", err) + } + after, err := machineRoot.Stat(".") + if err != nil { + machineRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("verify journal machine directory: %w", err) + } + afterStat, err := journalStat(after) + if err != nil || beforeStat.dev != afterStat.dev || beforeStat.ino != afterStat.ino { + machineRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("journal machine directory changed while opening") + } + directories = append(directories, &vacuumDirectory{path: basePath, root: machineRoot}) + } + return directories, nil +} + +func collectVacuumCandidates(directories []*vacuumDirectory) ([]vacuumCandidate, uint64, error) { + candidates := make([]vacuumCandidate, 0) + var archivedBytes uint64 + for _, directory := range directories { + handle, err := directory.root.Open(".") + if err != nil { + return nil, 0, fmt.Errorf("open pinned journal directory: %w", err) + } + entries, readErr := handle.ReadDir(maxJournalFiles + 1) + closeErr := handle.Close() + if readErr != nil && readErr != io.EOF { + return nil, 0, fmt.Errorf("read pinned journal directory: %w", readErr) + } + if closeErr != nil { + return nil, 0, fmt.Errorf("close pinned journal directory: %w", closeErr) + } + if len(entries) > maxJournalFiles { + return nil, 0, fmt.Errorf("journal directory has too many entries (maximum %d)", maxJournalFiles) + } + for _, entry := range entries { + if !isArchivedJournalName(entry.Name()) || entry.Type()&fs.ModeSymlink != 0 { + continue + } + info, err := directory.root.Lstat(entry.Name()) + if err != nil { + return nil, 0, fmt.Errorf("inspect archived journal: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&fs.ModeSymlink != 0 { + continue + } + stat, err := journalStat(info) + if err != nil { + return nil, 0, fmt.Errorf("inspect archived journal allocation: %w", err) + } + if stat.nlink != 1 { + continue + } + if archivedBytes > math.MaxUint64-stat.allocated { + return nil, 0, fmt.Errorf("archived journal allocation total overflow") + } + archivedBytes += stat.allocated + candidates = append(candidates, vacuumCandidate{ + directory: directory, + name: entry.Name(), + modTime: info.ModTime(), + size: info.Size(), + stat: stat, + }) + if len(candidates) > maxJournalFiles { + return nil, 0, fmt.Errorf("systemd target has too many archived journal files (maximum %d)", maxJournalFiles) + } + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].modTime.Equal(candidates[j].modTime) { + if candidates[i].directory.path == candidates[j].directory.path { + return candidates[i].name < candidates[j].name + } + return candidates[i].directory.path < candidates[j].directory.path + } + return candidates[i].modTime.Before(candidates[j].modTime) + }) + return candidates, archivedBytes, nil +} + +func revalidateVacuumCandidate(candidate vacuumCandidate) error { + info, err := candidate.directory.root.Lstat(candidate.name) + if err != nil { + return fmt.Errorf("revalidate archived journal: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&fs.ModeSymlink != 0 || info.Size() != candidate.size || !info.ModTime().Equal(candidate.modTime) { + return fmt.Errorf("archived journal changed before deletion") + } + stat, err := journalStat(info) + if err != nil { + return fmt.Errorf("revalidate archived journal allocation: %w", err) + } + if stat.dev != candidate.stat.dev || stat.ino != candidate.stat.ino || stat.nlink != 1 || stat.blocks != candidate.stat.blocks { + return fmt.Errorf("archived journal identity changed before deletion") + } + return nil +} + +func isArchivedJournalName(name string) bool { + if strings.IndexByte(name, '/') >= 0 || strings.IndexByte(name, 0) >= 0 || !strings.HasSuffix(name, ".journal") { + return false + } + stem := strings.TrimSuffix(name, ".journal") + separator := strings.LastIndexByte(stem, '@') + if separator <= 0 || separator == len(stem)-1 { + return false + } + parts := strings.Split(stem[separator+1:], "-") + return len(parts) == 3 && validHexLength(parts[0], 32) && validHexLength(parts[1], 16) && validHexLength(parts[2], 16) +} + +func validHexLength(value string, length int) bool { + if len(value) != length { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} + +func closeVacuumDirectories(directories []*vacuumDirectory) { + for _, directory := range directories { + directory.root.Close() + } +} + +func vacuumPartialError(result builtins.JournalVacuumResult, err error) error { + if result.Files == 0 { + return err + } + return fmt.Errorf("journal vacuum stopped after deleting %d files (%d bytes): %w", result.Files, result.Bytes, err) +} diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go new file mode 100644 index 00000000..a3f6fff9 --- /dev/null +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -0,0 +1,197 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux || darwin + +package systemd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +const testMachineID = "0123456789abcdef0123456789abcdef" + +func archivedJournalName(index int) string { + return fmt.Sprintf("system@abcdef0123456789abcdef0123456789-%016x-%016x.journal", index, index) +} + +func newVacuumTestClient(t *testing.T) (*Client, string) { + t.Helper() + root := t.TempDir() + machineIDPath := filepath.Join(root, "machine-id") + journalRoot := filepath.Join(root, "journal") + machineDir := filepath.Join(journalRoot, testMachineID) + require.NoError(t, os.WriteFile(machineIDPath, []byte(testMachineID+"\n"), 0o600)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + return NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}), machineDir +} + +func writeVacuumFile(t *testing.T, directory, name string, modTime time.Time) string { + t.Helper() + path := filepath.Join(directory, name) + require.NoError(t, os.WriteFile(path, make([]byte, 8192), 0o600)) + require.NoError(t, os.Chtimes(path, modTime, modTime)) + return path +} + +func TestVacuumJournalDeletesOldestArchivesWithinRequest(t *testing.T) { + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) + client, directory := newVacuumTestClient(t) + active := writeVacuumFile(t, directory, "system.journal", now.Add(-30*24*time.Hour)) + namespaceActive := writeVacuumFile(t, directory, "system@tenant.journal", now.Add(-30*24*time.Hour)) + malformed := writeVacuumFile(t, directory, "system@old.journal", now.Add(-30*24*time.Hour)) + oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + old := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-5*24*time.Hour)) + recent := writeVacuumFile(t, directory, archivedJournalName(3), now.Add(-time.Hour)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Equal(t, 2, result.Files) + assert.Greater(t, result.Bytes, uint64(0)) + assert.NoFileExists(t, oldest) + assert.NoFileExists(t, old) + assert.FileExists(t, recent) + assert.FileExists(t, active) + assert.FileExists(t, namespaceActive) + assert.FileExists(t, malformed) +} + +func TestVacuumJournalDryRunDoesNotDelete(t *testing.T) { + now := time.Now() + client, directory := newVacuumTestClient(t) + archive := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-7*24*time.Hour)) + second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-6*24*time.Hour)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + DryRun: true, + }) + require.NoError(t, err) + assert.Equal(t, 2, result.Files) + assert.FileExists(t, archive) + assert.FileExists(t, second) +} + +func TestVacuumJournalHonorsAllocatedSizeTargetAndTimeFloor(t *testing.T) { + now := time.Now() + client, directory := newVacuumTestClient(t) + oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-9*24*time.Hour)) + recent := writeVacuumFile(t, directory, archivedJournalName(3), now.Add(-time.Hour)) + info, err := os.Lstat(oldest) + require.NoError(t, err) + stat, err := journalStat(info) + require.NoError(t, err) + require.Greater(t, stat.allocated, uint64(1)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + MaxBytes: stat.allocated - 1, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Equal(t, 2, result.Files) + assert.NoFileExists(t, oldest) + assert.NoFileExists(t, second) + assert.FileExists(t, recent) +} + +func TestVacuumJournalSkipsHardlinksAndSymlinks(t *testing.T) { + now := time.Now() + client, directory := newVacuumTestClient(t) + first := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + second := filepath.Join(directory, archivedJournalName(2)) + require.NoError(t, os.Link(first, second)) + symlink := filepath.Join(directory, archivedJournalName(3)) + require.NoError(t, os.Symlink(first, symlink)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Zero(t, result.Files) + assert.FileExists(t, first) + assert.FileExists(t, second) + _, err = os.Lstat(symlink) + require.NoError(t, err) +} + +func TestVacuumJournalRejectsSymlinkedMachineDirectory(t *testing.T) { + now := time.Now() + root := t.TempDir() + journalRoot := filepath.Join(root, "journal") + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(journalRoot, 0o700)) + require.NoError(t, os.MkdirAll(outside, 0o700)) + archive := writeVacuumFile(t, outside, archivedJournalName(1), now.Add(-10*24*time.Hour)) + require.NoError(t, os.Symlink(outside, filepath.Join(journalRoot, testMachineID))) + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(testMachineID+"\n"), 0o600)) + client := NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}) + + _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a real directory") + assert.FileExists(t, archive) +} + +func TestVacuumJournalRequiresRequestBounds(t *testing.T) { + now := time.Now() + client, _ := newVacuumTestClient(t) + _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Before: now.Add(-time.Hour)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "reference time") + + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now}) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a size or time limit") + + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, MaxBytes: 1}) + require.Error(t, err) + assert.Contains(t, err.Error(), "size vacuum requires a time cutoff") + + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, Before: now.Add(time.Hour)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be in the future") +} + +func TestIsArchivedJournalName(t *testing.T) { + for _, name := range []string{ + archivedJournalName(1), + "system@tenant@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal", + "user-1000@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal", + } { + assert.True(t, isArchivedJournalName(name), name) + } + for _, name := range []string{ + "system.journal", + "user-1000.journal", + "system@tenant.journal", + "system@old.journal", + "system@abcdef0123456789abcdef0123456789-1-1.journal", + "system@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal~", + "../" + archivedJournalName(1), + } { + assert.False(t, isArchivedJournalName(name), name) + } +} diff --git a/internal/systemd/journal_vacuum_unsupported.go b/internal/systemd/journal_vacuum_unsupported.go new file mode 100644 index 00000000..f9b6c402 --- /dev/null +++ b/internal/systemd/journal_vacuum_unsupported.go @@ -0,0 +1,20 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !linux && !darwin + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// VacuumJournal requires rooted Unix filesystem operations. +func (c *Client) VacuumJournal(context.Context, builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + return builtins.JournalVacuumResult{}, fmt.Errorf("%w: journal vacuum requires Linux or macOS", builtins.ErrSystemdUnsupported) +} diff --git a/internal/systemd/journal_varlink.go b/internal/systemd/journal_varlink.go new file mode 100644 index 00000000..c622da4f --- /dev/null +++ b/internal/systemd/journal_varlink.go @@ -0,0 +1,187 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "time" +) + +const ( + journalRotateMethod = "io.systemd.Journal.Rotate" + maxVarlinkMessageSize = 64 * 1024 +) + +type varlinkRequest struct { + Method string `json:"method"` +} + +type varlinkReply struct { + Parameters json.RawMessage `json:"parameters"` + Error string `json:"error"` + Continues bool `json:"continues"` +} + +func rotateJournalControl(ctx context.Context, path string) error { + before, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect journal control socket: %w", err) + } + if before.Mode()&os.ModeSymlink != 0 || before.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("journal control endpoint is not a Unix socket") + } + + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "unix", path) + if err != nil { + return contextIOError(ctx, "connect to journal control socket", err) + } + defer conn.Close() + + after, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("reinspect journal control socket: %w", err) + } + if !os.SameFile(before, after) { + return fmt.Errorf("journal control socket changed while connecting") + } + + return callJournalRotateVarlink(ctx, conn) +} + +func callJournalRotateVarlink(ctx context.Context, conn net.Conn) error { + stopCancel := context.AfterFunc(ctx, func() { + _ = conn.SetDeadline(time.Now()) + }) + defer stopCancel() + if deadline, ok := ctx.Deadline(); ok { + if err := conn.SetDeadline(deadline); err != nil { + return fmt.Errorf("set journal control deadline: %w", err) + } + } + + request, err := json.Marshal(varlinkRequest{Method: journalRotateMethod}) + if err != nil { + return fmt.Errorf("encode journal rotation request: %w", err) + } + request = append(request, 0) + if err := writeAll(conn, request); err != nil { + return contextIOError(ctx, "write journal rotation request", err) + } + + message, err := readVarlinkMessage(conn) + if err != nil { + return contextIOError(ctx, "read journal rotation response", err) + } + trimmed := bytes.TrimSpace(message) + if len(trimmed) < 2 || trimmed[0] != '{' || trimmed[len(trimmed)-1] != '}' { + return fmt.Errorf("journal control returned a malformed response") + } + + var reply varlinkReply + if err := json.Unmarshal(trimmed, &reply); err != nil { + return fmt.Errorf("decode journal rotation response: %w", err) + } + if reply.Continues { + return fmt.Errorf("journal control returned an unexpected streaming response") + } + if reply.Error != "" { + if !validVarlinkIdentifier(reply.Error) { + return fmt.Errorf("journald rejected journal rotation with an invalid protocol error") + } + return fmt.Errorf("journald rejected journal rotation: %s", reply.Error) + } + if len(reply.Parameters) > 0 { + var parameters map[string]json.RawMessage + if err := json.Unmarshal(reply.Parameters, ¶meters); err != nil || parameters == nil { + return fmt.Errorf("journal control returned malformed response parameters") + } + if len(parameters) != 0 { + return fmt.Errorf("journal control returned unexpected response parameters") + } + } + return nil +} + +func writeAll(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if err != nil { + return err + } + if n <= 0 || n > len(data) { + return io.ErrShortWrite + } + data = data[n:] + } + return nil +} + +func readVarlinkMessage(reader io.Reader) ([]byte, error) { + message := make([]byte, 0, 512) + buffer := make([]byte, 4096) + for { + n, err := reader.Read(buffer) + if n > 0 { + chunk := buffer[:n] + if terminator := bytes.IndexByte(chunk, 0); terminator >= 0 { + if len(message)+terminator > maxVarlinkMessageSize { + return nil, fmt.Errorf("Varlink response exceeds %d bytes", maxVarlinkMessageSize) + } + message = append(message, chunk[:terminator]...) + if len(chunk[terminator+1:]) > 0 { + return nil, fmt.Errorf("Varlink response contains trailing data") + } + if len(message) == 0 { + return nil, fmt.Errorf("Varlink response is empty") + } + return message, nil + } + if len(message)+len(chunk) > maxVarlinkMessageSize { + return nil, fmt.Errorf("Varlink response exceeds %d bytes", maxVarlinkMessageSize) + } + message = append(message, chunk...) + } + if err != nil { + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("Varlink response ended without a terminator") + } + return nil, err + } + } +} + +func validVarlinkIdentifier(value string) bool { + if value == "" || len(value) > 256 { + return false + } + for _, char := range []byte(value) { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '.' || char == '_' { + continue + } + return false + } + return true +} + +func contextIOError(ctx context.Context, operation string, err error) error { + if contextErr := ctx.Err(); contextErr != nil { + return contextErr + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if deadline, hasDeadline := ctx.Deadline(); hasDeadline && !time.Now().Before(deadline) { + return context.DeadlineExceeded + } + } + return fmt.Errorf("%s: %w", operation, err) +} diff --git a/internal/systemd/journal_varlink_test.go b/internal/systemd/journal_varlink_test.go new file mode 100644 index 00000000..86bec11d --- /dev/null +++ b/internal/systemd/journal_varlink_test.go @@ -0,0 +1,138 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func serveVarlinkResponse(conn net.Conn, response []byte, gate <-chan struct{}) (<-chan []byte, <-chan error) { + request := make(chan []byte, 1) + finished := make(chan error, 1) + go func() { + defer conn.Close() + message, err := readVarlinkMessage(conn) + if err != nil { + finished <- err + return + } + request <- message + if gate != nil { + <-gate + } + response = append(append([]byte(nil), response...), 0) + finished <- writeAll(conn, response) + }() + return request, finished +} + +func TestRotateJournalControlUsesFixedSynchronousMethod(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + gate := make(chan struct{}) + request, finished := serveVarlinkResponse(server, []byte(`{"parameters":{}}`), gate) + + rotationDone := make(chan error, 1) + go func() { + rotationDone <- callJournalRotateVarlink(context.Background(), client) + }() + + var decoded varlinkRequest + require.NoError(t, json.Unmarshal(<-request, &decoded)) + assert.Equal(t, journalRotateMethod, decoded.Method) + select { + case err := <-rotationDone: + t.Fatalf("rotation returned before journald replied: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(gate) + require.NoError(t, <-rotationDone) + require.NoError(t, <-finished) +} + +func TestRotateJournalControlReportsSafeProtocolErrors(t *testing.T) { + for _, test := range []struct { + name string + response string + needle string + }{ + {name: "daemon error", response: `{"error":"io.systemd.Journal.NotSupported","parameters":{}}`, needle: "io.systemd.Journal.NotSupported"}, + {name: "unsafe daemon error", response: "{\"error\":\"bad\\nterminal\"}", needle: "invalid protocol error"}, + {name: "streaming", response: `{"parameters":{},"continues":true}`, needle: "streaming response"}, + {name: "unexpected parameters", response: `{"parameters":{"path":"/host"}}`, needle: "unexpected response parameters"}, + {name: "malformed", response: `[]`, needle: "malformed response"}, + } { + t.Run(test.name, func(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + _, finished := serveVarlinkResponse(server, []byte(test.response), nil) + + err := callJournalRotateVarlink(context.Background(), client) + require.Error(t, err) + assert.Contains(t, err.Error(), test.needle) + if test.name == "unsafe daemon error" { + assert.NotContains(t, err.Error(), "terminal") + } + require.NoError(t, <-finished) + }) + } +} + +func TestRotateJournalControlHonorsCancellation(t *testing.T) { + client, server := net.Pipe() + gate := make(chan struct{}) + _, finished := serveVarlinkResponse(server, []byte(`{"parameters":{}}`), gate) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + err := callJournalRotateVarlink(ctx, client) + assert.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, client.Close()) + close(gate) + serverErr := <-finished + if serverErr != nil { + assert.True(t, errors.Is(serverErr, net.ErrClosed) || errors.Is(serverErr, io.ErrClosedPipe) || strings.Contains(serverErr.Error(), "broken pipe"), serverErr.Error()) + } +} + +func TestRotateJournalControlRejectsNonSocketEndpoints(t *testing.T) { + regular := filepath.Join(t.TempDir(), "journal.sock") + require.NoError(t, os.WriteFile(regular, []byte("not a socket"), 0o600)) + err := rotateJournalControl(context.Background(), regular) + assert.ErrorContains(t, err, "not a Unix socket") + + if runtime.GOOS == "windows" { + return + } + symlink := filepath.Join(t.TempDir(), "journal-link.sock") + require.NoError(t, os.Symlink(regular, symlink)) + err = rotateJournalControl(context.Background(), symlink) + assert.ErrorContains(t, err, "not a Unix socket") +} + +func TestReadVarlinkMessageBoundsResponses(t *testing.T) { + _, err := readVarlinkMessage(strings.NewReader(strings.Repeat("x", maxVarlinkMessageSize+1) + "\x00")) + assert.ErrorContains(t, err, "exceeds") + + _, err = readVarlinkMessage(strings.NewReader("{}")) + assert.ErrorContains(t, err, "without a terminator") + + _, err = readVarlinkMessage(strings.NewReader("{}\x00trailing")) + assert.ErrorContains(t, err, "trailing data") +} diff --git a/internal/systemd/target.go b/internal/systemd/target.go new file mode 100644 index 00000000..2d1b3355 --- /dev/null +++ b/internal/systemd/target.go @@ -0,0 +1,114 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +// Package systemd contains the trusted target and transport implementation +// used by systemd-aware builtins. +package systemd + +import ( + "fmt" + "path/filepath" + "strings" +) + +const MaxJournalDirs = 8 + +// Target identifies one systemd host. Root is accepted only as input to +// ResolveTarget; resolved targets contain explicit paths in the remaining +// fields. +type Target struct { + Root string + JournalDirs []string + MachineIDPath string + JournalControlSocket string + SystemBusSocket string +} + +// LocalTarget returns the standard paths for the local systemd host. +func LocalTarget() Target { + return targetFromRoot("/") +} + +// ResolveTarget validates and resolves a target. A zero target selects the +// local host. Root derives every standard path below that root and cannot be +// mixed with explicit fields. Once any explicit field is supplied, omitted +// fields remain empty and never fall back to local paths. +func ResolveTarget(target Target) (Target, error) { + if target.Root != "" { + if len(target.JournalDirs) > 0 || target.MachineIDPath != "" || target.JournalControlSocket != "" || target.SystemBusSocket != "" { + return Target{}, fmt.Errorf("systemd target root cannot be combined with explicit paths") + } + root, err := validateAbsolutePath("root", target.Root) + if err != nil { + return Target{}, err + } + return targetFromRoot(root), nil + } + + if len(target.JournalDirs) == 0 && target.MachineIDPath == "" && target.JournalControlSocket == "" && target.SystemBusSocket == "" { + return LocalTarget(), nil + } + if len(target.JournalDirs) > MaxJournalDirs { + return Target{}, fmt.Errorf("systemd target has %d journal directories; maximum is %d", len(target.JournalDirs), MaxJournalDirs) + } + if target.MachineIDPath == "" { + return Target{}, fmt.Errorf("systemd target machine ID path is required with explicit paths") + } + + resolved := Target{JournalDirs: make([]string, 0, len(target.JournalDirs))} + seenDirs := make(map[string]struct{}, len(target.JournalDirs)) + for i, dir := range target.JournalDirs { + clean, err := validateAbsolutePath(fmt.Sprintf("journal directory %d", i), dir) + if err != nil { + return Target{}, err + } + if _, exists := seenDirs[clean]; exists { + continue + } + seenDirs[clean] = struct{}{} + resolved.JournalDirs = append(resolved.JournalDirs, clean) + } + + var err error + if resolved.MachineIDPath, err = validateAbsolutePath("machine ID path", target.MachineIDPath); err != nil { + return Target{}, err + } + if resolved.JournalControlSocket, err = validateOptionalAbsolutePath("journal control socket", target.JournalControlSocket); err != nil { + return Target{}, err + } + if resolved.SystemBusSocket, err = validateOptionalAbsolutePath("system bus socket", target.SystemBusSocket); err != nil { + return Target{}, err + } + return resolved, nil +} + +func targetFromRoot(root string) Target { + return Target{ + JournalDirs: []string{ + filepath.Join(root, "var", "log", "journal"), + filepath.Join(root, "run", "log", "journal"), + }, + MachineIDPath: filepath.Join(root, "etc", "machine-id"), + JournalControlSocket: filepath.Join(root, "run", "systemd", "journal", "io.systemd.journal"), + SystemBusSocket: filepath.Join(root, "run", "dbus", "system_bus_socket"), + } +} + +func validateOptionalAbsolutePath(name, path string) (string, error) { + if path == "" { + return "", nil + } + return validateAbsolutePath(name, path) +} + +func validateAbsolutePath(name, path string) (string, error) { + if strings.IndexByte(path, 0) >= 0 { + return "", fmt.Errorf("systemd target %s contains a NUL byte", name) + } + if !filepath.IsAbs(path) { + return "", fmt.Errorf("systemd target %s %q must be absolute", name, path) + } + return filepath.Clean(path), nil +} diff --git a/internal/systemd/target_test.go b/internal/systemd/target_test.go new file mode 100644 index 00000000..c3a37782 --- /dev/null +++ b/internal/systemd/target_test.go @@ -0,0 +1,104 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !windows + +package systemd + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveTargetDefaultsToLocalPaths(t *testing.T) { + target, err := ResolveTarget(Target{}) + require.NoError(t, err) + + assert.Equal(t, []string{"/var/log/journal", "/run/log/journal"}, target.JournalDirs) + assert.Equal(t, "/etc/machine-id", target.MachineIDPath) + assert.Equal(t, "/run/systemd/journal/io.systemd.journal", target.JournalControlSocket) + assert.Equal(t, "/run/dbus/system_bus_socket", target.SystemBusSocket) +} + +func TestResolveTargetDerivesMountedRootPaths(t *testing.T) { + target, err := ResolveTarget(Target{Root: "/host"}) + require.NoError(t, err) + + assert.Equal(t, []string{filepath.FromSlash("/host/var/log/journal"), filepath.FromSlash("/host/run/log/journal")}, target.JournalDirs) + assert.Equal(t, filepath.FromSlash("/host/etc/machine-id"), target.MachineIDPath) + assert.Equal(t, filepath.FromSlash("/host/run/systemd/journal/io.systemd.journal"), target.JournalControlSocket) + assert.Equal(t, filepath.FromSlash("/host/run/dbus/system_bus_socket"), target.SystemBusSocket) +} + +func TestResolveTargetUsesOnlyExplicitPaths(t *testing.T) { + target, err := ResolveTarget(Target{ + JournalDirs: []string{"/mnt/logs", "/mnt/logs", "/mnt/runtime-logs/../runtime-logs"}, + MachineIDPath: "/mnt/etc/machine-id", + }) + require.NoError(t, err) + + assert.Equal(t, []string{"/mnt/logs", "/mnt/runtime-logs"}, target.JournalDirs) + assert.Equal(t, "/mnt/etc/machine-id", target.MachineIDPath) + assert.Empty(t, target.JournalControlSocket) + assert.Empty(t, target.SystemBusSocket) +} + +func TestResolveTargetRejectsInvalidConfiguration(t *testing.T) { + tests := []struct { + name string + target Target + needle string + }{ + { + name: "root with explicit field", + target: Target{Root: "/host", MachineIDPath: "/host/etc/machine-id"}, + needle: "cannot be combined", + }, + { + name: "relative root", + target: Target{Root: "host"}, + needle: "must be absolute", + }, + { + name: "explicit without machine ID", + target: Target{JournalDirs: []string{"/host/var/log/journal"}}, + needle: "machine ID path is required", + }, + { + name: "relative journal directory", + target: Target{JournalDirs: []string{"logs"}, MachineIDPath: "/etc/machine-id"}, + needle: "must be absolute", + }, + { + name: "relative socket", + target: Target{MachineIDPath: "/etc/machine-id", SystemBusSocket: "run/dbus/system_bus_socket"}, + needle: "must be absolute", + }, + { + name: "NUL path", + target: Target{MachineIDPath: "/etc/machine-id\x00suffix"}, + needle: "NUL byte", + }, + { + name: "too many journal directories", + target: Target{ + JournalDirs: []string{"/1", "/2", "/3", "/4", "/5", "/6", "/7", "/8", "/9"}, + MachineIDPath: "/etc/machine-id", + }, + needle: "maximum is 8", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ResolveTarget(test.target) + require.Error(t, err) + assert.Contains(t, err.Error(), test.needle) + }) + } +} diff --git a/internal/systemd/testdata/journal/README.md b/internal/systemd/testdata/journal/README.md new file mode 100644 index 00000000..c895f613 --- /dev/null +++ b/internal/systemd/testdata/journal/README.md @@ -0,0 +1,33 @@ +# Journal fixtures + +These fixtures contain four synthetic records with fixed machine, boot, and +timestamps. They contain no logs or identifiers copied from a real host. The +generation script also replaces the journal header's generator machine ID with +the fixed synthetic machine ID before verification. + +The two committed golden files were generated and verified with systemd 255.4 +on Ubuntu 24.04. `regular-uncompressed.journal.gz` exercises the original +regular layout. `compact-keyed-zstd.journal.gz` exercises compact offsets, +keyed hashes, and a Zstandard-compressed DATA object. Gzip only keeps the sparse +8 MiB journal files small in Git; tests decompress them in memory or into a +per-test temporary directory. Test runs do not write generated files here. + +The golden files remain committed so tests can validate files emitted by real +systemd on hosts and CI workers that do not provide `systemd-journal-remote`. +`fixture.export` is their synthetic source rather than generated output. + +On a Linux system with `systemd-journal-remote`, generate a fresh corpus in an +explicit temporary output directory and test it with: + +```sh +output_dir=$(mktemp -d) +./generate.sh "$output_dir" +RSHELL_JOURNAL_FIXTURE_DIR="$output_dir" \ + go test ../.. \ + -run 'TestRealJournalFixtures|TestJournalFixtureMatchesJournalctl|TestReadJournalUsesPureGoFixtureBackend' +``` + +Journal file and sequence IDs are randomized by systemd, so regeneration is +semantically deterministic but does not reproduce byte-identical files. The +generator never replaces the committed golden corpus unless this directory is +passed explicitly as its output directory. diff --git a/internal/systemd/testdata/journal/compact-keyed-zstd.journal.gz b/internal/systemd/testdata/journal/compact-keyed-zstd.journal.gz new file mode 100644 index 00000000..de806161 Binary files /dev/null and b/internal/systemd/testdata/journal/compact-keyed-zstd.journal.gz differ diff --git a/internal/systemd/testdata/journal/fixture.export b/internal/systemd/testdata/journal/fixture.export new file mode 100644 index 00000000..d337ab4b --- /dev/null +++ b/internal/systemd/testdata/journal/fixture.export @@ -0,0 +1,50 @@ +__REALTIME_TIMESTAMP=1700000000000000 +__MONOTONIC_TIMESTAMP=1000000 +__SEQNUM=1 +__SEQNUM_ID=cccccccccccccccccccccccccccccccc +_BOOT_ID=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +_MACHINE_ID=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +_HOSTNAME=fixture-host +_TRANSPORT=stdout +_SYSTEMD_UNIT=rshell-fixture.service +SYSLOG_IDENTIFIER=fixture-service +_PID=4242 +MESSAGE=service started + +__REALTIME_TIMESTAMP=1700000001000000 +__MONOTONIC_TIMESTAMP=2000000 +__SEQNUM=2 +__SEQNUM_ID=cccccccccccccccccccccccccccccccc +_BOOT_ID=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +_MACHINE_ID=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +_HOSTNAME=fixture-host +_TRANSPORT=journal +UNIT=rshell-fixture.service +SYSLOG_IDENTIFIER=systemd +_PID=1 +MESSAGE=manager noticed service + +__REALTIME_TIMESTAMP=1700000002000000 +__MONOTONIC_TIMESTAMP=3000000 +__SEQNUM=3 +__SEQNUM_ID=cccccccccccccccccccccccccccccccc +_BOOT_ID=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +_MACHINE_ID=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +_HOSTNAME=fixture-host +_TRANSPORT=kernel +_COMM=kernel +_PID=0 +MESSAGE=synthetic kernel event + +__REALTIME_TIMESTAMP=1700000003000000 +__MONOTONIC_TIMESTAMP=4000000 +__SEQNUM=4 +__SEQNUM_ID=cccccccccccccccccccccccccccccccc +_BOOT_ID=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +_MACHINE_ID=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +_HOSTNAME=fixture-host +_TRANSPORT=stdout +_SYSTEMD_UNIT=rshell-fixture.service +SYSLOG_IDENTIFIER=fixture-service +_PID=4242 +MESSAGE=compressed payload aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/internal/systemd/testdata/journal/generate.sh b/internal/systemd/testdata/journal/generate.sh new file mode 100755 index 00000000..d5223924 --- /dev/null +++ b/internal/systemd/testdata/journal/generate.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -eu + +fixture_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +remote=${SYSTEMD_JOURNAL_REMOTE:-/usr/lib/systemd/systemd-journal-remote} + +if [ "$#" -ne 1 ]; then + echo "usage: $0 OUTPUT_DIR" >&2 + exit 2 +fi + +if [ ! -x "$remote" ]; then + echo "systemd-journal-remote not found at $remote" >&2 + exit 1 +fi + +output_dir=$1 +mkdir -p "$output_dir" +output_dir=$(CDPATH= cd -- "$output_dir" && pwd) + +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT HUP INT TERM +export_file="$tmpdir/fixture.export" +cp "$fixture_dir/fixture.export" "$export_file" +# Journal Export Format terminates every record, including the last, with an +# empty line. Keep that delimiter out of the tracked source's trailing space. +printf '\n' >>"$export_file" + +normalize_machine_id() { + # journal-remote uses the generator host's ID in the file header. Keep the + # fixture synthetic and consistent with fixture.export instead. + printf '\252\252\252\252\252\252\252\252\252\252\252\252\252\252\252\252' | + dd of="$1" bs=1 seek=40 conv=notrunc status=none +} + +SYSTEMD_JOURNAL_COMPACT=0 \ + SYSTEMD_JOURNAL_KEYED_HASH=0 \ + SYSTEMD_JOURNAL_COMPRESS=none \ + "$remote" \ + --split-mode=none \ + --compress=no \ + --output="$tmpdir/regular-uncompressed.journal" \ + "$export_file" + +SYSTEMD_JOURNAL_COMPACT=1 \ + SYSTEMD_JOURNAL_KEYED_HASH=1 \ + SYSTEMD_JOURNAL_COMPRESS=zstd \ + "$remote" \ + --split-mode=none \ + --compress=yes \ + --output="$tmpdir/compact-keyed-zstd.journal" \ + "$export_file" + +normalize_machine_id "$tmpdir/regular-uncompressed.journal" +normalize_machine_id "$tmpdir/compact-keyed-zstd.journal" + +journalctl --verify --file="$tmpdir/regular-uncompressed.journal" +journalctl --verify --file="$tmpdir/compact-keyed-zstd.journal" + +gzip -n "$tmpdir/regular-uncompressed.journal" +gzip -n "$tmpdir/compact-keyed-zstd.journal" +mv "$tmpdir/regular-uncompressed.journal.gz" "$output_dir/regular-uncompressed.journal.gz" +mv "$tmpdir/compact-keyed-zstd.journal.gz" "$output_dir/compact-keyed-zstd.journal.gz" + +printf 'generated journal fixtures in %s\n' "$output_dir" diff --git a/internal/systemd/testdata/journal/regular-uncompressed.journal.gz b/internal/systemd/testdata/journal/regular-uncompressed.journal.gz new file mode 100644 index 00000000..d4c06a94 Binary files /dev/null and b/internal/systemd/testdata/journal/regular-uncompressed.journal.gz differ diff --git a/interp/api.go b/interp/api.go index 5e508dad..e94a72e9 100644 --- a/interp/api.go +++ b/interp/api.go @@ -29,6 +29,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry" "github.com/DataDog/rshell/allowedpaths" "github.com/DataDog/rshell/builtins" + internalsystemd "github.com/DataDog/rshell/internal/systemd" "github.com/DataDog/rshell/internal/version" ) @@ -80,10 +81,17 @@ type runnerConfig struct { // command. Intended for testing convenience. allowAllCommands bool - // allowedSystemServices maps exact service names to their permitted - // actions. It is independent of allowAllCommands and defaults to denying - // every service. - allowedSystemServices systemServiceGrants + // allowedSystemServices maps exact services to their permitted actions. It + // is independent of allowAllCommands and + // defaults to denying every systemd operation. + allowedSystemServices systemdGrants + + // systemdTarget identifies the local or mounted host used by systemd-aware + // builtins. It is resolved once during construction and shared by + // subshells. + systemdTarget internalsystemd.Target + systemdTargetConfigured bool + systemd *builtins.SystemdServices // maxExecutionTime bounds the duration of each Run call. Zero disables // the limit. When non-zero, Run derives a child context with this timeout. @@ -321,6 +329,16 @@ func New(opts ...RunnerOption) (*Runner, error) { if len(r.sandboxWarnings) > 0 { r.warningsWriter.Write(r.sandboxWarnings) } + if !r.systemdTargetConfigured { + r.systemdTarget = internalsystemd.LocalTarget() + } + systemdClient := internalsystemd.NewClient(r.systemdTarget) + r.systemd = &builtins.SystemdServices{ + Journal: systemdClient, + JournalStorage: systemdClient, + JournalCleaner: systemdClient, + JournalRotator: systemdClient, + } r.proc = builtins.NewProcProvider(r.procPath) return r, nil } diff --git a/interp/journal_vacuum_integration_test.go b/interp/journal_vacuum_integration_test.go new file mode 100644 index 00000000..55a432d3 --- /dev/null +++ b/interp/journal_vacuum_integration_test.go @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux || darwin + +package interp + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalctlVacuumEndToEnd(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineDir := filepath.Join(root, "var", "log", "journal", machineID) + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte(machineID+"\n"), 0o600)) + archive := filepath.Join(machineDir, "system@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal") + require.NoError(t, os.WriteFile(archive, make([]byte, 8192), 0o600)) + old := time.Now().Add(-7 * 24 * time.Hour) + require.NoError(t, os.Chtimes(archive, old, old)) + + var stdout, stderr bytes.Buffer + runner, err := New( + StdIO(nil, &stdout, &stderr), + AllowedCommands([]string{"rshell:journalctl"}), + AllowedSystemServices([]SystemdControlGrant{{ + Service: "systemd-journald.service", + Actions: []SystemdAction{SystemdActionClean}, + }}), + WithMode(ModeRemediation), + WithSystemdTarget(SystemdTargetConfig{Root: root}), + ) + require.NoError(t, err) + defer runner.Close() + + program, err := ParseScript("journalctl --vacuum-time=48h", "") + require.NoError(t, err) + require.NoError(t, runner.Run(context.Background(), program)) + assert.Empty(t, stderr.String()) + assert.Contains(t, stdout.String(), "Vacuuming done, freed") + assert.NoFileExists(t, archive) +} diff --git a/interp/register_builtins.go b/interp/register_builtins.go index 6d16ce2f..8d766ef5 100644 --- a/interp/register_builtins.go +++ b/interp/register_builtins.go @@ -24,6 +24,7 @@ import ( "github.com/DataDog/rshell/builtins/head" "github.com/DataDog/rshell/builtins/help" "github.com/DataDog/rshell/builtins/ip" + "github.com/DataDog/rshell/builtins/journalctl" "github.com/DataDog/rshell/builtins/logrotate" "github.com/DataDog/rshell/builtins/ls" "github.com/DataDog/rshell/builtins/ping" @@ -66,6 +67,7 @@ func registerBuiltins() { head.Cmd, help.Cmd, ip.Cmd, + journalctl.Cmd, logrotate.Cmd, ls.Cmd, ping.Cmd, diff --git a/interp/runner_exec.go b/interp/runner_exec.go index d49d52d4..d65d788e 100644 --- a/interp/runner_exec.go +++ b/interp/runner_exec.go @@ -662,6 +662,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { CommandAllowed: func(n string) bool { return r.allowAllCommands || r.allowedCommands[n] }, + AuthorizeSystemd: r.authorizeSystemd, AuthorizeSystemServices: r.authorizeSystemServices, AllowedPathsList: func() []builtins.AllowedPath { return allowedPathsList(r.sandbox) @@ -698,6 +699,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { // "exec read fails because read is a builtin, not an // executable on PATH" behaviour. Proc: r.proc, + Systemd: r.systemd, RemediationMode: r.remediationMode, } if r.remediationMode && r.sandbox != nil { @@ -791,6 +793,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { CommandAllowed: func(cmdName string) bool { return r.allowAllCommands || r.allowedCommands[cmdName] }, + AuthorizeSystemd: r.authorizeSystemd, AuthorizeSystemServices: r.authorizeSystemServices, AllowedPathsList: func() []builtins.AllowedPath { return allowedPathsList(r.sandbox) @@ -823,6 +826,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { return vr.Str, vr.IsSet() }, Proc: r.proc, + Systemd: r.systemd, RemediationMode: r.remediationMode, } if r.remediationMode && r.sandbox != nil { diff --git a/interp/system_services.go b/interp/system_services.go index aa66d254..88c70b99 100644 --- a/interp/system_services.go +++ b/interp/system_services.go @@ -13,40 +13,55 @@ import ( "github.com/DataDog/rshell/builtins" ) -// SystemServiceAction identifies an operation that may be granted for a -// system service. +// SystemdAction identifies an operation that may be granted for a systemd +// service. +type SystemdAction = builtins.SystemdAction + +const ( + SystemdActionRead = builtins.SystemdActionRead + SystemdActionClean = builtins.SystemdActionClean + SystemdActionReload = builtins.SystemdActionReload + SystemdActionRestart = builtins.SystemdActionRestart +) + +// SystemdOperation is one service action checked by the shared policy. +type SystemdOperation = builtins.SystemdOperation + +// Compatibility aliases for the original service-only policy. type SystemServiceAction = builtins.SystemServiceAction const ( SystemServiceRead = builtins.SystemServiceRead + SystemServiceClean = builtins.SystemServiceClean SystemServiceReload = builtins.SystemServiceReload SystemServiceRestart = builtins.SystemServiceRestart ) -// SystemServiceControlGrant grants Actions for one exact Service spelling. -// Service names are never normalized, expanded, or resolved as aliases. +// SystemServiceControlGrant grants Actions for one exact Service. type SystemServiceControlGrant struct { Service string Actions []SystemServiceAction } -type systemServiceGrants map[string]map[SystemServiceAction]struct{} +// SystemdControlGrant is an alias for the shared systemd policy grant type. +type SystemdControlGrant = SystemServiceControlGrant + +type systemdGrants map[string]map[SystemdAction]struct{} -// AllowedSystemServices configures the system services and actions that -// system-service builtins may use. A grant matches its Service exactly: for -// example, "mysql" and "mysql.service" are different service names. +// AllowedSystemServices configures the services and actions that systemd-aware +// builtins may use. Service names are matched exactly: for example, "mysql" +// and "mysql.service" are different services. // -// Grants without actions are ignored. Empty service names and names containing -// whitespace, control characters, path separators, or glob patterns are -// skipped with a warning. Supported actions are read, reload, and restart; -// unsupported actions are skipped with a warning. Duplicate services and -// actions are accepted and combined idempotently. +// Grants without actions are ignored. Invalid services and unsupported actions +// are skipped with a warning. Supported actions are read, clean, reload, and +// restart. Duplicate services and actions are accepted and combined +// idempotently. // -// When not set (default), or when passed an empty slice, every system service -// is denied. This policy is not bypassed by allowing all commands. +// When not set (default), or when passed an empty slice, every systemd +// operation is denied. This policy is not bypassed by allowing all commands. func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { return func(r *Runner) error { - allowed := make(systemServiceGrants, len(grants)) + allowed := make(systemdGrants, len(grants)) for i, grant := range grants { if len(grant.Actions) == 0 { continue @@ -59,13 +74,13 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { actions := allowed[grant.Service] for _, action := range grant.Actions { - if !validSystemServiceAction(action) { + if !validSystemdAction(action) { warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, grant.Service) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } if actions == nil { - actions = make(map[SystemServiceAction]struct{}, len(grant.Actions)) + actions = make(map[SystemdAction]struct{}, len(grant.Actions)) allowed[grant.Service] = actions } actions[action] = struct{}{} @@ -76,13 +91,11 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { } } -func validSystemServiceAction(action SystemServiceAction) bool { - switch action { - case SystemServiceRead, SystemServiceReload, SystemServiceRestart: - return true - default: - return false - } +func validSystemdAction(action SystemdAction) bool { + return action == SystemdActionRead || + action == SystemdActionClean || + action == SystemdActionReload || + action == SystemdActionRestart } func validateSystemServiceName(service string) error { @@ -92,6 +105,9 @@ func validateSystemServiceName(service string) error { if strings.ContainsRune(service, '/') || strings.ContainsRune(service, '\\') { return fmt.Errorf("system service name %q must not contain a path separator", service) } + if strings.ContainsRune(service, ':') { + return fmt.Errorf("system service name %q must not contain ':'", service) + } for _, r := range service { switch r { case '*', '?', '[', ']': @@ -104,25 +120,36 @@ func validateSystemServiceName(service string) error { return nil } -func (r *Runner) authorizeSystemServices(action SystemServiceAction, services ...string) error { - if !r.remediationMode { - return fmt.Errorf("system service actions require remediation mode") - } - if !validSystemServiceAction(action) { - return fmt.Errorf("unsupported system service action %q", action) - } - if len(services) == 0 { - return fmt.Errorf("at least one system service is required") +func (r *Runner) authorizeSystemd(operations ...SystemdOperation) error { + if len(operations) == 0 { + return fmt.Errorf("at least one systemd operation is required") } - for _, service := range services { - if err := validateSystemServiceName(service); err != nil { + for _, operation := range operations { + if err := validateSystemServiceName(operation.Service); err != nil { return err } - actions := r.allowedSystemServices[service] - if _, ok := actions[action]; !ok { - return fmt.Errorf("system service %q is not allowed for action %q", service, action) + if !validSystemdAction(operation.Action) { + return fmt.Errorf("unsupported systemd action %q for system service %q", operation.Action, operation.Service) + } + if operation.Action != SystemdActionRead && !r.remediationMode { + return fmt.Errorf("systemd action %q requires remediation mode", operation.Action) + } + actions := r.allowedSystemServices[operation.Service] + if _, ok := actions[operation.Action]; !ok { + return fmt.Errorf("system service %q is not allowed for action %q", operation.Service, operation.Action) } } return nil } + +func (r *Runner) authorizeSystemServices(action SystemServiceAction, services ...string) error { + if len(services) == 0 { + return fmt.Errorf("at least one system service is required") + } + operations := make([]SystemdOperation, len(services)) + for i, service := range services { + operations[i] = SystemdOperation{Service: service, Action: action} + } + return r.authorizeSystemd(operations...) +} diff --git a/interp/system_services_test.go b/interp/system_services_test.go index 2e5e1062..1cdea6b0 100644 --- a/interp/system_services_test.go +++ b/interp/system_services_test.go @@ -24,6 +24,7 @@ func TestAllowedSystemServicesAuthorizesExactServiceAndAction(t *testing.T) { SystemServiceRestart, SystemServiceReload, SystemServiceRead, + SystemServiceClean, }, }, { @@ -37,6 +38,7 @@ func TestAllowedSystemServicesAuthorizesExactServiceAndAction(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRestart, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceReload, "mysql.service")) + require.NoError(t, runner.authorizeSystemServices(SystemServiceClean, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service", "nginx.service")) err = runner.authorizeSystemServices(SystemServiceRestart, "nginx.service") @@ -60,19 +62,41 @@ func TestAllowedSystemServicesDefaultDenyIsIndependentOfAllowedCommands(t *testi assert.Contains(t, err.Error(), "not allowed") } -func TestAllowedSystemServicesRequiresRemediationMode(t *testing.T) { +func TestAllowedSystemServicesAllowsReadOutsideRemediationMode(t *testing.T) { runner, err := New(AllowedSystemServices([]SystemServiceControlGrant{ { Service: "mysql.service", - Actions: []SystemServiceAction{SystemServiceRead}, + Actions: []SystemServiceAction{SystemServiceRead, SystemServiceRestart, SystemServiceClean}, }, })) require.NoError(t, err) defer runner.Close() - err = runner.authorizeSystemServices(SystemServiceRead, "mysql.service") + require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) + err = runner.authorizeSystemServices(SystemServiceRestart, "mysql.service") + require.Error(t, err) + assert.Contains(t, err.Error(), `action "restart" requires remediation mode`) + + err = runner.authorizeSystemServices(SystemServiceClean, "mysql.service") + require.Error(t, err) + assert.Contains(t, err.Error(), `action "clean" requires remediation mode`) +} + +func TestAllowedSystemServicesReadDoesNotEnableMutation(t *testing.T) { + runner, err := New(AllowedSystemServices([]SystemdControlGrant{ + {Service: "systemd-journald.service", Actions: []SystemdAction{SystemdActionRead}}, + })) + require.NoError(t, err) + defer runner.Close() + + require.NoError(t, runner.authorizeSystemd( + SystemdOperation{Service: "systemd-journald.service", Action: SystemdActionRead}, + )) + err = runner.authorizeSystemd( + SystemdOperation{Service: "systemd-journald.service", Action: SystemdActionClean}, + ) require.Error(t, err) - assert.Contains(t, err.Error(), "require remediation mode") + assert.Contains(t, err.Error(), `action "clean" requires remediation mode`) } func TestAllowedSystemServicesCopiesAndCombinesGrants(t *testing.T) { @@ -107,6 +131,7 @@ func TestAllowedSystemServicesSkipsEmptyAndInvalidGrants(t *testing.T) { {Service: "/etc/systemd/system/mysql.service", Actions: []SystemServiceAction{SystemServiceRead}}, {Service: "C:\\svc", Actions: []SystemServiceAction{SystemServiceRead}}, {Service: "..\\svc", Actions: []SystemServiceAction{SystemServiceRead}}, + {Service: "tenant:mysql.service", Actions: []SystemServiceAction{SystemServiceRead}}, {Service: "mysql*.service", Actions: []SystemServiceAction{SystemServiceRead}}, }), // Applying AllowedPaths after AllowedSystemServices verifies that one @@ -121,13 +146,14 @@ func TestAllowedSystemServicesSkipsEmptyAndInvalidGrants(t *testing.T) { assert.NotContains(t, runner.allowedSystemServices, "ignored.service") warnings := runner.Warnings() - require.Len(t, warnings, 8) + require.Len(t, warnings, 9) for _, needle := range []string{ "AllowedSystemServices: skipping grant 2: system service name must not be empty", "whitespace or control characters", `AllowedSystemServices: skipping grant 6: system service name "C:\\svc" must not contain a path separator`, `AllowedSystemServices: skipping grant 7: system service name "..\\svc" must not contain a path separator`, "path separator", + "must not contain ':'", "glob pattern", "AllowedPaths: skipping", } { @@ -175,8 +201,9 @@ func TestAuthorizeSystemServicesRejectsInvalidRequests(t *testing.T) { services []string needle string }{ - {name: "unknown action", action: "stop", services: []string{"mysql.service"}, needle: "unsupported system service action"}, + {name: "unknown action", action: "stop", services: []string{"mysql.service"}, needle: "unsupported systemd action"}, {name: "no services", action: SystemServiceRead, needle: "at least one system service"}, + {name: "runtime resource separator", action: SystemServiceRead, services: []string{"tenant:mysql.service"}, needle: "must not contain ':'"}, {name: "runtime glob", action: SystemServiceRead, services: []string{"mysql*.service"}, needle: "glob pattern"}, {name: "runtime backslash path", action: SystemServiceRead, services: []string{"..\\mysql.service"}, needle: "path separator"}, } diff --git a/interp/systemd_target.go b/interp/systemd_target.go new file mode 100644 index 00000000..a12abad7 --- /dev/null +++ b/interp/systemd_target.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package interp + +import ( + "fmt" + + internalsystemd "github.com/DataDog/rshell/internal/systemd" +) + +// SystemdTargetConfig selects the systemd host used by systemd-aware +// builtins. The zero value uses standard local paths. Root derives all paths +// below one mounted host root and cannot be mixed with explicit fields. When +// explicit fields are used, omitted fields stay unavailable and never fall +// back to local paths. +type SystemdTargetConfig struct { + Root string + JournalDirs []string + MachineIDPath string + JournalControlSocket string + SystemBusSocket string +} + +// WithSystemdTarget configures the trusted systemd target. Scripts cannot +// override these paths. +func WithSystemdTarget(config SystemdTargetConfig) RunnerOption { + return func(r *Runner) error { + target, err := internalsystemd.ResolveTarget(internalsystemd.Target{ + Root: config.Root, + JournalDirs: append([]string(nil), config.JournalDirs...), + MachineIDPath: config.MachineIDPath, + JournalControlSocket: config.JournalControlSocket, + SystemBusSocket: config.SystemBusSocket, + }) + if err != nil { + return fmt.Errorf("WithSystemdTarget: %w", err) + } + r.systemdTarget = target + r.systemdTargetConfigured = true + return nil + } +} diff --git a/interp/systemd_target_test.go b/interp/systemd_target_test.go new file mode 100644 index 00000000..608f684d --- /dev/null +++ b/interp/systemd_target_test.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !windows + +package interp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithSystemdTargetDefaultsToLocal(t *testing.T) { + runner, err := New() + require.NoError(t, err) + defer runner.Close() + + assert.Equal(t, []string{"/var/log/journal", "/run/log/journal"}, runner.systemdTarget.JournalDirs) + assert.Equal(t, "/etc/machine-id", runner.systemdTarget.MachineIDPath) +} + +func TestWithSystemdTargetCopiesConfiguration(t *testing.T) { + dirs := []string{"/host/var/log/journal"} + runner, err := New(WithSystemdTarget(SystemdTargetConfig{ + JournalDirs: dirs, + MachineIDPath: "/host/etc/machine-id", + })) + require.NoError(t, err) + defer runner.Close() + + dirs[0] = "/changed" + assert.Equal(t, []string{"/host/var/log/journal"}, runner.systemdTarget.JournalDirs) + assert.Empty(t, runner.systemdTarget.SystemBusSocket) +} + +func TestWithSystemdTargetRejectsMixedRootAndExplicitPaths(t *testing.T) { + runner, err := New(WithSystemdTarget(SystemdTargetConfig{ + Root: "/host", + JournalDirs: []string{"/host/var/log/journal"}, + })) + if runner != nil { + runner.Close() + } + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be combined") +} diff --git a/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml b/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml new file mode 100644 index 00000000..bf56373e --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml @@ -0,0 +1,11 @@ +# Scenario runners do not grant systemd capabilities by default. +skip_assert_against_bash: true +description: journalctl disk usage enforces the journald service allowlist +input: + script: |+ + journalctl --disk-usage +expect: + stdout: |+ + stderr: |+ + journalctl: system service "systemd-journald.service" is not allowed for action "read" + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml b/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml new file mode 100644 index 00000000..34d462d7 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml @@ -0,0 +1,11 @@ +# Scenario runners do not grant systemd capabilities by default. +skip_assert_against_bash: true +description: journalctl enforces the shared systemd allowlist +input: + script: |+ + journalctl -u api.service +expect: + stdout: |+ + stderr: |+ + journalctl: system service "api.service" is not allowed for action "read" + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/missing_scope.yaml b/tests/scenarios/cmd/journalctl/errors/missing_scope.yaml new file mode 100644 index 00000000..e8b57969 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/missing_scope.yaml @@ -0,0 +1,11 @@ +# Bare journal reads are intentionally unavailable in rshell. +skip_assert_against_bash: true +description: journalctl requires an exact unit or kernel scope +input: + script: |+ + journalctl +expect: + stdout: |+ + stderr: |+ + journalctl: an exact --unit or --dmesg scope is required + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/raw_match.yaml b/tests/scenarios/cmd/journalctl/errors/raw_match.yaml new file mode 100644 index 00000000..c11b693a --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/raw_match.yaml @@ -0,0 +1,11 @@ +# Raw fields could bypass the resource capability model and are rejected. +skip_assert_against_bash: true +description: journalctl rejects arbitrary field matches +input: + script: |+ + journalctl -u api.service _PID=1 +expect: + stdout: |+ + stderr: |+ + journalctl: unexpected argument "_PID=1"; arbitrary journal matches are not supported + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml b/tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml new file mode 100644 index 00000000..a11d7083 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml @@ -0,0 +1,11 @@ +# Rotation mutates active journals and is unavailable outside remediation mode. +skip_assert_against_bash: true +description: journalctl rotation requires remediation mode +input: + script: |+ + journalctl --rotate +expect: + stdout: |+ + stderr: |+ + journalctl: systemd action "clean" requires remediation mode + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml new file mode 100644 index 00000000..8b1d784b --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml @@ -0,0 +1,12 @@ +# Remediation mode alone does not bypass the shared systemd allowlist. +skip_assert_against_bash: true +description: journalctl vacuum requires a journald clean grant +input: + mode: remediation + script: |+ + journalctl --vacuum-size=64M --vacuum-time=168h +expect: + stdout: |+ + stderr: |+ + journalctl: system service "systemd-journald.service" is not allowed for action "clean" + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml new file mode 100644 index 00000000..480420ba --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml @@ -0,0 +1,11 @@ +# Cleanup is intentionally unavailable outside remediation mode, even in dry-run mode. +skip_assert_against_bash: true +description: journalctl vacuum requires remediation mode +input: + script: |+ + journalctl --vacuum-time=168h --dry-run +expect: + stdout: |+ + stderr: |+ + journalctl: systemd action "clean" requires remediation mode + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml new file mode 100644 index 00000000..5db472d2 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml @@ -0,0 +1,13 @@ +# Size-only cleanup could delete newly rotated archives, so rshell requires an +# explicit retention floor before checking mutation authorization. +skip_assert_against_bash: true +description: journalctl size vacuum requires a time retention floor +input: + mode: remediation + script: |+ + journalctl --vacuum-size=64M +expect: + stdout: |+ + stderr: |+ + journalctl: --vacuum-size requires --vacuum-time to protect recently modified archives + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/help/help.yaml b/tests/scenarios/cmd/journalctl/help/help.yaml new file mode 100644 index 00000000..7c74a3c6 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/help/help.yaml @@ -0,0 +1,15 @@ +# rshell journalctl is a restricted builtin and may not be installed in the Bash image. +skip_assert_against_bash: true +description: journalctl help describes the bounded query surface +input: + script: |+ + journalctl --help +expect: + stdout_contains: + - "Usage: journalctl (-u UNIT...|-k) [OPTION]..." + - "--unit" + - "--dmesg" + - "--lines" + - "--rotate" + stderr: |+ + exit_code: 0