From 0c1c5638c115fcf2633099f56790398fcb9ed1a9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Mon, 13 Jul 2026 11:30:03 -0400 Subject: [PATCH 01/53] feat: add system service allowlist policy --- builtins/builtins.go | 10 +++- builtins/remediation_test.go | 54 +++++++++++++++++++ interp/builtin_truncate_pentest_test.go | 3 +- .../cmd/logrotate/errors/readonly_mode.yaml | 2 +- .../cmd/truncate/errors/readonly_mode.yaml | 4 +- .../truncate/errors/readonly_mode_help.yaml | 4 +- 6 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 builtins/remediation_test.go diff --git a/builtins/builtins.go b/builtins/builtins.go index 2cee22b9..08acbfd5 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -82,8 +82,8 @@ type Command struct { NormalizeArgs func(args []string) []string // RemediationOnly marks a builtin as only available in remediation mode. - // The help builtin uses this to move the command to the disabled list - // when the shell is in read-only mode. + // Register rejects the command before argument parsing in read-only mode; + // the help builtin also uses this to move the command to the disabled list. RemediationOnly bool } @@ -105,6 +105,7 @@ func (c Command) Register() { name := c.Name factory := c.MakeFlags normalize := c.NormalizeArgs + remediationOnly := c.RemediationOnly if _, exists := featureByName[name]; exists { panic("builtin name conflicts with rshell feature: " + name) } @@ -118,6 +119,11 @@ func (c Command) Register() { metaRegistry[name] = CommandMeta{Name: name, Description: c.Description, Help: c.Help, HasFlags: hasFlags, RemediationOnly: c.RemediationOnly} addToRegistry(name, func(ctx context.Context, callCtx *CallContext, args []string) Result { + if remediationOnly && !callCtx.RemediationMode { + callCtx.Errf("%s: command requires remediation mode\n", name) + return Result{Code: 1} + } + fs := pflag.NewFlagSet(name, pflag.ContinueOnError) fs.SetOutput(io.Discard) // handler formats errors itself handler := factory(fs) diff --git a/builtins/remediation_test.go b/builtins/remediation_test.go new file mode 100644 index 00000000..60431f35 --- /dev/null +++ b/builtins/remediation_test.go @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package builtins + +import ( + "bytes" + "context" + "testing" +) + +func TestCommandRegisterEnforcesRemediationOnly(t *testing.T) { + const name = "test-remediation-only" + defer delete(registry, name) + defer delete(metaRegistry, name) + + called := false + Command{ + Name: name, + Description: "test remediation enforcement", + RemediationOnly: true, + MakeFlags: NoFlags(func(context.Context, *CallContext, []string) Result { + called = true + return Result{} + }), + }.Register() + + handler, ok := Lookup(name) + if !ok { + t.Fatal("registered command was not found") + } + + var stderr bytes.Buffer + result := handler(context.Background(), &CallContext{Stderr: &stderr}, nil) + if result.Code != 1 { + t.Fatalf("read-only result code = %d, want 1", result.Code) + } + if called { + t.Fatal("remediation-only handler ran in read-only mode") + } + if got, want := stderr.String(), name+": command requires remediation mode\n"; got != want { + t.Fatalf("stderr = %q, want %q", got, want) + } + + result = handler(context.Background(), &CallContext{Stderr: &stderr, RemediationMode: true}, nil) + if result.Code != 0 { + t.Fatalf("remediation result code = %d, want 0", result.Code) + } + if !called { + t.Fatal("remediation-only handler did not run in remediation mode") + } +} diff --git a/interp/builtin_truncate_pentest_test.go b/interp/builtin_truncate_pentest_test.go index e601cdda..4f277836 100644 --- a/interp/builtin_truncate_pentest_test.go +++ b/interp/builtin_truncate_pentest_test.go @@ -45,7 +45,8 @@ func TestTruncatePentestSandboxReadOnly(t *testing.T) { _, stderr, code := runScript(t, "truncate -s 0 f.txt", dir, interp.AllowedPaths([]string{dir}), // Deliberately omit interp.WithMode(interp.ModeRemediation) — - // this means callCtx.Truncate is nil, which is the mode-guard path. + // central remediation-only dispatch must reject the command before its + // handler can reach the nil Truncate capability. ) assert.Equal(t, 1, code) assert.Contains(t, stderr, "truncate") diff --git a/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml b/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml index 31c0ffcf..f177bcef 100644 --- a/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml +++ b/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml @@ -12,5 +12,5 @@ input: expect: stdout: |+ stderr: |+ - logrotate: filesystem capability not available (remediation mode required) + logrotate: command requires remediation mode exit_code: 1 diff --git a/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml b/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml index b85bf000..ca12439e 100644 --- a/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml +++ b/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml @@ -1,5 +1,5 @@ # Security test: truncate in read-only mode (no remediation) is rejected. -description: truncate without remediation mode fails with "capability not available". +description: truncate without remediation mode is rejected before its handler runs. setup: files: - path: f.txt @@ -11,6 +11,6 @@ input: expect: stdout: |+ stderr: |+ - truncate: filesystem capability not available (remediation mode required) + truncate: command requires remediation mode exit_code: 1 skip_assert_against_bash: true diff --git a/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml b/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml index d03a8e9b..b1dc61cb 100644 --- a/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml +++ b/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml @@ -1,4 +1,4 @@ -description: truncate --help in read-only mode fails with the same error as truncate itself. +description: truncate --help in read-only mode is rejected before help is displayed. input: allowed_paths: ["$DIR"] script: |+ @@ -6,6 +6,6 @@ input: expect: stdout: |+ stderr: |+ - truncate: filesystem capability not available (remediation mode required) + truncate: command requires remediation mode exit_code: 1 skip_assert_against_bash: true From 2872a6dae364c2078f558f05ec3ebfd8f02ee9a9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 14:13:54 -0400 Subject: [PATCH 02/53] fix: remove redundant remediation dispatch gate --- builtins/builtins.go | 10 +--- builtins/remediation_test.go | 54 ------------------- interp/builtin_truncate_pentest_test.go | 3 +- interp/runner_exec.go | 3 +- .../cmd/logrotate/errors/readonly_mode.yaml | 2 +- .../cmd/truncate/errors/readonly_mode.yaml | 4 +- .../truncate/errors/readonly_mode_help.yaml | 4 +- 7 files changed, 9 insertions(+), 71 deletions(-) delete mode 100644 builtins/remediation_test.go diff --git a/builtins/builtins.go b/builtins/builtins.go index 08acbfd5..2cee22b9 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -82,8 +82,8 @@ type Command struct { NormalizeArgs func(args []string) []string // RemediationOnly marks a builtin as only available in remediation mode. - // Register rejects the command before argument parsing in read-only mode; - // the help builtin also uses this to move the command to the disabled list. + // The help builtin uses this to move the command to the disabled list + // when the shell is in read-only mode. RemediationOnly bool } @@ -105,7 +105,6 @@ func (c Command) Register() { name := c.Name factory := c.MakeFlags normalize := c.NormalizeArgs - remediationOnly := c.RemediationOnly if _, exists := featureByName[name]; exists { panic("builtin name conflicts with rshell feature: " + name) } @@ -119,11 +118,6 @@ func (c Command) Register() { metaRegistry[name] = CommandMeta{Name: name, Description: c.Description, Help: c.Help, HasFlags: hasFlags, RemediationOnly: c.RemediationOnly} addToRegistry(name, func(ctx context.Context, callCtx *CallContext, args []string) Result { - if remediationOnly && !callCtx.RemediationMode { - callCtx.Errf("%s: command requires remediation mode\n", name) - return Result{Code: 1} - } - fs := pflag.NewFlagSet(name, pflag.ContinueOnError) fs.SetOutput(io.Discard) // handler formats errors itself handler := factory(fs) diff --git a/builtins/remediation_test.go b/builtins/remediation_test.go deleted file mode 100644 index 60431f35..00000000 --- a/builtins/remediation_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -package builtins - -import ( - "bytes" - "context" - "testing" -) - -func TestCommandRegisterEnforcesRemediationOnly(t *testing.T) { - const name = "test-remediation-only" - defer delete(registry, name) - defer delete(metaRegistry, name) - - called := false - Command{ - Name: name, - Description: "test remediation enforcement", - RemediationOnly: true, - MakeFlags: NoFlags(func(context.Context, *CallContext, []string) Result { - called = true - return Result{} - }), - }.Register() - - handler, ok := Lookup(name) - if !ok { - t.Fatal("registered command was not found") - } - - var stderr bytes.Buffer - result := handler(context.Background(), &CallContext{Stderr: &stderr}, nil) - if result.Code != 1 { - t.Fatalf("read-only result code = %d, want 1", result.Code) - } - if called { - t.Fatal("remediation-only handler ran in read-only mode") - } - if got, want := stderr.String(), name+": command requires remediation mode\n"; got != want { - t.Fatalf("stderr = %q, want %q", got, want) - } - - result = handler(context.Background(), &CallContext{Stderr: &stderr, RemediationMode: true}, nil) - if result.Code != 0 { - t.Fatalf("remediation result code = %d, want 0", result.Code) - } - if !called { - t.Fatal("remediation-only handler did not run in remediation mode") - } -} diff --git a/interp/builtin_truncate_pentest_test.go b/interp/builtin_truncate_pentest_test.go index 4f277836..e601cdda 100644 --- a/interp/builtin_truncate_pentest_test.go +++ b/interp/builtin_truncate_pentest_test.go @@ -45,8 +45,7 @@ func TestTruncatePentestSandboxReadOnly(t *testing.T) { _, stderr, code := runScript(t, "truncate -s 0 f.txt", dir, interp.AllowedPaths([]string{dir}), // Deliberately omit interp.WithMode(interp.ModeRemediation) — - // central remediation-only dispatch must reject the command before its - // handler can reach the nil Truncate capability. + // this means callCtx.Truncate is nil, which is the mode-guard path. ) assert.Equal(t, 1, code) assert.Contains(t, stderr, "truncate") diff --git a/interp/runner_exec.go b/interp/runner_exec.go index d49d52d4..9e67ff64 100644 --- a/interp/runner_exec.go +++ b/interp/runner_exec.go @@ -697,8 +697,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { // available", which is the closest analogue to bash's // "exec read fails because read is a builtin, not an // executable on PATH" behaviour. - Proc: r.proc, - RemediationMode: r.remediationMode, + Proc: r.proc, } if r.remediationMode && r.sandbox != nil { child.Truncate = func(ctx context.Context, path string, size int64, create bool) error { diff --git a/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml b/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml index f177bcef..31c0ffcf 100644 --- a/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml +++ b/tests/scenarios/cmd/logrotate/errors/readonly_mode.yaml @@ -12,5 +12,5 @@ input: expect: stdout: |+ stderr: |+ - logrotate: command requires remediation mode + logrotate: filesystem capability not available (remediation mode required) exit_code: 1 diff --git a/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml b/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml index ca12439e..b85bf000 100644 --- a/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml +++ b/tests/scenarios/cmd/truncate/errors/readonly_mode.yaml @@ -1,5 +1,5 @@ # Security test: truncate in read-only mode (no remediation) is rejected. -description: truncate without remediation mode is rejected before its handler runs. +description: truncate without remediation mode fails with "capability not available". setup: files: - path: f.txt @@ -11,6 +11,6 @@ input: expect: stdout: |+ stderr: |+ - truncate: command requires remediation mode + truncate: filesystem capability not available (remediation mode required) exit_code: 1 skip_assert_against_bash: true diff --git a/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml b/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml index b1dc61cb..d03a8e9b 100644 --- a/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml +++ b/tests/scenarios/cmd/truncate/errors/readonly_mode_help.yaml @@ -1,4 +1,4 @@ -description: truncate --help in read-only mode is rejected before help is displayed. +description: truncate --help in read-only mode fails with the same error as truncate itself. input: allowed_paths: ["$DIR"] script: |+ @@ -6,6 +6,6 @@ input: expect: stdout: |+ stderr: |+ - truncate: command requires remediation mode + truncate: filesystem capability not available (remediation mode required) exit_code: 1 skip_assert_against_bash: true From ed14447aa0f2987d50f916b63b3704a7e346fdaa Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 12:40:32 -0400 Subject: [PATCH 03/53] feat(systemd): generalize capability policy --- README.md | 22 +++--- SHELL_FEATURES.md | 2 +- builtins/builtins.go | 62 +++++++++++++--- cmd/rshell/main.go | 74 +++++++++++++----- cmd/rshell/main_test.go | 64 +++++++++------- interp/api.go | 8 +- interp/runner_exec.go | 2 + interp/system_services.go | 132 ++++++++++++++++++++++++++------- interp/system_services_test.go | 90 ++++++++++++++++++++-- 9 files changed, 355 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 0a2c197b..a3f64a5c 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Every access path is default-deny: | Resource | Default | Opt-in | |----------------------|-------------------------------------|----------------------------------------------| | Command execution | All commands blocked (exit code 127)| `AllowedCommands` with namespaced command list (e.g. `rshell:cat`) | -| System services | All services and actions blocked | `AllowedSystemServices` with exact service/action grants | +| Systemd resources | All resources and actions blocked | `AllowedSystemd` with exact resource/action grants | | External commands | Blocked (exit code 127) | Provide an `ExecHandler` | | Filesystem access | Blocked | Configure `AllowedPaths` with `PATH[:ro|:rw]` root specs | | Environment variables| Empty (no host env inherited) | Pass variables via the `Env` option | @@ -69,22 +69,26 @@ Every access path is default-deny: **AllowedCommands** restricts which commands (builtins or external) the interpreter may execute. Commands must be specified with the `rshell:` namespace prefix (e.g. `rshell:cat`, `rshell:echo`). If not set, no commands are allowed. -**AllowedSystemServices** configures exact service/action grants for future system-service builtins. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different grants. The supported actions are `read`, `reload`, and `restart`. Grants without actions are ignored; entries with empty, whitespace-containing, path-like, or glob-pattern service names and unsupported actions are skipped with a warning. The policy defaults to denying every service, remains enforced when all commands are allowed, and requires remediation mode before any grant can be authorized. +**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact resource (`unit:NAME`, `journal:all`, `journal:kernel`, `journal:storage`, or `manager`) with generic actions (`read`, `clean`, `reload`, or `restart`). Invalid resource/action combinations are rejected. Unit names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `unit:mysql` and `unit:mysql.service` are different resources. Empty unit names, whitespace, path-like names, and glob patterns are rejected. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go -interp.AllowedSystemServices([]interp.SystemServiceControlGrant{ +interp.AllowedSystemd([]interp.SystemdControlGrant{ { - Service: "mysql.service", - Actions: []interp.SystemServiceAction{ - interp.SystemServiceRestart, - interp.SystemServiceReload, - interp.SystemServiceRead, + Resource: interp.SystemdUnitResource("mysql.service"), + Actions: []interp.SystemdAction{ + interp.SystemdActionRestart, + interp.SystemdActionReload, + interp.SystemdActionRead, }, }, + { + Resource: interp.SystemdResourceJournalStorage, + Actions: []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionClean}, + }, }) ``` -The development CLI accepts the equivalent syntax through `--allowed-services mysql.service:restart+reload+read,nginx.service:read`. The policy and authorization capability are implemented, but `systemctl` and `journalctl` builtins are not yet available. +The development CLI accepts equivalent grants through `--allowed-systemd unit:mysql.service:restart+reload+read,journal:storage:read+clean`. The older `AllowedSystemServices` API and `--allowed-services` flag remain as unit-only compatibility shorthands backed by the same allowlist. The policy and authorization capability are implemented, but `systemctl` and `journalctl` builtins are not yet available. **AllowedPaths** restricts all file operations to specified directories using Go's `os.Root` API for reads and openat-based write handling for writes. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 3833ae17..2430ebf0 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -121,7 +121,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c ## Execution - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed -- ✅ AllowedSystemServices policy — configures exact, case-sensitive `read`, `reload`, and `restart` grants for system service names through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]`; names are not normalized (`mysql` does not match `mysql.service`), grants without actions are ignored, invalid names and unsupported actions are skipped with warnings, all services are denied by default, and allowing all commands does not bypass this policy; the authorization capability is implemented for future consumers, but `systemctl` and `journalctl` builtins are not yet available +- ✅ AllowedSystemServices policy — one shared default-deny capability map for resources with generic `read`, `clean`, `reload`, and `restart` actions; invalid resource/action pairs are rejected, unit names are case-sensitive and are not normalized, `read` works in read-only mode, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure through `interp.AllowedSystemd` or CLI `--allowed-systemd RESOURCE:ACTION[+ACTION...]`; `AllowedSystemServices` and `--allowed-services` remain unit-only compatibility shorthands - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command - ✅ ProcPath — overrides the proc filesystem path used by `ps` (default `/proc`; Linux-only; useful for testing/container environments); `ps` does not read `/proc//cmdline` diff --git a/builtins/builtins.go b/builtins/builtins.go index 2cee22b9..a73b096d 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -57,14 +57,51 @@ type AllowedPath struct { Access AllowedPathAccess } -// SystemServiceAction identifies an operation that a builtin may perform on -// an explicitly configured system service. -type SystemServiceAction string +// SystemdAction identifies an operation that a builtin may perform on an +// explicitly configured systemd resource. +type SystemdAction string const ( - SystemServiceRead SystemServiceAction = "read" - SystemServiceReload SystemServiceAction = "reload" - SystemServiceRestart SystemServiceAction = "restart" + SystemdActionRead SystemdAction = "read" + SystemdActionClean SystemdAction = "clean" + SystemdActionReload SystemdAction = "reload" + SystemdActionRestart SystemdAction = "restart" +) + +// SystemdResource identifies an exact resource in the shared systemd +// capability policy. Unit resources use the "unit:" prefix. Journal and +// manager resources are fixed constants so builtins cannot authorize +// arbitrary filesystem paths or D-Bus objects. +type SystemdResource string + +const ( + SystemdResourceJournalAll SystemdResource = "journal:all" + SystemdResourceJournalKernel SystemdResource = "journal:kernel" + SystemdResourceJournalStorage SystemdResource = "journal:storage" + SystemdResourceManager SystemdResource = "manager" +) + +// SystemdUnitResource returns the policy resource for one exact unit name. +// The interpreter validates the name before accepting or authorizing it. +func SystemdUnitResource(name string) SystemdResource { + return SystemdResource("unit:" + name) +} + +// SystemdOperation is one resource/action pair that must be authorized before +// a builtin interacts with systemd. +type SystemdOperation struct { + Resource SystemdResource + Action SystemdAction +} + +// Deprecated compatibility aliases for the service-only policy introduced +// before the shared systemd capability model. +type SystemServiceAction = SystemdAction + +const ( + SystemServiceRead = SystemdActionRead + SystemServiceReload = SystemdActionReload + SystemServiceRestart = SystemdActionRestart ) // Command pairs a builtin name with its flag-declaring factory. MakeFlags @@ -255,10 +292,15 @@ type CallContext struct { // commands. CommandAllowed func(name string) bool - // AuthorizeSystemServices reports whether all named services may be used - // for action under the current shell policy. Implementations must authorize - // the complete list before a builtin performs any operation so multi-service - // requests cannot partially execute. Service names are matched exactly. + // AuthorizeSystemd reports whether every operation may be performed under + // the current shell policy. Implementations must authorize the complete + // list before a builtin performs any operation so compound requests cannot + // partially execute. + AuthorizeSystemd func(operations ...SystemdOperation) error + + // AuthorizeSystemServices is the deprecated unit-only authorization + // capability retained for compatibility with callers built against the + // original service allowlist API. AuthorizeSystemServices func(action SystemServiceAction, services ...string) error // AllowedPathsList returns the resolved absolute paths and configured diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index e67830ca..3916be2e 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -40,6 +40,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowedPaths string allowedCommands string allowedServices string + allowedSystemd string allowAllCmds bool timeout time.Duration procPath string @@ -85,7 +86,14 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmds = strings.Split(allowedCommands, ",") } - serviceGrants := parseAllowedServices(allowedServices) + serviceGrants, err := parseAllowedServices(allowedServices) + if err != nil { + return err + } + systemdGrants, err := parseAllowedSystemd(allowedSystemd) + if err != nil { + return err + } parsedMode := interp.Mode(mode) if parsedMode != interp.ModeReadOnly && parsedMode != interp.ModeRemediation { @@ -96,6 +104,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowedPaths: paths, allowedCommands: cmds, allowedServices: serviceGrants, + allowedSystemd: systemdGrants, allowAllCommands: allowAllCmds, procPath: procPath, mode: parsedMode, @@ -148,7 +157,8 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().MarkHidden("command") //nolint:errcheck // flag is guaranteed to exist cmd.Flags().StringVarP(&allowedPaths, "allowed-paths", "p", "", "comma-separated list of PATH[:ro|:rw] directories the shell is allowed to access; entries without a suffix are read-only") cmd.Flags().StringVar(&allowedCommands, "allowed-commands", "", "comma-separated list of namespaced commands (e.g. rshell:cat,rshell:find)") - cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated exact service grants in SERVICE:ACTION[+ACTION...] form; actions: read, reload, restart") + cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated exact unit grants in SERVICE:ACTION[+ACTION...] form; shorthand for unit: resources") + cmd.Flags().StringVar(&allowedSystemd, "allowed-systemd", "", "comma-separated systemd grants in RESOURCE:ACTION[+ACTION...] form; resources: unit:NAME, journal:all, journal:kernel, journal:storage, manager") cmd.Flags().BoolVar(&allowAllCmds, "allow-all-commands", false, "allow execution of all commands (builtins and external)") cmd.Flags().DurationVar(&timeout, "timeout", 0, "maximum execution time for the entire shell run (e.g. 100ms, 5s, 1m)") cmd.Flags().StringVar(&procPath, "proc-path", "", "path to the proc filesystem used by ps (default \"/proc\")") @@ -221,6 +231,7 @@ type executeOpts struct { allowedPaths []string allowedCommands []string allowedServices []interp.SystemServiceControlGrant + allowedSystemd []interp.SystemdControlGrant allowAllCommands bool procPath string mode interp.Mode @@ -247,8 +258,15 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i } else if len(opts.allowedCommands) > 0 { runOpts = append(runOpts, interp.AllowedCommands(opts.allowedCommands)) } - if len(opts.allowedServices) > 0 { - runOpts = append(runOpts, interp.AllowedSystemServices(opts.allowedServices)) + if len(opts.allowedServices) > 0 || len(opts.allowedSystemd) > 0 { + grants := append([]interp.SystemdControlGrant(nil), opts.allowedSystemd...) + for _, grant := range opts.allowedServices { + grants = append(grants, interp.SystemdControlGrant{ + Resource: interp.SystemdUnitResource(grant.Service), + Actions: append([]interp.SystemdAction(nil), grant.Actions...), + }) + } + runOpts = append(runOpts, interp.AllowedSystemd(grants)) } if opts.procPath != "" { runOpts = append(runOpts, interp.ProcPath(opts.procPath)) @@ -266,32 +284,54 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i return runner.Run(ctx, prog) } -func parseAllowedServices(value string) []interp.SystemServiceControlGrant { +func parseAllowedServices(value string) ([]interp.SystemServiceControlGrant, error) { if value == "" { - return nil + return nil, nil } entries := strings.Split(value, ",") grants := make([]interp.SystemServiceControlGrant, 0, len(entries)) for _, entry := range entries { separator := strings.LastIndexByte(entry, ':') - if separator < 0 { - grants = append(grants, interp.SystemServiceControlGrant{Service: entry}) - continue + if separator <= 0 || separator == len(entry)-1 { + return nil, fmt.Errorf("--allowed-services: invalid grant %q (expected SERVICE:ACTION[+ACTION...])", entry) } - var actions []interp.SystemServiceAction - if separator < len(entry)-1 { - actionNames := strings.Split(entry[separator+1:], "+") - actions = make([]interp.SystemServiceAction, len(actionNames)) - for i, action := range actionNames { - actions[i] = interp.SystemServiceAction(action) - } + actionNames := strings.Split(entry[separator+1:], "+") + actions := make([]interp.SystemServiceAction, len(actionNames)) + for i, action := range actionNames { + actions[i] = interp.SystemServiceAction(action) } grants = append(grants, interp.SystemServiceControlGrant{ Service: entry[:separator], Actions: actions, }) } - return grants + return grants, nil +} + +func parseAllowedSystemd(value string) ([]interp.SystemdControlGrant, error) { + if value == "" { + return nil, nil + } + + entries := strings.Split(value, ",") + grants := make([]interp.SystemdControlGrant, 0, len(entries)) + for _, entry := range entries { + separator := strings.LastIndexByte(entry, ':') + if separator <= 0 || separator == len(entry)-1 { + return nil, fmt.Errorf("--allowed-systemd: invalid grant %q (expected RESOURCE:ACTION[+ACTION...])", entry) + } + + actionNames := strings.Split(entry[separator+1:], "+") + actions := make([]interp.SystemdAction, len(actionNames)) + for i, action := range actionNames { + actions[i] = interp.SystemdAction(action) + } + grants = append(grants, interp.SystemdControlGrant{ + Resource: interp.SystemdResource(entry[:separator]), + Actions: actions, + }) + } + return grants, nil } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index 03861a7a..23a7b439 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -181,6 +181,8 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "--allowed-commands") assert.Contains(t, stdout, "--allowed-services") assert.Contains(t, stdout, "SERVICE:ACTION[+ACTION...]") + assert.Contains(t, stdout, "--allowed-systemd") + assert.Contains(t, stdout, "RESOURCE:ACTION[+ACTION...]") assert.Contains(t, stdout, "--allow-all-commands") assert.Contains(t, stdout, "file-target output redirections within :rw AllowedPaths roots") assert.Contains(t, stdout, "--timeout") @@ -298,15 +300,14 @@ func TestAllowedServicesFlag(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedServicesFlagSkipsGrantsWithoutActions(t *testing.T) { - code, stdout, stderr := runCLI(t, +func TestAllowedServicesFlagRejectsInvalidGrant(t *testing.T) { + code, _, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "mysql.service,redis.service:,,nginx.service:stop", + "--allowed-services", "mysql.service", "-c", `echo hello`, ) - assert.Equal(t, 0, code) - assert.Equal(t, "hello\n", stdout) - assert.Equal(t, "AllowedSystemServices: skipping unsupported action \"stop\" in grant 3 for \"nginx.service\"\n", stderr) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, "invalid grant") } func TestAllowedServicesFlagWarnsAndSkipsUnknownAction(t *testing.T) { @@ -320,45 +321,54 @@ func TestAllowedServicesFlagWarnsAndSkipsUnknownAction(t *testing.T) { assert.Contains(t, stderr, `skipping unsupported action "stop"`) } -func TestAllowedServicesFlagWarnsAndSkipsEmptyAction(t *testing.T) { +func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "mysql.service:read++reload", + "--allowed-services", "mysql*.service:read", "-c", `echo hello`, ) assert.Equal(t, 0, code) assert.Equal(t, "hello\n", stdout) - assert.Equal(t, "AllowedSystemServices: skipping unsupported action \"\" in grant 0 for \"mysql.service\"\n", stderr) + assert.Contains(t, stderr, "AllowedSystemServices: skipping") + assert.Contains(t, stderr, "glob pattern") } -func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { +func TestParseAllowedServicesUsesLastColon(t *testing.T) { + grants, err := parseAllowedServices("tenant:mysql.service:read+reload") + require.NoError(t, err) + require.Len(t, grants, 1) + assert.Equal(t, "tenant:mysql.service", grants[0].Service) + assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemServiceReload}, grants[0].Actions) +} + +func TestAllowedSystemdFlag(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", ":read,mysql*.service:read,nginx.service:read", + "--allowed-systemd", "unit:mysql.service:read+restart,journal:kernel:read,journal:storage:read+clean,manager:reload", + "--mode", "remediation", "-c", `echo hello`, ) assert.Equal(t, 0, code) assert.Equal(t, "hello\n", stdout) - assert.Contains(t, stderr, "AllowedSystemServices: skipping grant 0: system service name must not be empty") - assert.Contains(t, stderr, "AllowedSystemServices: skipping grant 1") - assert.Contains(t, stderr, "glob pattern") + assert.Empty(t, stderr) } -func TestParseAllowedServicesUsesLastColon(t *testing.T) { - grants := parseAllowedServices("tenant:mysql.service:read+reload") - require.Len(t, grants, 1) - assert.Equal(t, "tenant:mysql.service", grants[0].Service) - assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemServiceReload}, grants[0].Actions) +func TestAllowedSystemdFlagRejectsInvalidCombination(t *testing.T) { + code, _, stderr := runCLI(t, + "--allow-all-commands", + "--allowed-systemd", "journal:storage:restart", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, `unsupported operation "restart" on "journal:storage"`) } -func TestParseAllowedServicesPreservesAPIGrantSemantics(t *testing.T) { - grants := parseAllowedServices("mysql.service,redis.service:,:read,nginx.service:read++reload") - assert.Equal(t, []interp.SystemServiceControlGrant{ - {Service: "mysql.service"}, - {Service: "redis.service"}, - {Service: "", Actions: []interp.SystemServiceAction{interp.SystemServiceRead}}, - {Service: "nginx.service", Actions: []interp.SystemServiceAction{interp.SystemServiceRead, "", interp.SystemServiceReload}}, - }, grants) +func TestParseAllowedSystemdUsesLastColon(t *testing.T) { + grants, err := parseAllowedSystemd("unit:tenant:mysql.service:read+reload") + require.NoError(t, err) + require.Len(t, grants, 1) + assert.Equal(t, interp.SystemdResource("unit:tenant:mysql.service"), grants[0].Resource) + assert.Equal(t, []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionReload}, grants[0].Actions) } func TestAllowAllCommandsFlag(t *testing.T) { diff --git a/interp/api.go b/interp/api.go index 5e508dad..33a0f61b 100644 --- a/interp/api.go +++ b/interp/api.go @@ -80,10 +80,10 @@ type runnerConfig struct { // command. Intended for testing convenience. allowAllCommands bool - // allowedSystemServices maps exact service names to their permitted - // actions. It is independent of allowAllCommands and defaults to denying - // every service. - allowedSystemServices systemServiceGrants + // allowedSystemd maps exact resources to their permitted actions. It is + // independent of allowAllCommands and defaults to denying every systemd + // operation. + allowedSystemd systemdGrants // maxExecutionTime bounds the duration of each Run call. Zero disables // the limit. When non-zero, Run derives a child context with this timeout. diff --git a/interp/runner_exec.go b/interp/runner_exec.go index 9e67ff64..2e4bb71f 100644 --- a/interp/runner_exec.go +++ b/interp/runner_exec.go @@ -662,6 +662,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { CommandAllowed: func(n string) bool { return r.allowAllCommands || r.allowedCommands[n] }, + AuthorizeSystemd: r.authorizeSystemd, AuthorizeSystemServices: r.authorizeSystemServices, AllowedPathsList: func() []builtins.AllowedPath { return allowedPathsList(r.sandbox) @@ -790,6 +791,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { CommandAllowed: func(cmdName string) bool { return r.allowAllCommands || r.allowedCommands[cmdName] }, + AuthorizeSystemd: r.authorizeSystemd, AuthorizeSystemServices: r.authorizeSystemServices, AllowedPathsList: func() []builtins.AllowedPath { return allowedPathsList(r.sandbox) diff --git a/interp/system_services.go b/interp/system_services.go index aa66d254..ab81364b 100644 --- a/interp/system_services.go +++ b/interp/system_services.go @@ -13,8 +13,42 @@ import ( "github.com/DataDog/rshell/builtins" ) -// SystemServiceAction identifies an operation that may be granted for a -// system service. +// SystemdAction identifies an operation that may be granted for a systemd +// resource. +type SystemdAction = builtins.SystemdAction + +const ( + SystemdActionRead = builtins.SystemdActionRead + SystemdActionClean = builtins.SystemdActionClean + SystemdActionReload = builtins.SystemdActionReload + SystemdActionRestart = builtins.SystemdActionRestart +) + +// SystemdResource identifies an exact resource in the shared systemd policy. +type SystemdResource = builtins.SystemdResource + +const ( + SystemdResourceJournalAll = builtins.SystemdResourceJournalAll + SystemdResourceJournalKernel = builtins.SystemdResourceJournalKernel + SystemdResourceJournalStorage = builtins.SystemdResourceJournalStorage + SystemdResourceManager = builtins.SystemdResourceManager +) + +// SystemdUnitResource returns the policy resource for one exact unit name. +func SystemdUnitResource(name string) SystemdResource { + return builtins.SystemdUnitResource(name) +} + +// SystemdOperation is one resource/action pair checked by the shared policy. +type SystemdOperation = builtins.SystemdOperation + +// SystemdControlGrant grants Actions for one exact Resource. +type SystemdControlGrant struct { + Resource SystemdResource + Actions []SystemdAction +} + +// Deprecated compatibility aliases for the original service-only policy. type SystemServiceAction = builtins.SystemServiceAction const ( @@ -30,7 +64,7 @@ type SystemServiceControlGrant struct { Actions []SystemServiceAction } -type systemServiceGrants map[string]map[SystemServiceAction]struct{} +type systemdGrants map[SystemdResource]map[SystemdAction]struct{} // AllowedSystemServices configures the system services and actions that // system-service builtins may use. A grant matches its Service exactly: for @@ -42,11 +76,11 @@ type systemServiceGrants map[string]map[SystemServiceAction]struct{} // unsupported actions are skipped with a warning. Duplicate services and // actions are accepted and combined idempotently. // -// When not set (default), or when passed an empty slice, every system service -// is denied. This policy is not bypassed by allowing all commands. -func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { +// When not set (default), or when passed an empty slice, every systemd +// operation is denied. This policy is not bypassed by allowing all commands. +func AllowedSystemd(grants []SystemdControlGrant) RunnerOption { return func(r *Runner) error { - allowed := make(systemServiceGrants, len(grants)) + allowed := make(systemdGrants, len(grants)) for i, grant := range grants { if len(grant.Actions) == 0 { continue @@ -71,20 +105,55 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { actions[action] = struct{}{} } } - r.allowedSystemServices = allowed + r.allowedSystemd = allowed return nil } } -func validSystemServiceAction(action SystemServiceAction) bool { - switch action { - case SystemServiceRead, SystemServiceReload, SystemServiceRestart: - return true +// AllowedSystemServices is a compatibility wrapper that adds the unit: prefix +// to each exact service name and stores the grants in the shared systemd +// allowlist. +func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { + systemdGrants := make([]SystemdControlGrant, len(grants)) + for i, grant := range grants { + systemdGrants[i] = SystemdControlGrant{ + Resource: SystemdUnitResource(grant.Service), + Actions: append([]SystemdAction(nil), grant.Actions...), + } + } + return AllowedSystemd(systemdGrants) +} + +func validSystemdOperation(operation SystemdOperation) bool { + switch { + case strings.HasPrefix(string(operation.Resource), "unit:"): + return operation.Action == SystemdActionRead || operation.Action == SystemdActionReload || operation.Action == SystemdActionRestart + case operation.Resource == SystemdResourceJournalAll, + operation.Resource == SystemdResourceJournalKernel: + return operation.Action == SystemdActionRead + case operation.Resource == SystemdResourceJournalStorage: + return operation.Action == SystemdActionRead || operation.Action == SystemdActionClean + case operation.Resource == SystemdResourceManager: + return operation.Action == SystemdActionRead || operation.Action == SystemdActionReload default: return false } } +func validateSystemdResource(resource SystemdResource) error { + const unitPrefix = "unit:" + resourceName := string(resource) + if strings.HasPrefix(resourceName, unitPrefix) { + return validateSystemServiceName(resourceName[len(unitPrefix):]) + } + switch resource { + case SystemdResourceJournalAll, SystemdResourceJournalKernel, SystemdResourceJournalStorage, SystemdResourceManager: + return nil + default: + return fmt.Errorf("unsupported systemd resource %q", resource) + } +} + func validateSystemServiceName(service string) error { if service == "" { return fmt.Errorf("system service name must not be empty") @@ -104,25 +173,36 @@ func validateSystemServiceName(service string) error { return nil } -func (r *Runner) authorizeSystemServices(action SystemServiceAction, services ...string) error { - if !r.remediationMode { - return fmt.Errorf("system service actions require remediation mode") - } - if !validSystemServiceAction(action) { - return fmt.Errorf("unsupported system service action %q", action) - } - if len(services) == 0 { - return fmt.Errorf("at least one system service is required") +func (r *Runner) authorizeSystemd(operations ...SystemdOperation) error { + if len(operations) == 0 { + return fmt.Errorf("at least one systemd operation is required") } - for _, service := range services { - if err := validateSystemServiceName(service); err != nil { + for _, operation := range operations { + if err := validateSystemdResource(operation.Resource); err != nil { return err } - actions := r.allowedSystemServices[service] - if _, ok := actions[action]; !ok { - return fmt.Errorf("system service %q is not allowed for action %q", service, action) + if !validSystemdOperation(operation) { + return fmt.Errorf("unsupported systemd operation %q on %q", operation.Action, operation.Resource) + } + if operation.Action != SystemdActionRead && !r.remediationMode { + return fmt.Errorf("systemd action %q requires remediation mode", operation.Action) + } + actions := r.allowedSystemd[operation.Resource] + if _, ok := actions[operation.Action]; !ok { + return fmt.Errorf("systemd resource %q is not allowed for action %q", operation.Resource, operation.Action) } } return nil } + +func (r *Runner) authorizeSystemServices(action SystemServiceAction, services ...string) error { + if len(services) == 0 { + return fmt.Errorf("at least one system service is required") + } + operations := make([]SystemdOperation, len(services)) + for i, service := range services { + operations[i] = SystemdOperation{Resource: SystemdUnitResource(service), Action: action} + } + return r.authorizeSystemd(operations...) +} diff --git a/interp/system_services_test.go b/interp/system_services_test.go index 2e5e1062..47b65986 100644 --- a/interp/system_services_test.go +++ b/interp/system_services_test.go @@ -41,12 +41,12 @@ func TestAllowedSystemServicesAuthorizesExactServiceAndAction(t *testing.T) { err = runner.authorizeSystemServices(SystemServiceRestart, "nginx.service") require.Error(t, err) - assert.Contains(t, err.Error(), `system service "nginx.service" is not allowed for action "restart"`) + assert.Contains(t, err.Error(), `systemd resource "unit:nginx.service" is not allowed for action "restart"`) for _, service := range []string{"mysql", "MYSQL.service"} { err = runner.authorizeSystemServices(SystemServiceRead, service) require.Error(t, err) - assert.Contains(t, err.Error(), `system service "`+service+`" is not allowed`) + assert.Contains(t, err.Error(), `systemd resource "unit:`+service+`" is not allowed`) } } @@ -60,19 +60,60 @@ func TestAllowedSystemServicesDefaultDenyIsIndependentOfAllowedCommands(t *testi assert.Contains(t, err.Error(), "not allowed") } -func TestAllowedSystemServicesRequiresRemediationMode(t *testing.T) { +func TestAllowedSystemServicesAllowsReadOutsideRemediationMode(t *testing.T) { runner, err := New(AllowedSystemServices([]SystemServiceControlGrant{ { Service: "mysql.service", - Actions: []SystemServiceAction{SystemServiceRead}, + Actions: []SystemServiceAction{SystemServiceRead, SystemServiceRestart}, }, })) require.NoError(t, err) defer runner.Close() - err = runner.authorizeSystemServices(SystemServiceRead, "mysql.service") + require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) + err = runner.authorizeSystemServices(SystemServiceRestart, "mysql.service") require.Error(t, err) - assert.Contains(t, err.Error(), "require remediation mode") + assert.Contains(t, err.Error(), `action "restart" requires remediation mode`) +} + +func TestAllowedSystemdAuthorizesJournalAndManagerResources(t *testing.T) { + runner, err := New( + WithMode(ModeRemediation), + AllowedSystemd([]SystemdControlGrant{ + {Resource: SystemdResourceJournalKernel, Actions: []SystemdAction{SystemdActionRead}}, + {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead, SystemdActionClean}}, + {Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionReload}}, + }), + ) + require.NoError(t, err) + defer runner.Close() + + require.NoError(t, runner.authorizeSystemd( + SystemdOperation{Resource: SystemdResourceJournalKernel, Action: SystemdActionRead}, + SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionClean}, + SystemdOperation{Resource: SystemdResourceManager, Action: SystemdActionReload}, + )) + + err = runner.authorizeSystemd(SystemdOperation{Resource: SystemdResourceJournalAll, Action: SystemdActionRead}) + require.Error(t, err) + assert.Contains(t, err.Error(), `resource "journal:all" is not allowed`) +} + +func TestAllowedSystemdReadDoesNotEnableMutation(t *testing.T) { + runner, err := New(AllowedSystemd([]SystemdControlGrant{ + {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead}}, + })) + require.NoError(t, err) + defer runner.Close() + + require.NoError(t, runner.authorizeSystemd( + SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionRead}, + )) + err = runner.authorizeSystemd( + SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionClean}, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), `action "clean" requires remediation mode`) } func TestAllowedSystemServicesCopiesAndCombinesGrants(t *testing.T) { @@ -159,6 +200,41 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { assert.Contains(t, warningOutput.String(), `AllowedSystemServices: skipping unsupported action "enable" in grant 1 for "ignored.service"`) } +func TestAllowedSystemdRejectsInvalidResourceActionCombinations(t *testing.T) { + tests := []struct { + name string + grant SystemdControlGrant + }{ + { + name: "unknown resource", + grant: SystemdControlGrant{Resource: "journal:namespace", Actions: []SystemdAction{SystemdActionRead}}, + }, + { + name: "clean unit", + grant: SystemdControlGrant{Resource: SystemdUnitResource("mysql.service"), Actions: []SystemdAction{SystemdActionClean}}, + }, + { + name: "restart journal", + grant: SystemdControlGrant{Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRestart}}, + }, + { + name: "clean manager", + grant: SystemdControlGrant{Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionClean}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runner, err := New(AllowedSystemd([]SystemdControlGrant{test.grant})) + if runner != nil { + runner.Close() + } + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported") + }) + } +} + func TestAuthorizeSystemServicesRejectsInvalidRequests(t *testing.T) { runner, err := New(WithMode(ModeRemediation), AllowedSystemServices([]SystemServiceControlGrant{ { @@ -175,7 +251,7 @@ func TestAuthorizeSystemServicesRejectsInvalidRequests(t *testing.T) { services []string needle string }{ - {name: "unknown action", action: "stop", services: []string{"mysql.service"}, needle: "unsupported system service action"}, + {name: "unknown action", action: "stop", services: []string{"mysql.service"}, needle: "unsupported systemd operation"}, {name: "no services", action: SystemServiceRead, needle: "at least one system service"}, {name: "runtime glob", action: SystemServiceRead, services: []string{"mysql*.service"}, needle: "glob pattern"}, {name: "runtime backslash path", action: SystemServiceRead, services: []string{"..\\mysql.service"}, needle: "path separator"}, From 033677a945030d840a0d56b497bcfc8ac8473ffc Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 12:46:51 -0400 Subject: [PATCH 04/53] feat(systemd): add configurable target paths --- README.md | 2 + SHELL_FEATURES.md | 1 + cmd/rshell/main.go | 32 +++++++++ cmd/rshell/main_test.go | 38 ++++++++++ internal/systemd/target.go | 119 ++++++++++++++++++++++++++++++++ internal/systemd/target_test.go | 107 ++++++++++++++++++++++++++++ interp/api.go | 10 +++ interp/systemd_target.go | 47 +++++++++++++ interp/systemd_target_test.go | 50 ++++++++++++++ 9 files changed, 406 insertions(+) create mode 100644 internal/systemd/target.go create mode 100644 internal/systemd/target_test.go create mode 100644 interp/systemd_target.go create mode 100644 interp/systemd_target_test.go diff --git a/README.md b/README.md index a3f64a5c..b7b8222c 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ interp.AllowedSystemd([]interp.SystemdControlGrant{ The development CLI accepts equivalent grants through `--allowed-systemd unit:mysql.service:restart+reload+read,journal:storage:read+clean`. The older `AllowedSystemServices` API and `--allowed-services` flag remain as unit-only compatibility shorthands backed by the same allowlist. The policy and authorization capability are implemented, but `systemctl` and `journalctl` builtins are not yet available. +**SystemdTargetConfig** selects which Linux host those builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, system bus socket, and journald runtime directory. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target so later backends can reject mixed-host configurations. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. + **AllowedPaths** restricts all file operations to specified directories using Go's `os.Root` API for reads and openat-based write handling for writes. - **Sandbox mechanism:** Reads go through `os.Root`; writes are checked against the most-specific path mode and, on Unix, opened with a no-symlink `openat` walk. Files outside the allowlist cannot be opened, created, truncated, or appended to. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 2430ebf0..4f92a6f0 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -122,6 +122,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed - ✅ AllowedSystemServices policy — one shared default-deny capability map for resources with generic `read`, `clean`, `reload`, and `restart` actions; invalid resource/action pairs are rejected, unit names are case-sensitive and are not normalized, `read` works in read-only mode, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure through `interp.AllowedSystemd` or CLI `--allowed-systemd RESOURCE:ACTION[+ACTION...]`; `AllowedSystemServices` and `--allowed-services` remain unit-only compatibility shorthands +- ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives a complete mounted-host target, while explicit journal, machine-id, Varlink, D-Bus, and runtime paths support split mount layouts without falling back to local endpoints - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command - ✅ ProcPath — overrides the proc filesystem path used by `ps` (default `/proc`; Linux-only; useful for testing/container environments); `ps` does not read `/proc//cmdline` diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index 3916be2e..6afbb5a6 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -44,6 +44,12 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowAllCmds bool timeout time.Duration procPath string + systemdRoot string + journalDirs string + machineIDPath string + journalSocket string + systemBusSocket string + journalRuntime string mode string ) @@ -100,6 +106,12 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. return fmt.Errorf("--mode must be one of: read-only, remediation") } + var configuredJournalDirs []string + if journalDirs != "" { + configuredJournalDirs = strings.Split(journalDirs, ",") + } + systemdTargetSet := systemdRoot != "" || journalDirs != "" || machineIDPath != "" || journalSocket != "" || systemBusSocket != "" || journalRuntime != "" + execOpts := executeOpts{ allowedPaths: paths, allowedCommands: cmds, @@ -107,6 +119,15 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowedSystemd: systemdGrants, allowAllCommands: allowAllCmds, procPath: procPath, + systemdTarget: interp.SystemdTargetConfig{ + Root: systemdRoot, + JournalDirs: configuredJournalDirs, + MachineIDPath: machineIDPath, + JournalControlSocket: journalSocket, + SystemBusSocket: systemBusSocket, + JournalRuntimeDir: journalRuntime, + }, + systemdTargetSet: systemdTargetSet, mode: parsedMode, } @@ -162,6 +183,12 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().BoolVar(&allowAllCmds, "allow-all-commands", false, "allow execution of all commands (builtins and external)") cmd.Flags().DurationVar(&timeout, "timeout", 0, "maximum execution time for the entire shell run (e.g. 100ms, 5s, 1m)") cmd.Flags().StringVar(&procPath, "proc-path", "", "path to the proc filesystem used by ps (default \"/proc\")") + cmd.Flags().StringVar(&systemdRoot, "systemd-root", "", "mounted systemd host root used to derive all target paths; cannot be combined with explicit systemd paths") + cmd.Flags().StringVar(&journalDirs, "systemd-journal-dirs", "", "comma-separated journal root directories for an explicit systemd target") + cmd.Flags().StringVar(&machineIDPath, "systemd-machine-id-path", "", "machine-id file for an explicit systemd target") + cmd.Flags().StringVar(&journalSocket, "systemd-journal-socket", "", "journald Varlink socket for an explicit systemd target") + cmd.Flags().StringVar(&systemBusSocket, "systemd-bus-socket", "", "system D-Bus socket for an explicit systemd target") + cmd.Flags().StringVar(&journalRuntime, "systemd-runtime-dir", "", "journald runtime directory for legacy control acknowledgements") cmd.Flags().StringVar(&mode, "mode", "read-only", "shell execution mode: read-only (default) or remediation (enables file-target output redirections within :rw AllowedPaths roots)") if err := cmd.ExecuteContext(ctx); err != nil { @@ -234,6 +261,8 @@ type executeOpts struct { allowedSystemd []interp.SystemdControlGrant allowAllCommands bool procPath string + systemdTarget interp.SystemdTargetConfig + systemdTargetSet bool mode interp.Mode } @@ -271,6 +300,9 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i if opts.procPath != "" { runOpts = append(runOpts, interp.ProcPath(opts.procPath)) } + if opts.systemdTargetSet { + runOpts = append(runOpts, interp.WithSystemdTarget(opts.systemdTarget)) + } if opts.mode != "" { runOpts = append(runOpts, interp.WithMode(opts.mode)) } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index 23a7b439..bb0733e8 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -186,6 +186,12 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "--allow-all-commands") assert.Contains(t, stdout, "file-target output redirections within :rw AllowedPaths roots") assert.Contains(t, stdout, "--timeout") + assert.Contains(t, stdout, "--systemd-root") + assert.Contains(t, stdout, "--systemd-journal-dirs") + assert.Contains(t, stdout, "--systemd-machine-id-path") + assert.Contains(t, stdout, "--systemd-journal-socket") + assert.Contains(t, stdout, "--systemd-bus-socket") + assert.Contains(t, stdout, "--systemd-runtime-dir") assert.NotContains(t, stdout, "--command", "-c/--command should be hidden from help") } @@ -371,6 +377,38 @@ func TestParseAllowedSystemdUsesLastColon(t *testing.T) { assert.Equal(t, []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionReload}, grants[0].Actions) } +func TestSystemdRootFlag(t *testing.T) { + code, stdout, stderr := runCLI(t, + "--allow-all-commands", + "--systemd-root", "/host", + "-c", `echo hello`, + ) + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Empty(t, stderr) +} + +func TestSystemdTargetFlagsRejectMixedRootAndExplicitPaths(t *testing.T) { + code, _, stderr := runCLI(t, + "--allow-all-commands", + "--systemd-root", "/host", + "--systemd-machine-id-path", "/host/etc/machine-id", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, "cannot be combined") +} + +func TestExplicitSystemdTargetRequiresMachineIDPath(t *testing.T) { + code, _, stderr := runCLI(t, + "--allow-all-commands", + "--systemd-journal-dirs", "/host/var/log/journal,/host/run/log/journal", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, "machine ID path is required") +} + func TestAllowAllCommandsFlag(t *testing.T) { code, stdout, _ := runCLI(t, "--allow-all-commands", "-c", `echo hello`) assert.Equal(t, 0, code) diff --git a/internal/systemd/target.go b/internal/systemd/target.go new file mode 100644 index 00000000..b39f2b29 --- /dev/null +++ b/internal/systemd/target.go @@ -0,0 +1,119 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +// Package systemd contains the trusted target and transport implementation +// used by systemd-aware builtins. +package systemd + +import ( + "fmt" + "path/filepath" + "strings" +) + +const MaxJournalDirs = 8 + +// Target identifies one systemd host. Root is accepted only as input to +// ResolveTarget; resolved targets contain explicit paths in the remaining +// fields. +type Target struct { + Root string + JournalDirs []string + MachineIDPath string + JournalControlSocket string + SystemBusSocket string + JournalRuntimeDir string +} + +// LocalTarget returns the standard paths for the local systemd host. +func LocalTarget() Target { + return targetFromRoot("/") +} + +// ResolveTarget validates and resolves a target. A zero target selects the +// local host. Root derives every standard path below that root and cannot be +// mixed with explicit fields. Once any explicit field is supplied, omitted +// fields remain empty and never fall back to local paths. +func ResolveTarget(target Target) (Target, error) { + if target.Root != "" { + if len(target.JournalDirs) > 0 || target.MachineIDPath != "" || target.JournalControlSocket != "" || target.SystemBusSocket != "" || target.JournalRuntimeDir != "" { + return Target{}, fmt.Errorf("systemd target root cannot be combined with explicit paths") + } + root, err := validateAbsolutePath("root", target.Root) + if err != nil { + return Target{}, err + } + return targetFromRoot(root), nil + } + + if len(target.JournalDirs) == 0 && target.MachineIDPath == "" && target.JournalControlSocket == "" && target.SystemBusSocket == "" && target.JournalRuntimeDir == "" { + return LocalTarget(), nil + } + if len(target.JournalDirs) > MaxJournalDirs { + return Target{}, fmt.Errorf("systemd target has %d journal directories; maximum is %d", len(target.JournalDirs), MaxJournalDirs) + } + if target.MachineIDPath == "" { + return Target{}, fmt.Errorf("systemd target machine ID path is required with explicit paths") + } + + resolved := Target{JournalDirs: make([]string, 0, len(target.JournalDirs))} + seenDirs := make(map[string]struct{}, len(target.JournalDirs)) + for i, dir := range target.JournalDirs { + clean, err := validateAbsolutePath(fmt.Sprintf("journal directory %d", i), dir) + if err != nil { + return Target{}, err + } + if _, exists := seenDirs[clean]; exists { + continue + } + seenDirs[clean] = struct{}{} + resolved.JournalDirs = append(resolved.JournalDirs, clean) + } + + var err error + if resolved.MachineIDPath, err = validateAbsolutePath("machine ID path", target.MachineIDPath); err != nil { + return Target{}, err + } + if resolved.JournalControlSocket, err = validateOptionalAbsolutePath("journal control socket", target.JournalControlSocket); err != nil { + return Target{}, err + } + if resolved.SystemBusSocket, err = validateOptionalAbsolutePath("system bus socket", target.SystemBusSocket); err != nil { + return Target{}, err + } + if resolved.JournalRuntimeDir, err = validateOptionalAbsolutePath("journal runtime directory", target.JournalRuntimeDir); err != nil { + return Target{}, err + } + return resolved, nil +} + +func targetFromRoot(root string) Target { + return Target{ + JournalDirs: []string{ + filepath.Join(root, "var", "log", "journal"), + filepath.Join(root, "run", "log", "journal"), + }, + MachineIDPath: filepath.Join(root, "etc", "machine-id"), + JournalControlSocket: filepath.Join(root, "run", "systemd", "journal", "io.systemd.journal"), + SystemBusSocket: filepath.Join(root, "run", "dbus", "system_bus_socket"), + JournalRuntimeDir: filepath.Join(root, "run", "systemd", "journal"), + } +} + +func validateOptionalAbsolutePath(name, path string) (string, error) { + if path == "" { + return "", nil + } + return validateAbsolutePath(name, path) +} + +func validateAbsolutePath(name, path string) (string, error) { + if strings.IndexByte(path, 0) >= 0 { + return "", fmt.Errorf("systemd target %s contains a NUL byte", name) + } + if !filepath.IsAbs(path) { + return "", fmt.Errorf("systemd target %s %q must be absolute", name, path) + } + return filepath.Clean(path), nil +} diff --git a/internal/systemd/target_test.go b/internal/systemd/target_test.go new file mode 100644 index 00000000..a9288a64 --- /dev/null +++ b/internal/systemd/target_test.go @@ -0,0 +1,107 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !windows + +package systemd + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveTargetDefaultsToLocalPaths(t *testing.T) { + target, err := ResolveTarget(Target{}) + require.NoError(t, err) + + assert.Equal(t, []string{"/var/log/journal", "/run/log/journal"}, target.JournalDirs) + assert.Equal(t, "/etc/machine-id", target.MachineIDPath) + assert.Equal(t, "/run/systemd/journal/io.systemd.journal", target.JournalControlSocket) + assert.Equal(t, "/run/dbus/system_bus_socket", target.SystemBusSocket) + assert.Equal(t, "/run/systemd/journal", target.JournalRuntimeDir) +} + +func TestResolveTargetDerivesMountedRootPaths(t *testing.T) { + target, err := ResolveTarget(Target{Root: "/host"}) + require.NoError(t, err) + + assert.Equal(t, []string{filepath.FromSlash("/host/var/log/journal"), filepath.FromSlash("/host/run/log/journal")}, target.JournalDirs) + assert.Equal(t, filepath.FromSlash("/host/etc/machine-id"), target.MachineIDPath) + assert.Equal(t, filepath.FromSlash("/host/run/systemd/journal/io.systemd.journal"), target.JournalControlSocket) + assert.Equal(t, filepath.FromSlash("/host/run/dbus/system_bus_socket"), target.SystemBusSocket) + assert.Equal(t, filepath.FromSlash("/host/run/systemd/journal"), target.JournalRuntimeDir) +} + +func TestResolveTargetUsesOnlyExplicitPaths(t *testing.T) { + target, err := ResolveTarget(Target{ + JournalDirs: []string{"/mnt/logs", "/mnt/logs", "/mnt/runtime-logs/../runtime-logs"}, + MachineIDPath: "/mnt/etc/machine-id", + }) + require.NoError(t, err) + + assert.Equal(t, []string{"/mnt/logs", "/mnt/runtime-logs"}, target.JournalDirs) + assert.Equal(t, "/mnt/etc/machine-id", target.MachineIDPath) + assert.Empty(t, target.JournalControlSocket) + assert.Empty(t, target.SystemBusSocket) + assert.Empty(t, target.JournalRuntimeDir) +} + +func TestResolveTargetRejectsInvalidConfiguration(t *testing.T) { + tests := []struct { + name string + target Target + needle string + }{ + { + name: "root with explicit field", + target: Target{Root: "/host", MachineIDPath: "/host/etc/machine-id"}, + needle: "cannot be combined", + }, + { + name: "relative root", + target: Target{Root: "host"}, + needle: "must be absolute", + }, + { + name: "explicit without machine ID", + target: Target{JournalDirs: []string{"/host/var/log/journal"}}, + needle: "machine ID path is required", + }, + { + name: "relative journal directory", + target: Target{JournalDirs: []string{"logs"}, MachineIDPath: "/etc/machine-id"}, + needle: "must be absolute", + }, + { + name: "relative socket", + target: Target{MachineIDPath: "/etc/machine-id", SystemBusSocket: "run/dbus/system_bus_socket"}, + needle: "must be absolute", + }, + { + name: "NUL path", + target: Target{MachineIDPath: "/etc/machine-id\x00suffix"}, + needle: "NUL byte", + }, + { + name: "too many journal directories", + target: Target{ + JournalDirs: []string{"/1", "/2", "/3", "/4", "/5", "/6", "/7", "/8", "/9"}, + MachineIDPath: "/etc/machine-id", + }, + needle: "maximum is 8", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ResolveTarget(test.target) + require.Error(t, err) + assert.Contains(t, err.Error(), test.needle) + }) + } +} diff --git a/interp/api.go b/interp/api.go index 33a0f61b..58629607 100644 --- a/interp/api.go +++ b/interp/api.go @@ -29,6 +29,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry" "github.com/DataDog/rshell/allowedpaths" "github.com/DataDog/rshell/builtins" + internalsystemd "github.com/DataDog/rshell/internal/systemd" "github.com/DataDog/rshell/internal/version" ) @@ -85,6 +86,12 @@ type runnerConfig struct { // operation. allowedSystemd systemdGrants + // systemdTarget identifies the local or mounted host used by systemd-aware + // builtins. It is resolved once during construction and shared by + // subshells. + systemdTarget internalsystemd.Target + systemdTargetConfigured bool + // maxExecutionTime bounds the duration of each Run call. Zero disables // the limit. When non-zero, Run derives a child context with this timeout. maxExecutionTime time.Duration @@ -321,6 +328,9 @@ func New(opts ...RunnerOption) (*Runner, error) { if len(r.sandboxWarnings) > 0 { r.warningsWriter.Write(r.sandboxWarnings) } + if !r.systemdTargetConfigured { + r.systemdTarget = internalsystemd.LocalTarget() + } r.proc = builtins.NewProcProvider(r.procPath) return r, nil } diff --git a/interp/systemd_target.go b/interp/systemd_target.go new file mode 100644 index 00000000..ca4a6364 --- /dev/null +++ b/interp/systemd_target.go @@ -0,0 +1,47 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package interp + +import ( + "fmt" + + internalsystemd "github.com/DataDog/rshell/internal/systemd" +) + +// SystemdTargetConfig selects the systemd host used by systemd-aware +// builtins. The zero value uses standard local paths. Root derives all paths +// below one mounted host root and cannot be mixed with explicit fields. When +// explicit fields are used, omitted fields stay unavailable and never fall +// back to local paths. +type SystemdTargetConfig struct { + Root string + JournalDirs []string + MachineIDPath string + JournalControlSocket string + SystemBusSocket string + JournalRuntimeDir string +} + +// WithSystemdTarget configures the trusted systemd target. Scripts cannot +// override these paths. +func WithSystemdTarget(config SystemdTargetConfig) RunnerOption { + return func(r *Runner) error { + target, err := internalsystemd.ResolveTarget(internalsystemd.Target{ + Root: config.Root, + JournalDirs: append([]string(nil), config.JournalDirs...), + MachineIDPath: config.MachineIDPath, + JournalControlSocket: config.JournalControlSocket, + SystemBusSocket: config.SystemBusSocket, + JournalRuntimeDir: config.JournalRuntimeDir, + }) + if err != nil { + return fmt.Errorf("WithSystemdTarget: %w", err) + } + r.systemdTarget = target + r.systemdTargetConfigured = true + return nil + } +} diff --git a/interp/systemd_target_test.go b/interp/systemd_target_test.go new file mode 100644 index 00000000..608f684d --- /dev/null +++ b/interp/systemd_target_test.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !windows + +package interp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithSystemdTargetDefaultsToLocal(t *testing.T) { + runner, err := New() + require.NoError(t, err) + defer runner.Close() + + assert.Equal(t, []string{"/var/log/journal", "/run/log/journal"}, runner.systemdTarget.JournalDirs) + assert.Equal(t, "/etc/machine-id", runner.systemdTarget.MachineIDPath) +} + +func TestWithSystemdTargetCopiesConfiguration(t *testing.T) { + dirs := []string{"/host/var/log/journal"} + runner, err := New(WithSystemdTarget(SystemdTargetConfig{ + JournalDirs: dirs, + MachineIDPath: "/host/etc/machine-id", + })) + require.NoError(t, err) + defer runner.Close() + + dirs[0] = "/changed" + assert.Equal(t, []string{"/host/var/log/journal"}, runner.systemdTarget.JournalDirs) + assert.Empty(t, runner.systemdTarget.SystemBusSocket) +} + +func TestWithSystemdTargetRejectsMixedRootAndExplicitPaths(t *testing.T) { + runner, err := New(WithSystemdTarget(SystemdTargetConfig{ + Root: "/host", + JournalDirs: []string{"/host/var/log/journal"}, + })) + if runner != nil { + runner.Close() + } + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be combined") +} From ea4d52610a54ba3a5cbef790f5e0ed72bc523e0a Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:00:13 -0400 Subject: [PATCH 05/53] feat(systemd): add bounded journal reader --- .github/workflows/compliance.yml | 2 + .github/workflows/test.yml | 5 + builtins/builtins.go | 4 + builtins/systemd.go | 48 ++++ docs/RULES.md | 15 ++ go.mod | 1 + go.sum | 2 + internal/systemd/client.go | 19 ++ internal/systemd/journal_files.go | 121 +++++++++ internal/systemd/journal_files_test.go | 77 ++++++ internal/systemd/journal_reader.go | 250 ++++++++++++++++++ internal/systemd/journal_reader_linux_cgo.go | 38 +++ internal/systemd/journal_reader_test.go | 248 +++++++++++++++++ .../systemd/journal_reader_unsupported.go | 27 ++ interp/api.go | 2 + interp/runner_exec.go | 5 +- 16 files changed, 863 insertions(+), 1 deletion(-) create mode 100644 builtins/systemd.go create mode 100644 internal/systemd/client.go create mode 100644 internal/systemd/journal_files.go create mode 100644 internal/systemd/journal_files_test.go create mode 100644 internal/systemd/journal_reader.go create mode 100644 internal/systemd/journal_reader_linux_cgo.go create mode 100644 internal/systemd/journal_reader_test.go create mode 100644 internal/systemd/journal_reader_unsupported.go diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 6294fd04..04c66313 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version + - name: Install systemd development headers + run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run compliance checks env: RSHELL_COMPLIANCE_TEST: "1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1d909e9b..d3ab86d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,9 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version + - name: Install systemd development headers + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run tests with race detector run: go test -race -v -timeout 10m ./... @@ -69,6 +72,8 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version + - name: Install systemd development headers + run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run bash comparison tests env: RSHELL_BASH_TEST: "1" diff --git a/builtins/builtins.go b/builtins/builtins.go index a73b096d..cff26da5 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -373,6 +373,10 @@ type CallContext struct { // Proc provides access to the proc filesystem for the ps builtin. // The path is fixed at construction time and cannot be overridden by callers. Proc *ProcProvider + + // Systemd contains structured backends for systemd-aware builtins. Target + // paths and transports are fixed by trusted runner configuration. + Systemd *SystemdServices } // Out writes a string to stdout. diff --git a/builtins/systemd.go b/builtins/systemd.go new file mode 100644 index 00000000..c7cb67d4 --- /dev/null +++ b/builtins/systemd.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package builtins + +import ( + "context" + "errors" + "time" +) + +// ErrSystemdUnsupported reports that the current platform cannot provide a +// requested systemd operation. +var ErrSystemdUnsupported = errors.New("systemd operation is not supported") + +// JournalQuery is the bounded, structured query accepted by the trusted +// journal backend. Callers cannot provide raw journal matches or paths. +type JournalQuery struct { + Units []string + Kernel bool + CurrentBoot bool + Since time.Time + MaxEntries int +} + +// JournalEntry contains only the fields a journalctl builtin may expose. The +// backend deliberately does not return arbitrary journal fields. +type JournalEntry struct { + Timestamp time.Time + Hostname string + Identifier string + PID string + Message string +} + +// JournalReader reads a bounded journal query and yields entries oldest first. +type JournalReader interface { + ReadJournal(ctx context.Context, query JournalQuery, yield func(JournalEntry) error) error +} + +// SystemdServices contains the trusted backends available to systemd-aware +// builtins. Additional manager and journal-maintenance interfaces can be added +// here without exposing transports to command implementations. +type SystemdServices struct { + Journal JournalReader +} diff --git a/docs/RULES.md b/docs/RULES.md index 98c175d0..80ca90e0 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -71,6 +71,21 @@ f, err := os.Open(path) Using `os` constants (`os.O_RDONLY`, `os.FileMode`) and types (`*os.File` for stdin) is fine; only the filesystem-accessing *functions* are forbidden. +#### Trusted systemd target exception + +Systemd-aware builtins MUST use the structured services on `callCtx.Systemd` and +MUST NOT open target paths themselves. The trusted `internal/systemd` backend may +read paths selected by `interp.WithSystemdTarget`; those paths intentionally +bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding +application and cannot be supplied by shell scripts. + +The journal reader is limited to regular, non-symlink `.journal` files directly +under the configured machine-ID directories. It reads the configured machine ID, +adds it to every native journal query, verifies it again on every returned entry, +and applies fixed file, field-size, entry-count, and cancellation bounds. Builtins +receive selected fields only and never receive a raw journal handle, target path, +or arbitrary field-match capability. + --- ## Implementation Rules diff --git a/go.mod b/go.mod index 71108233..841320b5 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.26.2 require ( github.com/DataDog/datadog-agent/pkg/fleet/installer v0.78.0 + github.com/coreos/go-systemd/v22 v22.7.0 github.com/prometheus-community/pro-bing v0.8.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 diff --git a/go.sum b/go.sum index 4ffe9e64..59fa7c52 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/DataDog/datadog-agent/pkg/util/scrubber v0.78.0 h1:MydZgoGPYjGt8jCwo2 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.78.0/go.mod h1:zM5uAINxu8o9OZGCCvqeH1E/vIsX6baiO2iPf6bs1xs= github.com/DataDog/datadog-agent/pkg/version v0.78.0 h1:piLsyG7dP/ie6d4MQ17pTh5XPt/BV55dy8910eXiNj0= github.com/DataDog/datadog-agent/pkg/version v0.78.0/go.mod h1:cOWF+29ZahL6qcC3KAntJEupdMdteiOGaLmIMz8zYBg= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/systemd/client.go b/internal/systemd/client.go new file mode 100644 index 00000000..49d2d876 --- /dev/null +++ b/internal/systemd/client.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +// Client implements the trusted systemd backends for one resolved target. +// Target paths are copied at construction so runner configuration remains +// immutable while commands execute. +type Client struct { + target Target +} + +// NewClient creates a client for one resolved systemd target. +func NewClient(target Target) *Client { + target.JournalDirs = append([]string(nil), target.JournalDirs...) + return &Client{target: target} +} diff --git a/internal/systemd/journal_files.go b/internal/systemd/journal_files.go new file mode 100644 index 00000000..5fdeea44 --- /dev/null +++ b/internal/systemd/journal_files.go @@ -0,0 +1,121 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + maxJournalFiles = 4096 + maxMachineIDFileSize = 64 +) + +func (c *Client) journalFiles() (string, []string, error) { + if c.target.MachineIDPath == "" { + return "", nil, fmt.Errorf("systemd target machine ID path is not configured") + } + if len(c.target.JournalDirs) == 0 { + return "", nil, fmt.Errorf("systemd target journal directories are not configured") + } + + machineID, err := readMachineID(c.target.MachineIDPath) + if err != nil { + return "", nil, err + } + + files := make([]string, 0) + for _, baseDir := range c.target.JournalDirs { + dirPath := filepath.Join(baseDir, machineID) + dir, err := os.Open(dirPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return "", nil, fmt.Errorf("open journal directory %q: %w", dirPath, err) + } + + entries, readErr := dir.ReadDir(maxJournalFiles + 1) + closeErr := dir.Close() + if readErr != nil && readErr != io.EOF { + return "", nil, fmt.Errorf("read journal directory %q: %w", dirPath, readErr) + } + if closeErr != nil { + return "", nil, fmt.Errorf("close journal directory %q: %w", dirPath, closeErr) + } + if len(entries) > maxJournalFiles { + return "", nil, fmt.Errorf("journal directory %q has too many entries (maximum %d)", dirPath, maxJournalFiles) + } + + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".journal") || entry.Type()&fs.ModeSymlink != 0 { + continue + } + info, err := entry.Info() + if err != nil { + return "", nil, fmt.Errorf("inspect journal file %q: %w", filepath.Join(dirPath, entry.Name()), err) + } + if !info.Mode().IsRegular() { + continue + } + files = append(files, filepath.Join(dirPath, entry.Name())) + if len(files) > maxJournalFiles { + return "", nil, fmt.Errorf("systemd target has too many journal files (maximum %d)", maxJournalFiles) + } + } + } + + if len(files) == 0 { + return "", nil, fmt.Errorf("no journal files found for machine %s", machineID) + } + sort.Strings(files) + return machineID, files, nil +} + +func readMachineID(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open systemd machine ID %q: %w", path, err) + } + data, readErr := io.ReadAll(io.LimitReader(file, maxMachineIDFileSize+1)) + closeErr := file.Close() + if readErr != nil { + return "", fmt.Errorf("read systemd machine ID %q: %w", path, readErr) + } + if closeErr != nil { + return "", fmt.Errorf("close systemd machine ID %q: %w", path, closeErr) + } + if len(data) > maxMachineIDFileSize { + return "", fmt.Errorf("systemd machine ID file %q is too large", path) + } + + machineID := strings.TrimSpace(string(data)) + if len(machineID) != 32 { + return "", fmt.Errorf("systemd machine ID in %q must contain exactly 32 hexadecimal characters", path) + } + if !validID128(machineID) { + return "", fmt.Errorf("systemd machine ID in %q contains a non-hexadecimal character", path) + } + return strings.ToLower(machineID), nil +} + +func validID128(value string) bool { + if len(value) != 32 { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go new file mode 100644 index 00000000..ecb09111 --- /dev/null +++ b/internal/systemd/journal_files_test.go @@ -0,0 +1,77 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadMachineIDValidatesAndNormalizes(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "machine-id") + require.NoError(t, os.WriteFile(path, []byte("0123456789ABCDEF0123456789ABCDEF\n"), 0o600)) + + machineID, err := readMachineID(path) + require.NoError(t, err) + assert.Equal(t, "0123456789abcdef0123456789abcdef", machineID) + + require.NoError(t, os.WriteFile(path, []byte("not-a-machine-id\n"), 0o600)) + _, err = readMachineID(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "exactly 32 hexadecimal characters") + + require.NoError(t, os.WriteFile(path, []byte(strings.Repeat("a", maxMachineIDFileSize+1)), 0o600)) + _, err = readMachineID(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "too large") +} + +func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + + firstBase := filepath.Join(root, "run-journal") + secondBase := filepath.Join(root, "var-journal") + firstMachineDir := filepath.Join(firstBase, machineID) + secondMachineDir := filepath.Join(secondBase, machineID) + require.NoError(t, os.MkdirAll(firstMachineDir, 0o700)) + require.NoError(t, os.MkdirAll(secondMachineDir, 0o700)) + + firstJournal := filepath.Join(firstMachineDir, "system.journal") + secondJournal := filepath.Join(secondMachineDir, "system@0001.journal") + require.NoError(t, os.WriteFile(firstJournal, nil, 0o600)) + require.NoError(t, os.WriteFile(secondJournal, nil, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(firstMachineDir, "ignored.txt"), nil, 0o600)) + require.NoError(t, os.Mkdir(filepath.Join(firstMachineDir, "directory.journal"), 0o700)) + if runtime.GOOS != "windows" { + require.NoError(t, os.Symlink(firstJournal, filepath.Join(firstMachineDir, "link.journal"))) + } + + client := NewClient(Target{ + JournalDirs: []string{secondBase, firstBase, filepath.Join(root, "missing")}, + MachineIDPath: machineIDPath, + }) + gotMachineID, files, err := client.journalFiles() + require.NoError(t, err) + assert.Equal(t, machineID, gotMachineID) + assert.Equal(t, []string{firstJournal, secondJournal}, files) +} + +func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { + client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}) + _, _, err := client.journalFiles() + require.Error(t, err) + assert.Contains(t, err.Error(), "journal directories are not configured") +} diff --git a/internal/systemd/journal_reader.go b/internal/systemd/journal_reader.go new file mode 100644 index 00000000..ffcfc89f --- /dev/null +++ b/internal/systemd/journal_reader.go @@ -0,0 +1,250 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "errors" + "fmt" + "strings" + "syscall" + "time" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalEntries = 1000 + maxJournalUnits = 32 + maxJournalFieldSize = 64 * 1024 + maxBootSearch = 1024 +) + +type journalHandle interface { + AddMatch(match string) error + AddDisjunction() error + AddConjunction() error + FlushMatches() + Next() (uint64, error) + Previous() (uint64, error) + PreviousSkip(skip uint64) (uint64, error) + GetData(field string) (string, error) + GetRealtimeUsec() (uint64, error) + SetDataThreshold(threshold uint64) error + SeekHead() error + SeekTail() error +} + +func validateJournalQuery(query builtins.JournalQuery) error { + if query.MaxEntries < 0 || query.MaxEntries > maxJournalEntries { + return fmt.Errorf("journal query entry limit must be between 0 and %d", maxJournalEntries) + } + if query.Kernel && len(query.Units) > 0 { + return fmt.Errorf("journal query cannot combine kernel and unit scopes") + } + if !query.Kernel && len(query.Units) == 0 { + return fmt.Errorf("journal query requires a kernel or unit scope") + } + if len(query.Units) > maxJournalUnits { + return fmt.Errorf("journal query has too many units (maximum %d)", maxJournalUnits) + } + for _, unit := range query.Units { + if unit == "" || len(unit) > 256 || strings.IndexByte(unit, 0) >= 0 { + return fmt.Errorf("journal query contains an invalid unit name") + } + } + return nil +} + +func readJournal(ctx context.Context, journal journalHandle, machineID string, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { + if err := validateJournalQuery(query); err != nil { + return err + } + if query.MaxEntries == 0 { + return nil + } + if err := journal.SetDataThreshold(maxJournalFieldSize + 64); err != nil { + return fmt.Errorf("set journal field limit: %w", err) + } + + bootID := "" + if query.CurrentBoot { + var err error + bootID, err = newestBootID(ctx, journal, machineID) + if err != nil { + return err + } + journal.FlushMatches() + } + if err := addJournalMatches(journal, machineID, bootID, query); err != nil { + return err + } + if err := seekJournalTail(journal, query.MaxEntries); err != nil { + return err + } + + for scanned := 0; scanned < query.MaxEntries; scanned++ { + if err := ctx.Err(); err != nil { + return err + } + advanced, err := journal.Next() + if err != nil { + return fmt.Errorf("iterate journal: %w", err) + } + if advanced == 0 { + return nil + } + + entry, err := selectedJournalEntry(journal, machineID) + if err != nil { + return err + } + if !query.Since.IsZero() && entry.Timestamp.Before(query.Since) { + continue + } + if err := yield(entry); err != nil { + return err + } + } + return nil +} + +func newestBootID(ctx context.Context, journal journalHandle, machineID string) (string, error) { + if err := journal.AddMatch("_MACHINE_ID=" + machineID); err != nil { + return "", fmt.Errorf("match journal machine ID: %w", err) + } + if err := journal.SeekTail(); err != nil { + return "", fmt.Errorf("seek journal tail: %w", err) + } + for i := 0; i < maxBootSearch; i++ { + if err := ctx.Err(); err != nil { + return "", err + } + advanced, err := journal.Previous() + if err != nil { + return "", fmt.Errorf("search current journal boot: %w", err) + } + if advanced == 0 { + break + } + bootID, found, err := journalData(journal, "_BOOT_ID") + if err != nil { + return "", err + } + if found && validID128(bootID) { + return strings.ToLower(bootID), nil + } + } + return "", fmt.Errorf("could not determine the current boot from the selected journal") +} + +func addJournalMatches(journal journalHandle, machineID, bootID string, query builtins.JournalQuery) error { + if query.Kernel { + if err := journal.AddMatch("_TRANSPORT=kernel"); err != nil { + return fmt.Errorf("match kernel journal entries: %w", err) + } + } else { + for _, unit := range query.Units { + if err := journal.AddMatch("_SYSTEMD_UNIT=" + unit); err != nil { + return fmt.Errorf("match journal unit: %w", err) + } + } + if err := journal.AddDisjunction(); err != nil { + return fmt.Errorf("combine journal unit matches: %w", err) + } + if err := journal.AddMatch("_PID=1"); err != nil { + return fmt.Errorf("match system manager journal entries: %w", err) + } + for _, unit := range query.Units { + if err := journal.AddMatch("UNIT=" + unit); err != nil { + return fmt.Errorf("match manager messages about journal unit: %w", err) + } + } + if err := journal.AddConjunction(); err != nil { + return fmt.Errorf("constrain journal unit matches: %w", err) + } + } + if err := journal.AddMatch("_MACHINE_ID=" + machineID); err != nil { + return fmt.Errorf("match journal machine ID: %w", err) + } + if bootID != "" { + if err := journal.AddMatch("_BOOT_ID=" + bootID); err != nil { + return fmt.Errorf("match current journal boot: %w", err) + } + } + return nil +} + +func seekJournalTail(journal journalHandle, entries int) error { + if err := journal.SeekTail(); err != nil { + return fmt.Errorf("seek journal tail: %w", err) + } + skipped, err := journal.PreviousSkip(uint64(entries) + 1) + if err != nil { + return fmt.Errorf("seek to last %d journal entries: %w", entries, err) + } + if skipped != uint64(entries)+1 { + if err := journal.SeekHead(); err != nil { + return fmt.Errorf("seek journal head: %w", err) + } + } + return nil +} + +func selectedJournalEntry(journal journalHandle, machineID string) (builtins.JournalEntry, error) { + entryMachineID, found, err := journalData(journal, "_MACHINE_ID") + if err != nil { + return builtins.JournalEntry{}, err + } + if !found || entryMachineID != machineID { + return builtins.JournalEntry{}, fmt.Errorf("journal entry machine ID does not match the configured target") + } + + realtimeUsec, err := journal.GetRealtimeUsec() + if err != nil { + return builtins.JournalEntry{}, fmt.Errorf("read journal timestamp: %w", err) + } + entry := builtins.JournalEntry{ + Timestamp: time.Unix(int64(realtimeUsec/1_000_000), int64(realtimeUsec%1_000_000)*1000), + } + if entry.Hostname, _, err = journalData(journal, "_HOSTNAME"); err != nil { + return builtins.JournalEntry{}, err + } + if entry.Identifier, _, err = journalData(journal, "SYSLOG_IDENTIFIER"); err != nil { + return builtins.JournalEntry{}, err + } + if entry.Identifier == "" { + if entry.Identifier, _, err = journalData(journal, "_COMM"); err != nil { + return builtins.JournalEntry{}, err + } + } + if entry.PID, _, err = journalData(journal, "_PID"); err != nil { + return builtins.JournalEntry{}, err + } + if entry.Message, _, err = journalData(journal, "MESSAGE"); err != nil { + return builtins.JournalEntry{}, err + } + return entry, nil +} + +func journalData(journal journalHandle, field string) (string, bool, error) { + data, err := journal.GetData(field) + if err != nil { + if errors.Is(err, syscall.ENOENT) { + return "", false, nil + } + return "", false, fmt.Errorf("read journal field %s: %w", field, err) + } + prefix := field + "=" + if !strings.HasPrefix(data, prefix) { + return "", false, fmt.Errorf("journal returned malformed field %s", field) + } + value := strings.TrimPrefix(data, prefix) + if len(value) > maxJournalFieldSize { + value = value[:maxJournalFieldSize] + } + return value, true, nil +} diff --git a/internal/systemd/journal_reader_linux_cgo.go b/internal/systemd/journal_reader_linux_cgo.go new file mode 100644 index 00000000..69db9722 --- /dev/null +++ b/internal/systemd/journal_reader_linux_cgo.go @@ -0,0 +1,38 @@ +//go:build linux && cgo + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/coreos/go-systemd/v22/sdjournal" + + "github.com/DataDog/rshell/builtins" +) + +// ReadJournal opens only regular journal files belonging to the configured +// target machine and executes a bounded structured query against them. +func (c *Client) ReadJournal(ctx context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { + if err := validateJournalQuery(query); err != nil { + return err + } + if query.MaxEntries == 0 { + return nil + } + machineID, files, err := c.journalFiles() + if err != nil { + return err + } + journal, err := sdjournal.NewJournalFromFiles(files...) + if err != nil { + return fmt.Errorf("open systemd journal: %w", err) + } + defer journal.Close() + return readJournal(ctx, journal, machineID, query, yield) +} diff --git a/internal/systemd/journal_reader_test.go b/internal/systemd/journal_reader_test.go new file mode 100644 index 00000000..7463607a --- /dev/null +++ b/internal/systemd/journal_reader_test.go @@ -0,0 +1,248 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +type fakeJournalEntry struct { + realtimeUsec uint64 + fields map[string]string +} + +type fakeJournal struct { + calls []string + entries []fakeJournalEntry + position int + threshold uint64 + dataFields []string +} + +func (j *fakeJournal) AddMatch(match string) error { + j.calls = append(j.calls, "match "+match) + return nil +} + +func (j *fakeJournal) AddDisjunction() error { + j.calls = append(j.calls, "or") + return nil +} + +func (j *fakeJournal) AddConjunction() error { + j.calls = append(j.calls, "and") + return nil +} + +func (j *fakeJournal) FlushMatches() { + j.calls = append(j.calls, "flush") +} + +func (j *fakeJournal) Next() (uint64, error) { + if j.position+1 >= len(j.entries) { + j.position = len(j.entries) + return 0, nil + } + j.position++ + return 1, nil +} + +func (j *fakeJournal) Previous() (uint64, error) { + if j.position <= 0 { + j.position = -1 + return 0, nil + } + j.position-- + return 1, nil +} + +func (j *fakeJournal) PreviousSkip(skip uint64) (uint64, error) { + available := j.position + if available < 0 { + available = 0 + } + actual := int(skip) + if actual > available { + actual = available + } + j.position -= actual + return uint64(actual), nil +} + +func (j *fakeJournal) GetData(field string) (string, error) { + j.dataFields = append(j.dataFields, field) + if j.position < 0 || j.position >= len(j.entries) { + return "", fmt.Errorf("no current entry") + } + value, ok := j.entries[j.position].fields[field] + if !ok { + return "", fmt.Errorf("missing field: %w", syscall.ENOENT) + } + return field + "=" + value, nil +} + +func (j *fakeJournal) GetRealtimeUsec() (uint64, error) { + if j.position < 0 || j.position >= len(j.entries) { + return 0, fmt.Errorf("no current entry") + } + return j.entries[j.position].realtimeUsec, nil +} + +func (j *fakeJournal) SetDataThreshold(threshold uint64) error { + j.threshold = threshold + return nil +} + +func (j *fakeJournal) SeekHead() error { + j.calls = append(j.calls, "head") + j.position = -1 + return nil +} + +func (j *fakeJournal) SeekTail() error { + j.calls = append(j.calls, "tail") + j.position = len(j.entries) + return nil +} + +func TestAddJournalMatchesUsesRestrictedUnitExpansion(t *testing.T) { + journal := &fakeJournal{} + err := addJournalMatches(journal, "machine", "boot", builtins.JournalQuery{ + Units: []string{"api.service", "worker.service"}, + }) + require.NoError(t, err) + assert.Equal(t, []string{ + "match _SYSTEMD_UNIT=api.service", + "match _SYSTEMD_UNIT=worker.service", + "or", + "match _PID=1", + "match UNIT=api.service", + "match UNIT=worker.service", + "and", + "match _MACHINE_ID=machine", + "match _BOOT_ID=boot", + }, journal.calls) +} + +func TestAddJournalMatchesConstrainsKernelToTarget(t *testing.T) { + journal := &fakeJournal{} + err := addJournalMatches(journal, "machine", "boot", builtins.JournalQuery{Kernel: true}) + require.NoError(t, err) + assert.Equal(t, []string{ + "match _TRANSPORT=kernel", + "match _MACHINE_ID=machine", + "match _BOOT_ID=boot", + }, journal.calls) +} + +func TestReadJournalYieldsOnlySelectedFields(t *testing.T) { + machineID := "0123456789abcdef0123456789abcdef" + start := time.Unix(1_700_000_000, 0) + journal := &fakeJournal{ + position: -1, + entries: []fakeJournalEntry{ + { + realtimeUsec: uint64(start.Add(-time.Minute).UnixMicro()), + fields: map[string]string{ + "_MACHINE_ID": machineID, + "_HOSTNAME": "host", + "SYSLOG_IDENTIFIER": "api", + "_PID": "12", + "MESSAGE": "old", + "_CMDLINE": "secret-token", + }, + }, + { + realtimeUsec: uint64(start.UnixMicro()), + fields: map[string]string{ + "_MACHINE_ID": machineID, + "_HOSTNAME": "host", + "SYSLOG_IDENTIFIER": "api", + "_PID": "13", + "MESSAGE": "ready", + }, + }, + }, + } + + var entries []builtins.JournalEntry + err := readJournal(context.Background(), journal, machineID, builtins.JournalQuery{ + Units: []string{"api.service"}, + Since: start, + MaxEntries: 2, + }, func(entry builtins.JournalEntry) error { + entries = append(entries, entry) + return nil + }) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "ready", entries[0].Message) + assert.Equal(t, uint64(maxJournalFieldSize+64), journal.threshold) + assert.NotContains(t, journal.dataFields, "_CMDLINE") + assert.Contains(t, journal.calls, "head") +} + +func TestReadJournalFindsCurrentBootInTargetJournal(t *testing.T) { + machineID := "0123456789abcdef0123456789abcdef" + bootID := "abcdef0123456789abcdef0123456789" + journal := &fakeJournal{ + position: -1, + entries: []fakeJournalEntry{{ + realtimeUsec: 1, + fields: map[string]string{ + "_MACHINE_ID": machineID, + "_BOOT_ID": bootID, + "MESSAGE": "booted", + }, + }}, + } + + err := readJournal(context.Background(), journal, machineID, builtins.JournalQuery{ + Kernel: true, + CurrentBoot: true, + MaxEntries: 1, + }, func(builtins.JournalEntry) error { return nil }) + require.NoError(t, err) + assert.Contains(t, journal.calls, "flush") + assert.Contains(t, journal.calls, "match _BOOT_ID="+bootID) +} + +func TestReadJournalRejectsEntryFromAnotherMachine(t *testing.T) { + journal := &fakeJournal{ + position: -1, + entries: []fakeJournalEntry{{ + realtimeUsec: 1, + fields: map[string]string{ + "_MACHINE_ID": "ffffffffffffffffffffffffffffffff", + }, + }}, + } + err := readJournal(context.Background(), journal, "0123456789abcdef0123456789abcdef", builtins.JournalQuery{ + Kernel: true, + MaxEntries: 1, + }, func(builtins.JournalEntry) error { return nil }) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not match") +} + +func TestValidateJournalQueryRejectsUnboundedOrMixedScopes(t *testing.T) { + for _, query := range []builtins.JournalQuery{ + {MaxEntries: maxJournalEntries + 1, Kernel: true}, + {MaxEntries: 1}, + {MaxEntries: 1, Kernel: true, Units: []string{"api.service"}}, + } { + require.Error(t, validateJournalQuery(query)) + } +} diff --git a/internal/systemd/journal_reader_unsupported.go b/internal/systemd/journal_reader_unsupported.go new file mode 100644 index 00000000..a1c9f4bf --- /dev/null +++ b/internal/systemd/journal_reader_unsupported.go @@ -0,0 +1,27 @@ +//go:build !linux || !cgo + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// ReadJournal reports the platform requirement without attempting filesystem +// access. Journal reading uses libsystemd's sd-journal API on Linux. +func (c *Client) ReadJournal(_ context.Context, query builtins.JournalQuery, _ func(builtins.JournalEntry) error) error { + if err := validateJournalQuery(query); err != nil { + return err + } + if query.MaxEntries == 0 { + return nil + } + return fmt.Errorf("%w: journal reading requires Linux with cgo", builtins.ErrSystemdUnsupported) +} diff --git a/interp/api.go b/interp/api.go index 58629607..f2fb6fe3 100644 --- a/interp/api.go +++ b/interp/api.go @@ -91,6 +91,7 @@ type runnerConfig struct { // subshells. systemdTarget internalsystemd.Target systemdTargetConfigured bool + systemd *builtins.SystemdServices // maxExecutionTime bounds the duration of each Run call. Zero disables // the limit. When non-zero, Run derives a child context with this timeout. @@ -331,6 +332,7 @@ func New(opts ...RunnerOption) (*Runner, error) { if !r.systemdTargetConfigured { r.systemdTarget = internalsystemd.LocalTarget() } + r.systemd = &builtins.SystemdServices{Journal: internalsystemd.NewClient(r.systemdTarget)} r.proc = builtins.NewProcProvider(r.procPath) return r, nil } diff --git a/interp/runner_exec.go b/interp/runner_exec.go index 2e4bb71f..d65d788e 100644 --- a/interp/runner_exec.go +++ b/interp/runner_exec.go @@ -698,7 +698,9 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { // available", which is the closest analogue to bash's // "exec read fails because read is a builtin, not an // executable on PATH" behaviour. - Proc: r.proc, + Proc: r.proc, + Systemd: r.systemd, + RemediationMode: r.remediationMode, } if r.remediationMode && r.sandbox != nil { child.Truncate = func(ctx context.Context, path string, size int64, create bool) error { @@ -824,6 +826,7 @@ func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { return vr.Str, vr.IsSet() }, Proc: r.proc, + Systemd: r.systemd, RemediationMode: r.remediationMode, } if r.remediationMode && r.sandbox != nil { From be532a7efc4c4236f95876d7aaf41516aaf8a821 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:08:25 -0400 Subject: [PATCH 06/53] feat(journalctl): add bounded log queries --- analysis/symbols_builtins.go | 31 +- builtins/journalctl/journalctl.go | 278 ++++++++++++++++++ builtins/journalctl/journalctl_test.go | 275 +++++++++++++++++ builtins/systemd.go | 8 + internal/systemd/journal_reader.go | 10 +- internal/systemd/journal_reader_test.go | 2 +- interp/register_builtins.go | 2 + .../cmd/journalctl/errors/denied_unit.yaml | 11 + .../cmd/journalctl/errors/missing_scope.yaml | 11 + .../cmd/journalctl/errors/raw_match.yaml | 11 + tests/scenarios/cmd/journalctl/help/help.yaml | 14 + 11 files changed, 643 insertions(+), 10 deletions(-) create mode 100644 builtins/journalctl/journalctl.go create mode 100644 builtins/journalctl/journalctl_test.go create mode 100644 tests/scenarios/cmd/journalctl/errors/denied_unit.yaml create mode 100644 tests/scenarios/cmd/journalctl/errors/missing_scope.yaml create mode 100644 tests/scenarios/cmd/journalctl/errors/raw_match.yaml create mode 100644 tests/scenarios/cmd/journalctl/help/help.yaml diff --git a/analysis/symbols_builtins.go b/analysis/symbols_builtins.go index b0fb36be..e66813ee 100644 --- a/analysis/symbols_builtins.go +++ b/analysis/symbols_builtins.go @@ -194,6 +194,21 @@ var builtinPerCommandSymbols = map[string][]string{ "os.O_RDONLY", // 🟢 read-only file flag constant; cannot open files by itself. "strconv.ParseInt", // 🟢 string-to-int conversion with base/bit-size; pure function, no I/O. }, + "journalctl": { + "context.Context", // 🟢 deadline/cancellation plumbing; pure interface, no side effects. + "io.WriteString", // 🟠 writes formatted journal lines to callCtx.Stdout; no filesystem access, delegates to Write. + "io.Writer", // 🟢 interface type for the already-authorized output stream; no side effects. + "strconv.ParseInt", // 🟢 parses the bounded --lines value; pure function, no I/O. + "strings.Builder", // 🟢 constructs one bounded output line in memory; no I/O. + "time.Parse", // 🟢 parses an absolute RFC3339 timestamp; pure function, no I/O. + "time.ParseDuration", // 🟢 parses a bounded lookback duration; pure function, no I/O. + "time.ParseInLocation", // 🟢 parses a local timestamp using the runner's captured location; pure function, no I/O. + "time.RFC3339Nano", // 🟢 standard timestamp layout constant; no side effects. + "time.Time", // 🟢 time value type; pure data, no side effects. + "unicode.IsGraphic", // 🟢 identifies non-graphic runes that must be escaped at the output boundary; pure function, no I/O. + "unicode/utf8.DecodeRuneInString", // 🟢 decodes output text so control and malformed bytes can be escaped; pure function, no I/O. + "unicode/utf8.RuneError", // 🟢 replacement rune constant used to detect malformed UTF-8; no side effects. + }, "logrotate": { "context.Context", // 🟢 deadline/cancellation plumbing; pure interface, no side effects. "errors.Is", // 🟢 error comparison; pure function, no I/O. @@ -528,15 +543,17 @@ var builtinPerCommandSymbols = map[string][]string{ }, } -// callCtxAllFields lists every function-typed field of CallContext that the -// analyzer tracks. Plain data fields (Stdin, Stdout, Stderr, Now, InLoop, +// callCtxAllFields lists every elevated-capability field of CallContext that +// the analyzer tracks. Plain data fields (Stdin, Stdout, Stderr, Now, InLoop, // LastExitCode, Proc) are not tracked; they are universally available and -// carry no elevated capability. +// carry no elevated capability. Systemd is tracked despite being a pointer +// because it provides privileged host introspection backends. // // Every entry must match a field name declared in builtins/builtins.go. var callCtxAllFields = []string{ "AccessFile", "AllowedPathsList", + "AuthorizeSystemd", "CanonicalizeRootPrefix", "ChangeDir", "CommandAllowed", @@ -556,6 +573,7 @@ var callCtxAllFields = []string{ "RunCommandWithStdin", "SetVar", "StatFile", + "Systemd", "Truncate", "TruncateToZeroIfAtLeast", "WorkDir", @@ -636,6 +654,10 @@ var builtinPerCommandCallContextFields = map[string][]string{ "ip": { "PortableErr", }, + "journalctl": { + "AuthorizeSystemd", + "Systemd", + }, "logrotate": { "PortableErr", "TruncateToZeroIfAtLeast", @@ -846,7 +868,10 @@ var builtinAllowedSymbols = []string{ "time.Hour", // 🟢 constant representing one hour; no side effects. "time.Millisecond", // 🟢 constant representing one millisecond; no side effects. "time.Minute", // 🟢 constant representing one minute; no side effects. + "time.Parse", // 🟢 parses timestamps according to a caller-supplied layout; pure function, no I/O. "time.ParseDuration", // 🟢 parses Go duration strings (e.g. "1s"); pure function, no I/O. + "time.ParseInLocation", // 🟢 parses timestamps in a caller-supplied location; pure function, no I/O. + "time.RFC3339Nano", // 🟢 standard RFC3339 timestamp layout with optional fractional seconds; pure constant. "time.Second", // 🟢 constant representing one second; no side effects. "time.Time", // 🟢 time value type; pure data, no side effects. "time.Unix", // 🟢 constructs an absolute Time at Unix-epoch + (sec, nsec); pure constructor, no I/O or side effects. diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go new file mode 100644 index 00000000..a2e9f7f4 --- /dev/null +++ b/builtins/journalctl/journalctl.go @@ -0,0 +1,278 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +// Package journalctl implements a bounded systemd journal query builtin. +// +// Usage: journalctl (-u UNIT...|-k) [OPTION]... +// +// This is a deliberately restricted journalctl subset. Every log query must +// select one or more exact system units, or the current boot's kernel log. +// Arbitrary field matches, unrestricted journal reads, follow mode, alternate +// files/directories, cursors, and machine/namespace selection are not exposed. +// +// Accepted flags: +// +// -u, --unit=UNIT select an exact system unit; may be repeated +// -k, --dmesg select kernel messages from the current target boot +// -b, --boot restrict a unit query to the current target boot +// -n, --lines=COUNT show at most COUNT entries (default 100, maximum 1000) +// -S, --since=TIME show entries since an RFC3339 timestamp, local +// YYYY-MM-DD HH:MM:SS timestamp, or lookback duration +// -o, --output=FORMAT output format: short (default) or cat +// -h, --help print usage and exit +package journalctl + +import ( + "context" + "io" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/DataDog/rshell/builtins" +) + +const ( + shortTime = "Jan 02 15:04:05" + localSinceTime = "2006-01-02 15:04:05" +) + +// Cmd is the journalctl builtin command descriptor. +var Cmd = builtins.Command{ + Name: "journalctl", + Description: "query bounded systemd journal logs", + MakeFlags: makeFlags, +} + +type flags struct { + units *[]string + kernel *bool + boot *bool + lines *string + since *string + output *string + help *bool +} + +func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { + options := flags{ + units: fs.StringArrayP("unit", "u", nil, "show logs for exact UNIT (repeatable)"), + kernel: fs.BoolP("dmesg", "k", false, "show current-boot kernel messages"), + boot: fs.BoolP("boot", "b", false, "show messages from the current boot"), + lines: fs.StringP("lines", "n", "100", "show at most COUNT entries (maximum 1000)"), + since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), + output: fs.StringP("output", "o", "short", "output format: short or cat"), + help: fs.BoolP("help", "h", false, "print usage and exit"), + } + return options.run(fs) +} + +func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { + return func(ctx context.Context, callCtx *builtins.CallContext, args []string) builtins.Result { + if *options.help { + callCtx.Out("Usage: journalctl (-u UNIT...|-k) [OPTION]...\n") + callCtx.Out("Show a bounded selection of logs from the configured systemd target.\n") + callCtx.Out("Queries require exact unit scopes or the current-boot kernel scope.\n\n") + fs.SetOutput(callCtx.Stdout) + fs.PrintDefaults() + return builtins.Result{} + } + if len(args) > 0 { + callCtx.Errf("journalctl: unexpected argument %q; arbitrary journal matches are not supported\n", args[0]) + return builtins.Result{Code: 1} + } + if len(*options.units) > builtins.MaxJournalQueryUnits { + callCtx.Errf("journalctl: too many unit scopes (maximum %d)\n", builtins.MaxJournalQueryUnits) + return builtins.Result{Code: 1} + } + + units := uniqueUnits(*options.units) + if *options.kernel && len(units) > 0 { + callCtx.Errf("journalctl: --dmesg cannot be combined with --unit\n") + return builtins.Result{Code: 1} + } + if !*options.kernel && len(units) == 0 { + callCtx.Errf("journalctl: an exact --unit or --dmesg scope is required\n") + return builtins.Result{Code: 1} + } + + lines, ok := parseLineCount(*options.lines) + if !ok { + callCtx.Errf("journalctl: invalid line count %q (must be between 0 and %d)\n", *options.lines, builtins.MaxJournalQueryEntries) + return builtins.Result{Code: 1} + } + if *options.output != "short" && *options.output != "cat" { + callCtx.Errf("journalctl: unsupported output format %q (supported: short, cat)\n", *options.output) + return builtins.Result{Code: 1} + } + + var since time.Time + if fs.Changed("since") { + if since, ok = parseSince(*options.since, callCtx.Now); !ok { + callCtx.Errf("journalctl: invalid --since value %q; use RFC3339, YYYY-MM-DD HH:MM:SS, or a lookback duration such as 15m\n", *options.since) + return builtins.Result{Code: 1} + } + } + + operations := make([]builtins.SystemdOperation, 0, len(units)+1) + if *options.kernel { + operations = append(operations, builtins.SystemdOperation{ + Resource: builtins.SystemdResourceJournalKernel, + Action: builtins.SystemdActionRead, + }) + } else { + for _, unit := range units { + operations = append(operations, builtins.SystemdOperation{ + Resource: builtins.SystemdUnitResource(unit), + Action: builtins.SystemdActionRead, + }) + } + } + if callCtx.AuthorizeSystemd == nil { + callCtx.Errf("journalctl: systemd authorization capability is not available\n") + return builtins.Result{Code: 1} + } + if err := callCtx.AuthorizeSystemd(operations...); err != nil { + callCtx.Errf("journalctl: %s\n", err) + return builtins.Result{Code: 1} + } + if callCtx.Systemd == nil || callCtx.Systemd.Journal == nil { + callCtx.Errf("journalctl: systemd journal capability is not available\n") + return builtins.Result{Code: 1} + } + + query := builtins.JournalQuery{ + Units: units, + Kernel: *options.kernel, + CurrentBoot: *options.kernel || *options.boot, + Since: since, + MaxEntries: lines, + } + err := callCtx.Systemd.Journal.ReadJournal(ctx, query, func(entry builtins.JournalEntry) error { + return writeEntry(callCtx.Stdout, entry, *options.output) + }) + if err == nil || builtins.IsBrokenPipe(err) { + return builtins.Result{} + } + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } +} + +func uniqueUnits(units []string) []string { + unique := make([]string, 0, len(units)) + seen := make(map[string]struct{}, len(units)) + for _, unit := range units { + if _, exists := seen[unit]; exists { + continue + } + seen[unit] = struct{}{} + unique = append(unique, unit) + } + return unique +} + +func parseLineCount(value string) (int, bool) { + if value == "" || value[0] == '+' { + return 0, false + } + parsed, err := strconv.ParseInt(value, 10, 32) + if err != nil || parsed < 0 || parsed > builtins.MaxJournalQueryEntries { + return 0, false + } + return int(parsed), true +} + +func parseSince(value string, now time.Time) (time.Time, bool) { + if value == "" { + return time.Time{}, false + } + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { + return parsed, true + } + if !now.IsZero() { + if parsed, err := time.ParseInLocation(localSinceTime, value, now.Location()); err == nil { + return parsed, true + } + if lookback, err := time.ParseDuration(value); err == nil && lookback >= 0 { + return now.Add(-lookback), true + } + } + return time.Time{}, false +} + +func writeEntry(writer io.Writer, entry builtins.JournalEntry, output string) error { + var line strings.Builder + if output == "short" { + line.WriteString(entry.Timestamp.Format(shortTime)) + line.WriteByte(' ') + if entry.Hostname == "" { + line.WriteByte('-') + } else { + appendEscaped(&line, entry.Hostname) + } + line.WriteByte(' ') + if entry.Identifier == "" { + line.WriteString("journal") + } else { + appendEscaped(&line, entry.Identifier) + } + if entry.PID != "" { + line.WriteByte('[') + appendEscaped(&line, entry.PID) + line.WriteByte(']') + } + line.WriteString(": ") + } + appendEscaped(&line, entry.Message) + line.WriteByte('\n') + _, err := io.WriteString(writer, line.String()) + return err +} + +func appendEscaped(output *strings.Builder, value string) { + for value != "" { + r, size := utf8.DecodeRuneInString(value) + if r == utf8.RuneError && size == 1 { + appendHexEscape(output, 'x', uint32(value[0]), 2) + value = value[1:] + continue + } + switch r { + case '\n': + output.WriteString("\\n") + case '\r': + output.WriteString("\\r") + case '\t': + output.WriteString("\\t") + default: + if !unicode.IsGraphic(r) { + if r <= 0xff { + appendHexEscape(output, 'x', uint32(r), 2) + } else if r <= 0xffff { + appendHexEscape(output, 'u', uint32(r), 4) + } else { + appendHexEscape(output, 'U', uint32(r), 8) + } + } else { + output.WriteString(value[:size]) + } + } + value = value[size:] + } +} + +func appendHexEscape(output *strings.Builder, prefix byte, value uint32, width int) { + const hex = "0123456789abcdef" + output.WriteByte('\\') + output.WriteByte(prefix) + for shift := (width - 1) * 4; shift >= 0; shift -= 4 { + output.WriteByte(hex[(value>>shift)&0x0f]) + } +} diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go new file mode 100644 index 00000000..1c78bd55 --- /dev/null +++ b/builtins/journalctl/journalctl_test.go @@ -0,0 +1,275 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package journalctl + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "syscall" + "testing" + "time" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +type fakeJournalReader struct { + queries []builtins.JournalQuery + entries []builtins.JournalEntry + err error +} + +func (r *fakeJournalReader) ReadJournal(_ context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { + r.queries = append(r.queries, query) + for _, entry := range r.entries { + if err := yield(entry); err != nil { + return err + } + } + return r.err +} + +func runJournalctl(t *testing.T, args []string, callCtx *builtins.CallContext) builtins.Result { + t.Helper() + fs := pflag.NewFlagSet("journalctl", pflag.ContinueOnError) + fs.SetOutput(io.Discard) + handler := makeFlags(fs) + require.NoError(t, fs.Parse(args)) + return handler(context.Background(), callCtx, fs.Args()) +} + +func TestJournalctlBuildsBoundedUnitQuery(t *testing.T) { + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{Message: "ready"}}} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + callCtx := &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: now, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append([]builtins.SystemdOperation(nil), operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + } + + result := runJournalctl(t, []string{ + "-u", "api.service", + "-u", "worker.service", + "-u", "api.service", + "-b", + "-n", "2", + "--since", "15m", + "--output", "cat", + }, callCtx) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "ready\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{ + {Resource: builtins.SystemdUnitResource("api.service"), Action: builtins.SystemdActionRead}, + {Resource: builtins.SystemdUnitResource("worker.service"), Action: builtins.SystemdActionRead}, + }, authorized) + require.Len(t, reader.queries, 1) + assert.Equal(t, builtins.JournalQuery{ + Units: []string{"api.service", "worker.service"}, + CurrentBoot: true, + Since: now.Add(-15 * time.Minute), + MaxEntries: 2, + }, reader.queries[0]) +} + +func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{ + Timestamp: time.Date(2026, time.July, 14, 12, 34, 56, 0, time.UTC), + Hostname: "host\tname", + Identifier: "kernel\x1b", + PID: "\xff", + Message: "cafe\u0301\nline\x00\u202e", + }}} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"-k", "-n1"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Jul 14 12:34:56 host\\tname kernel\\x1b[\\xff]: cafe\u0301\\nline\\x00\\u202e\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Resource: builtins.SystemdResourceJournalKernel, + Action: builtins.SystemdActionRead, + }}, authorized) + require.Len(t, reader.queries, 1) + assert.True(t, reader.queries[0].Kernel) + assert.True(t, reader.queries[0].CurrentBoot) +} + +func TestJournalctlAuthorizesEveryScopeBeforeReading(t *testing.T) { + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + authorizationErr := errors.New("worker.service is denied") + result := runJournalctl(t, []string{"-u", "api.service", "-u", "worker.service"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + require.Len(t, operations, 2) + return authorizationErr + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), authorizationErr.Error()) + assert.Empty(t, reader.queries) +} + +func TestJournalctlRejectsUnboundedInputsBeforeAuthorization(t *testing.T) { + tests := []struct { + name string + args []string + }{ + {name: "missing scope"}, + {name: "mixed scopes", args: []string{"-u", "api.service", "-k"}}, + {name: "from head", args: []string{"-u", "api.service", "-n", "+10"}}, + {name: "negative lines", args: []string{"-u", "api.service", "--lines=-1"}}, + {name: "too many lines", args: []string{"-u", "api.service", "-n", "1001"}}, + {name: "structured output", args: []string{"-u", "api.service", "-o", "json"}}, + {name: "raw match", args: []string{"-u", "api.service", "_PID=1"}}, + {name: "bad since", args: []string{"-u", "api.service", "--since", "yesterday"}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + authorizeCalls := 0 + result := runJournalctl(t, test.args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + authorizeCalls++ + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.NotEmpty(t, stderr.String()) + assert.Zero(t, authorizeCalls) + assert.Empty(t, reader.queries) + }) + } +} + +func TestJournalctlRejectsDangerousJournalctlOptions(t *testing.T) { + Cmd.Register() + handler, ok := builtins.Lookup("journalctl") + require.True(t, ok) + + for _, args := range [][]string{ + {"--follow"}, + {"--file=/tmp/system.journal"}, + {"--directory=/var/log/journal"}, + {"--root=/host"}, + {"--machine=host"}, + {"--namespace=tenant"}, + {"--cursor=cursor"}, + {"--grep=secret"}, + {"--reverse"}, + {"--all"}, + } { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + var stdout, stderr bytes.Buffer + result := handler(context.Background(), &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + }, args) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), "unrecognized option") + }) + } +} + +func TestJournalctlLimitsRepeatedUnitScopes(t *testing.T) { + args := make([]string, 0, (builtins.MaxJournalQueryUnits+1)*2) + for i := 0; i <= builtins.MaxJournalQueryUnits; i++ { + args = append(args, "-u", "api.service") + } + var stdout, stderr bytes.Buffer + result := runJournalctl(t, args, &builtins.CallContext{Stdout: &stdout, Stderr: &stderr}) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "too many unit scopes") +} + +func TestJournalctlHelpDoesNotRequireSystemdCapability(t *testing.T) { + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--help"}, &builtins.CallContext{Stdout: &stdout, Stderr: &stderr}) + assert.Equal(t, uint8(0), result.Code) + assert.Contains(t, stdout.String(), "Usage: journalctl") + assert.Contains(t, stdout.String(), "--unit") + assert.Empty(t, stderr.String()) +} + +type brokenPipeWriter struct{} + +func (brokenPipeWriter) Write([]byte) (int, error) { + return 0, syscall.EPIPE +} + +func TestJournalctlTreatsBrokenPipeAsSuccess(t *testing.T) { + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{Message: "entry"}}} + var stderr bytes.Buffer + result := runJournalctl(t, []string{"-k"}, &builtins.CallContext{ + Stdout: brokenPipeWriter{}, + Stderr: &stderr, + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) +} + +func TestParseSince(t *testing.T) { + location := time.FixedZone("test", -4*60*60) + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, location) + + absolute, ok := parseSince("2026-07-14T15:00:00Z", now) + require.True(t, ok) + assert.Equal(t, time.Date(2026, time.July, 14, 15, 0, 0, 0, time.UTC), absolute) + + local, ok := parseSince("2026-07-14 10:30:00", now) + require.True(t, ok) + assert.Equal(t, time.Date(2026, time.July, 14, 10, 30, 0, 0, location), local) + + lookback, ok := parseSince("90m", now) + require.True(t, ok) + assert.Equal(t, now.Add(-90*time.Minute), lookback) + + _, ok = parseSince("-1h", now) + assert.False(t, ok) + _, ok = parseSince(strings.Repeat("9", 100), now) + assert.False(t, ok) +} diff --git a/builtins/systemd.go b/builtins/systemd.go index c7cb67d4..ffac64bb 100644 --- a/builtins/systemd.go +++ b/builtins/systemd.go @@ -15,6 +15,14 @@ import ( // requested systemd operation. var ErrSystemdUnsupported = errors.New("systemd operation is not supported") +const ( + // MaxJournalQueryEntries is the hard per-invocation entry bound shared by + // journal builtins and backends. + MaxJournalQueryEntries = 1000 + // MaxJournalQueryUnits bounds exact unit scopes before any backend work. + MaxJournalQueryUnits = 32 +) + // JournalQuery is the bounded, structured query accepted by the trusted // journal backend. Callers cannot provide raw journal matches or paths. type JournalQuery struct { diff --git a/internal/systemd/journal_reader.go b/internal/systemd/journal_reader.go index ffcfc89f..f903c6c4 100644 --- a/internal/systemd/journal_reader.go +++ b/internal/systemd/journal_reader.go @@ -17,8 +17,6 @@ import ( ) const ( - maxJournalEntries = 1000 - maxJournalUnits = 32 maxJournalFieldSize = 64 * 1024 maxBootSearch = 1024 ) @@ -39,8 +37,8 @@ type journalHandle interface { } func validateJournalQuery(query builtins.JournalQuery) error { - if query.MaxEntries < 0 || query.MaxEntries > maxJournalEntries { - return fmt.Errorf("journal query entry limit must be between 0 and %d", maxJournalEntries) + if query.MaxEntries < 0 || query.MaxEntries > builtins.MaxJournalQueryEntries { + return fmt.Errorf("journal query entry limit must be between 0 and %d", builtins.MaxJournalQueryEntries) } if query.Kernel && len(query.Units) > 0 { return fmt.Errorf("journal query cannot combine kernel and unit scopes") @@ -48,8 +46,8 @@ func validateJournalQuery(query builtins.JournalQuery) error { if !query.Kernel && len(query.Units) == 0 { return fmt.Errorf("journal query requires a kernel or unit scope") } - if len(query.Units) > maxJournalUnits { - return fmt.Errorf("journal query has too many units (maximum %d)", maxJournalUnits) + if len(query.Units) > builtins.MaxJournalQueryUnits { + return fmt.Errorf("journal query has too many units (maximum %d)", builtins.MaxJournalQueryUnits) } for _, unit := range query.Units { if unit == "" || len(unit) > 256 || strings.IndexByte(unit, 0) >= 0 { diff --git a/internal/systemd/journal_reader_test.go b/internal/systemd/journal_reader_test.go index 7463607a..d69964d2 100644 --- a/internal/systemd/journal_reader_test.go +++ b/internal/systemd/journal_reader_test.go @@ -239,7 +239,7 @@ func TestReadJournalRejectsEntryFromAnotherMachine(t *testing.T) { func TestValidateJournalQueryRejectsUnboundedOrMixedScopes(t *testing.T) { for _, query := range []builtins.JournalQuery{ - {MaxEntries: maxJournalEntries + 1, Kernel: true}, + {MaxEntries: builtins.MaxJournalQueryEntries + 1, Kernel: true}, {MaxEntries: 1}, {MaxEntries: 1, Kernel: true, Units: []string{"api.service"}}, } { diff --git a/interp/register_builtins.go b/interp/register_builtins.go index 6d16ce2f..8d766ef5 100644 --- a/interp/register_builtins.go +++ b/interp/register_builtins.go @@ -24,6 +24,7 @@ import ( "github.com/DataDog/rshell/builtins/head" "github.com/DataDog/rshell/builtins/help" "github.com/DataDog/rshell/builtins/ip" + "github.com/DataDog/rshell/builtins/journalctl" "github.com/DataDog/rshell/builtins/logrotate" "github.com/DataDog/rshell/builtins/ls" "github.com/DataDog/rshell/builtins/ping" @@ -66,6 +67,7 @@ func registerBuiltins() { head.Cmd, help.Cmd, ip.Cmd, + journalctl.Cmd, logrotate.Cmd, ls.Cmd, ping.Cmd, diff --git a/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml b/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml new file mode 100644 index 00000000..6b07d136 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml @@ -0,0 +1,11 @@ +# Scenario runners do not grant systemd capabilities by default. +skip_assert_against_bash: true +description: journalctl enforces the shared systemd allowlist +input: + script: |+ + journalctl -u api.service +expect: + stdout: |+ + stderr: |+ + journalctl: systemd resource "unit:api.service" is not allowed for action "read" + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/missing_scope.yaml b/tests/scenarios/cmd/journalctl/errors/missing_scope.yaml new file mode 100644 index 00000000..e8b57969 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/missing_scope.yaml @@ -0,0 +1,11 @@ +# Bare journal reads are intentionally unavailable in rshell. +skip_assert_against_bash: true +description: journalctl requires an exact unit or kernel scope +input: + script: |+ + journalctl +expect: + stdout: |+ + stderr: |+ + journalctl: an exact --unit or --dmesg scope is required + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/raw_match.yaml b/tests/scenarios/cmd/journalctl/errors/raw_match.yaml new file mode 100644 index 00000000..c11b693a --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/raw_match.yaml @@ -0,0 +1,11 @@ +# Raw fields could bypass the resource capability model and are rejected. +skip_assert_against_bash: true +description: journalctl rejects arbitrary field matches +input: + script: |+ + journalctl -u api.service _PID=1 +expect: + stdout: |+ + stderr: |+ + journalctl: unexpected argument "_PID=1"; arbitrary journal matches are not supported + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/help/help.yaml b/tests/scenarios/cmd/journalctl/help/help.yaml new file mode 100644 index 00000000..f9020b5b --- /dev/null +++ b/tests/scenarios/cmd/journalctl/help/help.yaml @@ -0,0 +1,14 @@ +# rshell journalctl is a restricted builtin and may not be installed in the Bash image. +skip_assert_against_bash: true +description: journalctl help describes the bounded query surface +input: + script: |+ + journalctl --help +expect: + stdout_contains: + - "Usage: journalctl (-u UNIT...|-k) [OPTION]..." + - "--unit" + - "--dmesg" + - "--lines" + stderr: |+ + exit_code: 0 From c4334de93108459ce1298bbd371ad56adb496f03 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:11:29 -0400 Subject: [PATCH 07/53] feat(journalctl): report journal disk usage --- analysis/symbols_builtins.go | 1 + builtins/journalctl/journalctl.go | 56 +++++++++++++++ builtins/journalctl/journalctl_test.go | 69 +++++++++++++++++++ builtins/systemd.go | 15 +++- internal/systemd/journal_files.go | 3 - internal/systemd/journal_reader_linux_cgo.go | 3 + internal/systemd/journal_usage_unix.go | 56 +++++++++++++++ internal/systemd/journal_usage_unix_test.go | 67 ++++++++++++++++++ internal/systemd/journal_usage_unsupported.go | 20 ++++++ interp/api.go | 6 +- .../cmd/journalctl/errors/denied_storage.yaml | 11 +++ 11 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 internal/systemd/journal_usage_unix.go create mode 100644 internal/systemd/journal_usage_unix_test.go create mode 100644 internal/systemd/journal_usage_unsupported.go create mode 100644 tests/scenarios/cmd/journalctl/errors/denied_storage.yaml diff --git a/analysis/symbols_builtins.go b/analysis/symbols_builtins.go index e66813ee..7935f952 100644 --- a/analysis/symbols_builtins.go +++ b/analysis/symbols_builtins.go @@ -196,6 +196,7 @@ var builtinPerCommandSymbols = map[string][]string{ }, "journalctl": { "context.Context", // 🟢 deadline/cancellation plumbing; pure interface, no side effects. + "fmt.Sprintf", // 🟢 formats bounded journal storage counts in memory; no I/O. "io.WriteString", // 🟠 writes formatted journal lines to callCtx.Stdout; no filesystem access, delegates to Write. "io.Writer", // 🟢 interface type for the already-authorized output stream; no side effects. "strconv.ParseInt", // 🟢 parses the bounded --lines value; pure function, no I/O. diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index a2e9f7f4..a0dedf1d 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -21,11 +21,13 @@ // -S, --since=TIME show entries since an RFC3339 timestamp, local // YYYY-MM-DD HH:MM:SS timestamp, or lookback duration // -o, --output=FORMAT output format: short (default) or cat +// --disk-usage show allocated journal storage and exit // -h, --help print usage and exit package journalctl import ( "context" + "fmt" "io" "strconv" "strings" @@ -55,6 +57,7 @@ type flags struct { lines *string since *string output *string + usage *bool help *bool } @@ -66,6 +69,7 @@ func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { lines: fs.StringP("lines", "n", "100", "show at most COUNT entries (maximum 1000)"), since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), output: fs.StringP("output", "o", "short", "output format: short or cat"), + usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), help: fs.BoolP("help", "h", false, "print usage and exit"), } return options.run(fs) @@ -85,6 +89,9 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { callCtx.Errf("journalctl: unexpected argument %q; arbitrary journal matches are not supported\n", args[0]) return builtins.Result{Code: 1} } + if *options.usage { + return runDiskUsage(ctx, callCtx, fs) + } if len(*options.units) > builtins.MaxJournalQueryUnits { callCtx.Errf("journalctl: too many unit scopes (maximum %d)\n", builtins.MaxJournalQueryUnits) return builtins.Result{Code: 1} @@ -165,6 +172,55 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { } } +func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { + for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output"} { + if fs.Changed(flagName) { + callCtx.Errf("journalctl: --disk-usage cannot be combined with --%s\n", flagName) + return builtins.Result{Code: 1} + } + } + if callCtx.AuthorizeSystemd == nil { + callCtx.Errf("journalctl: systemd authorization capability is not available\n") + return builtins.Result{Code: 1} + } + err := callCtx.AuthorizeSystemd(builtins.SystemdOperation{ + Resource: builtins.SystemdResourceJournalStorage, + Action: builtins.SystemdActionRead, + }) + if err != nil { + callCtx.Errf("journalctl: %s\n", err) + return builtins.Result{Code: 1} + } + if callCtx.Systemd == nil || callCtx.Systemd.JournalStorage == nil { + callCtx.Errf("journalctl: systemd journal storage capability is not available\n") + return builtins.Result{Code: 1} + } + usage, err := callCtx.Systemd.JournalStorage.JournalDiskUsage(ctx) + if err != nil { + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } + callCtx.Outf("Archived and active journals take up %s in the file system.\n", formatUsage(usage.Bytes)) + return builtins.Result{} +} + +func formatUsage(bytes uint64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%dB", bytes) + } + value := float64(bytes) + for _, suffix := range []string{"K", "M", "G", "T", "P", "E"} { + value /= unit + if value < unit || suffix == "E" { + return fmt.Sprintf("%.1f%s", value, suffix) + } + } + return fmt.Sprintf("%dB", bytes) +} + func uniqueUnits(units []string) []string { unique := make([]string, 0, len(units)) seen := make(map[string]struct{}, len(units)) diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index 1c78bd55..a2d9ef06 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -28,6 +28,17 @@ type fakeJournalReader struct { err error } +type fakeJournalStorage struct { + usage builtins.JournalUsage + err error + calls int +} + +func (s *fakeJournalStorage) JournalDiskUsage(context.Context) (builtins.JournalUsage, error) { + s.calls++ + return s.usage, s.err +} + func (r *fakeJournalReader) ReadJournal(_ context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { r.queries = append(r.queries, query) for _, entry := range r.entries { @@ -121,6 +132,64 @@ func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { assert.True(t, reader.queries[0].CurrentBoot) } +func TestJournalctlDiskUsageUsesStorageReadCapability(t *testing.T) { + storage := &fakeJournalStorage{usage: builtins.JournalUsage{Bytes: 5 * 1024 * 1024, Files: 3}} + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"--disk-usage"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader, JournalStorage: storage}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Archived and active journals take up 5.0M in the file system.\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Resource: builtins.SystemdResourceJournalStorage, + Action: builtins.SystemdActionRead, + }}, authorized) + assert.Equal(t, 1, storage.calls) + assert.Empty(t, reader.queries) +} + +func TestJournalctlDiskUsageIsExclusive(t *testing.T) { + for _, args := range [][]string{ + {"--disk-usage", "-u", "api.service"}, + {"--disk-usage", "-k"}, + {"--disk-usage", "-b"}, + {"--disk-usage", "-n", "10"}, + {"--disk-usage", "--since", "1h"}, + {"--disk-usage", "-o", "cat"}, + } { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + storage := &fakeJournalStorage{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Systemd: &builtins.SystemdServices{JournalStorage: storage}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), "cannot be combined") + assert.Zero(t, storage.calls) + }) + } +} + +func TestFormatUsage(t *testing.T) { + assert.Equal(t, "0B", formatUsage(0)) + assert.Equal(t, "1023B", formatUsage(1023)) + assert.Equal(t, "1.0K", formatUsage(1024)) + assert.Equal(t, "1.5M", formatUsage(1536*1024)) +} + func TestJournalctlAuthorizesEveryScopeBeforeReading(t *testing.T) { reader := &fakeJournalReader{} var stdout, stderr bytes.Buffer diff --git a/builtins/systemd.go b/builtins/systemd.go index ffac64bb..c288171c 100644 --- a/builtins/systemd.go +++ b/builtins/systemd.go @@ -48,9 +48,22 @@ type JournalReader interface { ReadJournal(ctx context.Context, query JournalQuery, yield func(JournalEntry) error) error } +// JournalUsage is the allocated storage consumed by the selected target's +// active and archived journal files. +type JournalUsage struct { + Bytes uint64 + Files int +} + +// JournalStorageReader exposes read-only journal storage metadata. +type JournalStorageReader interface { + JournalDiskUsage(ctx context.Context) (JournalUsage, error) +} + // SystemdServices contains the trusted backends available to systemd-aware // builtins. Additional manager and journal-maintenance interfaces can be added // here without exposing transports to command implementations. type SystemdServices struct { - Journal JournalReader + Journal JournalReader + JournalStorage JournalStorageReader } diff --git a/internal/systemd/journal_files.go b/internal/systemd/journal_files.go index 5fdeea44..de90ab1c 100644 --- a/internal/systemd/journal_files.go +++ b/internal/systemd/journal_files.go @@ -74,9 +74,6 @@ func (c *Client) journalFiles() (string, []string, error) { } } - if len(files) == 0 { - return "", nil, fmt.Errorf("no journal files found for machine %s", machineID) - } sort.Strings(files) return machineID, files, nil } diff --git a/internal/systemd/journal_reader_linux_cgo.go b/internal/systemd/journal_reader_linux_cgo.go index 69db9722..ebbe1f63 100644 --- a/internal/systemd/journal_reader_linux_cgo.go +++ b/internal/systemd/journal_reader_linux_cgo.go @@ -29,6 +29,9 @@ func (c *Client) ReadJournal(ctx context.Context, query builtins.JournalQuery, y if err != nil { return err } + if len(files) == 0 { + return fmt.Errorf("no journal files found for machine %s", machineID) + } journal, err := sdjournal.NewJournalFromFiles(files...) if err != nil { return fmt.Errorf("open systemd journal: %w", err) diff --git a/internal/systemd/journal_usage_unix.go b/internal/systemd/journal_usage_unix.go new file mode 100644 index 00000000..bb4ffa33 --- /dev/null +++ b/internal/systemd/journal_usage_unix.go @@ -0,0 +1,56 @@ +//go:build linux || darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + "math" + "os" + "syscall" + + "github.com/DataDog/rshell/builtins" +) + +const journalBlockSize = 512 + +// JournalDiskUsage reports allocated blocks, matching journalctl's disk-usage +// semantics more closely than summing logical file lengths. +func (c *Client) JournalDiskUsage(ctx context.Context) (builtins.JournalUsage, error) { + _, files, err := c.journalFiles() + if err != nil { + return builtins.JournalUsage{}, err + } + usage := builtins.JournalUsage{Files: len(files)} + for _, path := range files { + if err := ctx.Err(); err != nil { + return builtins.JournalUsage{}, err + } + info, err := os.Lstat(path) + if err != nil { + return builtins.JournalUsage{}, fmt.Errorf("inspect journal file %q: %w", path, err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return builtins.JournalUsage{}, fmt.Errorf("journal file %q changed during usage scan", path) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Blocks < 0 { + return builtins.JournalUsage{}, fmt.Errorf("journal file %q has unavailable allocation metadata", path) + } + blocks := uint64(stat.Blocks) + if blocks > math.MaxUint64/journalBlockSize { + return builtins.JournalUsage{}, fmt.Errorf("journal allocation overflow for %q", path) + } + allocated := blocks * journalBlockSize + if usage.Bytes > math.MaxUint64-allocated { + return builtins.JournalUsage{}, fmt.Errorf("journal allocation total overflow") + } + usage.Bytes += allocated + } + return usage, nil +} diff --git a/internal/systemd/journal_usage_unix_test.go b/internal/systemd/journal_usage_unix_test.go new file mode 100644 index 00000000..7e17207b --- /dev/null +++ b/internal/systemd/journal_usage_unix_test.go @@ -0,0 +1,67 @@ +//go:build linux || darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalDiskUsageCountsAllocatedJournalFiles(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + journalDir := filepath.Join(root, "journal") + machineDir := filepath.Join(journalDir, machineID) + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), make([]byte, 8192), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(machineDir, "ignored.txt"), make([]byte, 8192), 0o600)) + + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) + usage, err := client.JournalDiskUsage(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, usage.Files) + assert.Greater(t, usage.Bytes, uint64(0)) +} + +func TestJournalDiskUsageReturnsZeroForEmptyTarget(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + + client := NewClient(Target{JournalDirs: []string{filepath.Join(root, "missing")}, MachineIDPath: machineIDPath}) + usage, err := client.JournalDiskUsage(context.Background()) + require.NoError(t, err) + assert.Zero(t, usage.Files) + assert.Zero(t, usage.Bytes) +} + +func TestJournalDiskUsageHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + journalDir := filepath.Join(root, "journal") + machineDir := filepath.Join(journalDir, machineID) + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), []byte("data"), 0o600)) + + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) + _, err := client.JournalDiskUsage(ctx) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/internal/systemd/journal_usage_unsupported.go b/internal/systemd/journal_usage_unsupported.go new file mode 100644 index 00000000..79e9308c --- /dev/null +++ b/internal/systemd/journal_usage_unsupported.go @@ -0,0 +1,20 @@ +//go:build !linux && !darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// JournalDiskUsage requires Unix allocation metadata. +func (c *Client) JournalDiskUsage(context.Context) (builtins.JournalUsage, error) { + return builtins.JournalUsage{}, fmt.Errorf("%w: journal disk usage requires Linux or macOS", builtins.ErrSystemdUnsupported) +} diff --git a/interp/api.go b/interp/api.go index f2fb6fe3..7a2d8a42 100644 --- a/interp/api.go +++ b/interp/api.go @@ -332,7 +332,11 @@ func New(opts ...RunnerOption) (*Runner, error) { if !r.systemdTargetConfigured { r.systemdTarget = internalsystemd.LocalTarget() } - r.systemd = &builtins.SystemdServices{Journal: internalsystemd.NewClient(r.systemdTarget)} + systemdClient := internalsystemd.NewClient(r.systemdTarget) + r.systemd = &builtins.SystemdServices{ + Journal: systemdClient, + JournalStorage: systemdClient, + } r.proc = builtins.NewProcProvider(r.procPath) return r, nil } diff --git a/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml b/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml new file mode 100644 index 00000000..b6b8703a --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml @@ -0,0 +1,11 @@ +# Scenario runners do not grant systemd capabilities by default. +skip_assert_against_bash: true +description: journalctl disk usage enforces the shared storage allowlist +input: + script: |+ + journalctl --disk-usage +expect: + stdout: |+ + stderr: |+ + journalctl: systemd resource "journal:storage" is not allowed for action "read" + exit_code: 1 From 9c19507af6cfcc901ecfed71ecf7388dc380d41b Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:16:16 -0400 Subject: [PATCH 08/53] feat(systemd): add journal vacuum safety policy --- cmd/rshell/main.go | 30 +++++++++ cmd/rshell/main_test.go | 28 +++++++++ internal/systemd/client.go | 12 +++- internal/systemd/journal_files_test.go | 4 +- internal/systemd/journal_usage_unix_test.go | 6 +- internal/systemd/vacuum_policy.go | 40 ++++++++++++ interp/api.go | 3 +- interp/journal_vacuum.go | 29 +++++++++ interp/journal_vacuum_test.go | 69 +++++++++++++++++++++ 9 files changed, 212 insertions(+), 9 deletions(-) create mode 100644 internal/systemd/vacuum_policy.go create mode 100644 interp/journal_vacuum.go create mode 100644 interp/journal_vacuum_test.go diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index 6afbb5a6..df6f24c1 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -50,6 +50,11 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. journalSocket string systemBusSocket string journalRuntime string + vacuumMinAge time.Duration + vacuumMinFiles int + vacuumMinBytes uint64 + vacuumMaxFiles int + vacuumMaxBytes uint64 mode string ) @@ -111,6 +116,21 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. configuredJournalDirs = strings.Split(journalDirs, ",") } systemdTargetSet := systemdRoot != "" || journalDirs != "" || machineIDPath != "" || journalSocket != "" || systemBusSocket != "" || journalRuntime != "" + vacuumPolicySet := cmd.Flags().Changed("journal-vacuum-min-age") || + cmd.Flags().Changed("journal-vacuum-min-files") || + cmd.Flags().Changed("journal-vacuum-min-bytes") || + cmd.Flags().Changed("journal-vacuum-max-delete-files") || + cmd.Flags().Changed("journal-vacuum-max-delete-bytes") + var vacuumPolicy *interp.JournalVacuumPolicy + if vacuumPolicySet { + vacuumPolicy = &interp.JournalVacuumPolicy{ + MinRetentionAge: vacuumMinAge, + MinRetainedFiles: vacuumMinFiles, + MinRetainedBytes: vacuumMinBytes, + MaxDeletedFiles: vacuumMaxFiles, + MaxDeletedBytes: vacuumMaxBytes, + } + } execOpts := executeOpts{ allowedPaths: paths, @@ -128,6 +148,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. JournalRuntimeDir: journalRuntime, }, systemdTargetSet: systemdTargetSet, + vacuumPolicy: vacuumPolicy, mode: parsedMode, } @@ -189,6 +210,11 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().StringVar(&journalSocket, "systemd-journal-socket", "", "journald Varlink socket for an explicit systemd target") cmd.Flags().StringVar(&systemBusSocket, "systemd-bus-socket", "", "system D-Bus socket for an explicit systemd target") cmd.Flags().StringVar(&journalRuntime, "systemd-runtime-dir", "", "journald runtime directory for legacy control acknowledgements") + cmd.Flags().DurationVar(&vacuumMinAge, "journal-vacuum-min-age", 0, "minimum age of archived journals eligible for cleanup (required to enable cleanup)") + cmd.Flags().IntVar(&vacuumMinFiles, "journal-vacuum-min-files", 0, "minimum archived journal files retained across cleanup") + cmd.Flags().Uint64Var(&vacuumMinBytes, "journal-vacuum-min-bytes", 0, "minimum allocated archived journal bytes retained across cleanup") + cmd.Flags().IntVar(&vacuumMaxFiles, "journal-vacuum-max-delete-files", 0, "maximum archived journal files deleted per invocation (required to enable cleanup)") + cmd.Flags().Uint64Var(&vacuumMaxBytes, "journal-vacuum-max-delete-bytes", 0, "maximum allocated journal bytes deleted per invocation (required to enable cleanup)") cmd.Flags().StringVar(&mode, "mode", "read-only", "shell execution mode: read-only (default) or remediation (enables file-target output redirections within :rw AllowedPaths roots)") if err := cmd.ExecuteContext(ctx); err != nil { @@ -263,6 +289,7 @@ type executeOpts struct { procPath string systemdTarget interp.SystemdTargetConfig systemdTargetSet bool + vacuumPolicy *interp.JournalVacuumPolicy mode interp.Mode } @@ -303,6 +330,9 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i if opts.systemdTargetSet { runOpts = append(runOpts, interp.WithSystemdTarget(opts.systemdTarget)) } + if opts.vacuumPolicy != nil { + runOpts = append(runOpts, interp.WithJournalVacuumPolicy(*opts.vacuumPolicy)) + } if opts.mode != "" { runOpts = append(runOpts, interp.WithMode(opts.mode)) } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index bb0733e8..6434e647 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -192,6 +192,9 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "--systemd-journal-socket") assert.Contains(t, stdout, "--systemd-bus-socket") assert.Contains(t, stdout, "--systemd-runtime-dir") + assert.Contains(t, stdout, "--journal-vacuum-min-age") + assert.Contains(t, stdout, "--journal-vacuum-max-delete-files") + assert.Contains(t, stdout, "--journal-vacuum-max-delete-bytes") assert.NotContains(t, stdout, "--command", "-c/--command should be hidden from help") } @@ -409,6 +412,31 @@ func TestExplicitSystemdTargetRequiresMachineIDPath(t *testing.T) { assert.Contains(t, stderr, "machine ID path is required") } +func TestJournalVacuumPolicyFlags(t *testing.T) { + code, stdout, stderr := runCLI(t, + "--allow-all-commands", + "--journal-vacuum-min-age", "24h", + "--journal-vacuum-min-files", "2", + "--journal-vacuum-min-bytes", "1048576", + "--journal-vacuum-max-delete-files", "4", + "--journal-vacuum-max-delete-bytes", "8388608", + "-c", `echo hello`, + ) + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Empty(t, stderr) +} + +func TestJournalVacuumPolicyFlagsRejectIncompletePolicy(t *testing.T) { + code, _, stderr := runCLI(t, + "--allow-all-commands", + "--journal-vacuum-min-age", "24h", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Contains(t, stderr, "maximum deleted files") +} + func TestAllowAllCommandsFlag(t *testing.T) { code, stdout, _ := runCLI(t, "--allow-all-commands", "-c", `echo hello`) assert.Equal(t, 0, code) diff --git a/internal/systemd/client.go b/internal/systemd/client.go index 49d2d876..ec5b773e 100644 --- a/internal/systemd/client.go +++ b/internal/systemd/client.go @@ -9,11 +9,17 @@ package systemd // Target paths are copied at construction so runner configuration remains // immutable while commands execute. type Client struct { - target Target + target Target + vacuumPolicy *JournalVacuumPolicy } // NewClient creates a client for one resolved systemd target. -func NewClient(target Target) *Client { +func NewClient(target Target, vacuumPolicy *JournalVacuumPolicy) *Client { target.JournalDirs = append([]string(nil), target.JournalDirs...) - return &Client{target: target} + client := &Client{target: target} + if vacuumPolicy != nil { + policyCopy := *vacuumPolicy + client.vacuumPolicy = &policyCopy + } + return client } diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go index ecb09111..5d3f46a1 100644 --- a/internal/systemd/journal_files_test.go +++ b/internal/systemd/journal_files_test.go @@ -62,7 +62,7 @@ func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { client := NewClient(Target{ JournalDirs: []string{secondBase, firstBase, filepath.Join(root, "missing")}, MachineIDPath: machineIDPath, - }) + }, nil) gotMachineID, files, err := client.journalFiles() require.NoError(t, err) assert.Equal(t, machineID, gotMachineID) @@ -70,7 +70,7 @@ func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { } func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { - client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}) + client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}, nil) _, _, err := client.journalFiles() require.Error(t, err) assert.Contains(t, err.Error(), "journal directories are not configured") diff --git a/internal/systemd/journal_usage_unix_test.go b/internal/systemd/journal_usage_unix_test.go index 7e17207b..83bf84e9 100644 --- a/internal/systemd/journal_usage_unix_test.go +++ b/internal/systemd/journal_usage_unix_test.go @@ -28,7 +28,7 @@ func TestJournalDiskUsageCountsAllocatedJournalFiles(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), make([]byte, 8192), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(machineDir, "ignored.txt"), make([]byte, 8192), 0o600)) - client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}, nil) usage, err := client.JournalDiskUsage(context.Background()) require.NoError(t, err) assert.Equal(t, 1, usage.Files) @@ -41,7 +41,7 @@ func TestJournalDiskUsageReturnsZeroForEmptyTarget(t *testing.T) { machineIDPath := filepath.Join(root, "machine-id") require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) - client := NewClient(Target{JournalDirs: []string{filepath.Join(root, "missing")}, MachineIDPath: machineIDPath}) + client := NewClient(Target{JournalDirs: []string{filepath.Join(root, "missing")}, MachineIDPath: machineIDPath}, nil) usage, err := client.JournalDiskUsage(context.Background()) require.NoError(t, err) assert.Zero(t, usage.Files) @@ -61,7 +61,7 @@ func TestJournalDiskUsageHonorsCancellation(t *testing.T) { require.NoError(t, os.MkdirAll(machineDir, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), []byte("data"), 0o600)) - client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}, nil) _, err := client.JournalDiskUsage(ctx) require.ErrorIs(t, err, context.Canceled) } diff --git a/internal/systemd/vacuum_policy.go b/internal/systemd/vacuum_policy.go new file mode 100644 index 00000000..a80d784c --- /dev/null +++ b/internal/systemd/vacuum_policy.go @@ -0,0 +1,40 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "time" +) + +// JournalVacuumPolicy is a trusted operator ceiling for journal cleanup. +// Script-provided vacuum thresholds can only narrow this policy. +type JournalVacuumPolicy struct { + MinRetentionAge time.Duration + MinRetainedFiles int + MinRetainedBytes uint64 + MaxDeletedFiles int + MaxDeletedBytes uint64 +} + +// Validate rejects policies that would leave repeated cleanup invocations +// unbounded. A positive age floor is mandatory even when file/byte floors are +// also configured. +func (policy JournalVacuumPolicy) Validate() error { + if policy.MinRetentionAge <= 0 { + return fmt.Errorf("minimum retention age must be greater than zero") + } + if policy.MinRetainedFiles < 0 || policy.MinRetainedFiles > maxJournalFiles { + return fmt.Errorf("minimum retained files must be between 0 and %d", maxJournalFiles) + } + if policy.MaxDeletedFiles <= 0 || policy.MaxDeletedFiles > maxJournalFiles { + return fmt.Errorf("maximum deleted files must be between 1 and %d", maxJournalFiles) + } + if policy.MaxDeletedBytes == 0 { + return fmt.Errorf("maximum deleted bytes must be greater than zero") + } + return nil +} diff --git a/interp/api.go b/interp/api.go index 7a2d8a42..08974836 100644 --- a/interp/api.go +++ b/interp/api.go @@ -92,6 +92,7 @@ type runnerConfig struct { systemdTarget internalsystemd.Target systemdTargetConfigured bool systemd *builtins.SystemdServices + journalVacuumPolicy *internalsystemd.JournalVacuumPolicy // maxExecutionTime bounds the duration of each Run call. Zero disables // the limit. When non-zero, Run derives a child context with this timeout. @@ -332,7 +333,7 @@ func New(opts ...RunnerOption) (*Runner, error) { if !r.systemdTargetConfigured { r.systemdTarget = internalsystemd.LocalTarget() } - systemdClient := internalsystemd.NewClient(r.systemdTarget) + systemdClient := internalsystemd.NewClient(r.systemdTarget, r.journalVacuumPolicy) r.systemd = &builtins.SystemdServices{ Journal: systemdClient, JournalStorage: systemdClient, diff --git a/interp/journal_vacuum.go b/interp/journal_vacuum.go new file mode 100644 index 00000000..b4b190b5 --- /dev/null +++ b/interp/journal_vacuum.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package interp + +import ( + "fmt" + + internalsystemd "github.com/DataDog/rshell/internal/systemd" +) + +// JournalVacuumPolicy is the trusted retention floor and per-invocation +// deletion ceiling used by journal cleanup operations. +type JournalVacuumPolicy = internalsystemd.JournalVacuumPolicy + +// WithJournalVacuumPolicy enables bounded journal cleanup. Without this +// option, cleanup remains disabled even when journal:storage/clean is granted. +func WithJournalVacuumPolicy(policy JournalVacuumPolicy) RunnerOption { + return func(r *Runner) error { + if err := policy.Validate(); err != nil { + return fmt.Errorf("WithJournalVacuumPolicy: %w", err) + } + policyCopy := policy + r.journalVacuumPolicy = &policyCopy + return nil + } +} diff --git a/interp/journal_vacuum_test.go b/interp/journal_vacuum_test.go new file mode 100644 index 00000000..2a8b6a13 --- /dev/null +++ b/interp/journal_vacuum_test.go @@ -0,0 +1,69 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package interp + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithJournalVacuumPolicyCopiesValidatedPolicy(t *testing.T) { + policy := JournalVacuumPolicy{ + MinRetentionAge: 24 * time.Hour, + MinRetainedFiles: 2, + MinRetainedBytes: 64 * 1024 * 1024, + MaxDeletedFiles: 4, + MaxDeletedBytes: 128 * 1024 * 1024, + } + runner, err := New(WithJournalVacuumPolicy(policy)) + require.NoError(t, err) + defer runner.Close() + + policy.MinRetainedFiles = 0 + require.NotNil(t, runner.journalVacuumPolicy) + assert.Equal(t, 2, runner.journalVacuumPolicy.MinRetainedFiles) +} + +func TestWithJournalVacuumPolicyIsDisabledByDefault(t *testing.T) { + runner, err := New() + require.NoError(t, err) + defer runner.Close() + assert.Nil(t, runner.journalVacuumPolicy) +} + +func TestWithJournalVacuumPolicyRejectsUnboundedPolicies(t *testing.T) { + valid := JournalVacuumPolicy{ + MinRetentionAge: time.Hour, + MaxDeletedFiles: 1, + MaxDeletedBytes: 1, + } + tests := []struct { + name string + mutate func(*JournalVacuumPolicy) + needle string + }{ + {name: "no retention age", mutate: func(p *JournalVacuumPolicy) { p.MinRetentionAge = 0 }, needle: "retention age"}, + {name: "negative retained files", mutate: func(p *JournalVacuumPolicy) { p.MinRetainedFiles = -1 }, needle: "retained files"}, + {name: "no file ceiling", mutate: func(p *JournalVacuumPolicy) { p.MaxDeletedFiles = 0 }, needle: "deleted files"}, + {name: "no byte ceiling", mutate: func(p *JournalVacuumPolicy) { p.MaxDeletedBytes = 0 }, needle: "deleted bytes"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + policy := valid + test.mutate(&policy) + runner, err := New(WithJournalVacuumPolicy(policy)) + if runner != nil { + runner.Close() + } + require.Error(t, err) + assert.Contains(t, err.Error(), test.needle) + }) + } +} From 4c5d199104a56c519183ff5e4cb29923aa092dc1 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:21:25 -0400 Subject: [PATCH 09/53] feat(systemd): add policy-bounded journal vacuum --- builtins/systemd.go | 22 ++ docs/RULES.md | 26 +- internal/systemd/journal_file_unix.go | 43 +++ internal/systemd/journal_usage_unix.go | 18 +- internal/systemd/journal_vacuum_unix.go | 289 ++++++++++++++++++ internal/systemd/journal_vacuum_unix_test.go | 235 ++++++++++++++ .../systemd/journal_vacuum_unsupported.go | 20 ++ interp/api.go | 1 + 8 files changed, 633 insertions(+), 21 deletions(-) create mode 100644 internal/systemd/journal_file_unix.go create mode 100644 internal/systemd/journal_vacuum_unix.go create mode 100644 internal/systemd/journal_vacuum_unix_test.go create mode 100644 internal/systemd/journal_vacuum_unsupported.go diff --git a/builtins/systemd.go b/builtins/systemd.go index c288171c..22c4344d 100644 --- a/builtins/systemd.go +++ b/builtins/systemd.go @@ -60,10 +60,32 @@ type JournalStorageReader interface { JournalDiskUsage(ctx context.Context) (JournalUsage, error) } +// JournalVacuumRequest contains only bounded cleanup predicates. Before is an +// absolute archive mtime cutoff; MaxBytes is an allocated archived-byte target. +type JournalVacuumRequest struct { + Now time.Time + Before time.Time + MaxBytes uint64 + DryRun bool +} + +// JournalVacuumResult reports the cleanup selected or completed without +// exposing host paths or journal filenames. +type JournalVacuumResult struct { + Files int + Bytes uint64 +} + +// JournalCleaner performs policy-bounded cleanup of archived journal files. +type JournalCleaner interface { + VacuumJournal(ctx context.Context, request JournalVacuumRequest) (JournalVacuumResult, error) +} + // SystemdServices contains the trusted backends available to systemd-aware // builtins. Additional manager and journal-maintenance interfaces can be added // here without exposing transports to command implementations. type SystemdServices struct { Journal JournalReader JournalStorage JournalStorageReader + JournalCleaner JournalCleaner } diff --git a/docs/RULES.md b/docs/RULES.md index 80ca90e0..b74df18a 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -79,21 +79,31 @@ read paths selected by `interp.WithSystemdTarget`; those paths intentionally bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding application and cannot be supplied by shell scripts. -The journal reader is limited to regular, non-symlink `.journal` files directly -under the configured machine-ID directories. It reads the configured machine ID, -adds it to every native journal query, verifies it again on every returned entry, -and applies fixed file, field-size, entry-count, and cancellation bounds. Builtins -receive selected fields only and never receive a raw journal handle, target path, -or arbitrary field-match capability. +Journal reads and storage metadata are limited to regular, non-symlink `.journal` +files directly under the configured machine-ID directories. The reader adds the +configured machine ID to every native query, verifies it again on every returned +entry, and applies fixed file, field-size, entry-count, and cancellation bounds. +Builtins receive selected fields only and never receive a raw journal handle, +target path, or arbitrary field-match capability. + +The only deletion exception is `JournalCleaner.VacuumJournal`. It is available +only through trusted systemd target configuration and a validated operator +vacuum policy. The backend pins each configured journal directory with `os.Root`, +checks directory identity across open, accepts only strict systemd archived-file +names, excludes symlinks and hardlinks, and revalidates file identity immediately +before rooted removal. Active, malformed, recently modified, and policy-retained +files must never be deleted. Every cleanup call has both file-count and byte +ceilings in addition to the shared `journal:storage/clean` authorization and +remediation-mode requirement. --- ## Implementation Rules ### File System Safety -- Commands MUST NOT write to any files on the system in any way +- Commands MUST NOT write to any files on the system except through an explicitly documented, structured remediation capability such as `TruncateToZeroIfAtLeast` or `JournalCleaner` - Commands MUST NOT execute any files or external binaries on the system in any way -- Commands MUST NOT create, modify, or delete files, directories, or symlinks +- Commands MUST NOT create, modify, or delete files, directories, or symlinks except as explicitly permitted by such a remediation capability - Commands MUST NOT follow symlinks during write operations (no writes = no risk, but verify) ### Memory Safety & Resource Limits diff --git a/internal/systemd/journal_file_unix.go b/internal/systemd/journal_file_unix.go new file mode 100644 index 00000000..baee3514 --- /dev/null +++ b/internal/systemd/journal_file_unix.go @@ -0,0 +1,43 @@ +//go:build linux || darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "math" + "os" + "syscall" +) + +const journalBlockSize = 512 + +type journalFileStat struct { + dev uint64 + ino uint64 + nlink uint64 + blocks uint64 + allocated uint64 +} + +func journalStat(info os.FileInfo) (journalFileStat, error) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Blocks < 0 { + return journalFileStat{}, fmt.Errorf("allocation metadata is unavailable") + } + blocks := uint64(stat.Blocks) + if blocks > math.MaxUint64/journalBlockSize { + return journalFileStat{}, fmt.Errorf("allocation metadata overflows") + } + return journalFileStat{ + dev: uint64(stat.Dev), + ino: uint64(stat.Ino), + nlink: uint64(stat.Nlink), + blocks: blocks, + allocated: blocks * journalBlockSize, + }, nil +} diff --git a/internal/systemd/journal_usage_unix.go b/internal/systemd/journal_usage_unix.go index bb4ffa33..085664e9 100644 --- a/internal/systemd/journal_usage_unix.go +++ b/internal/systemd/journal_usage_unix.go @@ -12,13 +12,10 @@ import ( "fmt" "math" "os" - "syscall" "github.com/DataDog/rshell/builtins" ) -const journalBlockSize = 512 - // JournalDiskUsage reports allocated blocks, matching journalctl's disk-usage // semantics more closely than summing logical file lengths. func (c *Client) JournalDiskUsage(ctx context.Context) (builtins.JournalUsage, error) { @@ -38,19 +35,14 @@ func (c *Client) JournalDiskUsage(ctx context.Context) (builtins.JournalUsage, e if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { return builtins.JournalUsage{}, fmt.Errorf("journal file %q changed during usage scan", path) } - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok || stat.Blocks < 0 { - return builtins.JournalUsage{}, fmt.Errorf("journal file %q has unavailable allocation metadata", path) - } - blocks := uint64(stat.Blocks) - if blocks > math.MaxUint64/journalBlockSize { - return builtins.JournalUsage{}, fmt.Errorf("journal allocation overflow for %q", path) + stat, err := journalStat(info) + if err != nil { + return builtins.JournalUsage{}, fmt.Errorf("journal file %q: %w", path, err) } - allocated := blocks * journalBlockSize - if usage.Bytes > math.MaxUint64-allocated { + if usage.Bytes > math.MaxUint64-stat.allocated { return builtins.JournalUsage{}, fmt.Errorf("journal allocation total overflow") } - usage.Bytes += allocated + usage.Bytes += stat.allocated } return usage, nil } diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go new file mode 100644 index 00000000..614769a6 --- /dev/null +++ b/internal/systemd/journal_vacuum_unix.go @@ -0,0 +1,289 @@ +//go:build linux || darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + "io" + "io/fs" + "math" + "os" + "sort" + "strings" + "time" + + "github.com/DataDog/rshell/builtins" +) + +type vacuumDirectory struct { + path string + root *os.Root +} + +type vacuumCandidate struct { + directory *vacuumDirectory + name string + modTime time.Time + size int64 + stat journalFileStat +} + +// VacuumJournal deletes only strictly recognized archived journals while +// enforcing the trusted policy on every invocation. +func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + if c.vacuumPolicy == nil { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum policy is not configured") + } + if err := c.vacuumPolicy.Validate(); err != nil { + return builtins.JournalVacuumResult{}, fmt.Errorf("invalid journal vacuum policy: %w", err) + } + if request.Now.IsZero() { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum reference time is required") + } + if request.MaxBytes == 0 && request.Before.IsZero() { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum requires a size or time limit") + } + if !request.Before.IsZero() && request.Before.After(request.Now) { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum time cutoff cannot be in the future") + } + + directories, err := c.openVacuumDirectories() + if err != nil { + return builtins.JournalVacuumResult{}, err + } + defer closeVacuumDirectories(directories) + + candidates, archivedBytes, err := collectVacuumCandidates(directories) + if err != nil { + return builtins.JournalVacuumResult{}, err + } + policyCutoff := request.Now.Add(-c.vacuumPolicy.MinRetentionAge) + remainingFiles := len(candidates) + remainingBytes := archivedBytes + result := builtins.JournalVacuumResult{} + + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return result, vacuumPartialError(result, err) + } + if candidate.modTime.After(policyCutoff) { + break + } + expired := !request.Before.IsZero() && !candidate.modTime.After(request.Before) + overSize := request.MaxBytes > 0 && remainingBytes > request.MaxBytes + if !expired && !overSize { + break + } + if remainingFiles-1 < c.vacuumPolicy.MinRetainedFiles || remainingBytes-candidate.stat.allocated < c.vacuumPolicy.MinRetainedBytes { + break + } + if result.Files >= c.vacuumPolicy.MaxDeletedFiles || candidate.stat.allocated > c.vacuumPolicy.MaxDeletedBytes-result.Bytes { + break + } + if err := revalidateVacuumCandidate(candidate); err != nil { + return result, vacuumPartialError(result, err) + } + if !request.DryRun { + if err := candidate.directory.root.Remove(candidate.name); err != nil { + return result, vacuumPartialError(result, fmt.Errorf("remove archived journal: %w", err)) + } + } + result.Files++ + result.Bytes += candidate.stat.allocated + remainingFiles-- + remainingBytes -= candidate.stat.allocated + } + return result, nil +} + +func (c *Client) openVacuumDirectories() ([]*vacuumDirectory, error) { + if c.target.MachineIDPath == "" { + return nil, fmt.Errorf("systemd target machine ID path is not configured") + } + if len(c.target.JournalDirs) == 0 { + return nil, fmt.Errorf("systemd target journal directories are not configured") + } + machineID, err := readMachineID(c.target.MachineIDPath) + if err != nil { + return nil, err + } + + directories := make([]*vacuumDirectory, 0, len(c.target.JournalDirs)) + for _, basePath := range c.target.JournalDirs { + baseRoot, err := os.OpenRoot(basePath) + if err != nil { + if os.IsNotExist(err) { + continue + } + closeVacuumDirectories(directories) + return nil, fmt.Errorf("open journal root %q: %w", basePath, err) + } + before, err := baseRoot.Lstat(machineID) + if err != nil { + baseRoot.Close() + if os.IsNotExist(err) { + continue + } + closeVacuumDirectories(directories) + return nil, fmt.Errorf("inspect journal machine directory: %w", err) + } + if !before.IsDir() || before.Mode()&fs.ModeSymlink != 0 { + baseRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("journal machine directory is not a real directory") + } + beforeStat, err := journalStat(before) + if err != nil { + baseRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("inspect journal machine directory identity: %w", err) + } + machineRoot, err := baseRoot.OpenRoot(machineID) + baseRoot.Close() + if err != nil { + closeVacuumDirectories(directories) + return nil, fmt.Errorf("open journal machine directory: %w", err) + } + after, err := machineRoot.Stat(".") + if err != nil { + machineRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("verify journal machine directory: %w", err) + } + afterStat, err := journalStat(after) + if err != nil || beforeStat.dev != afterStat.dev || beforeStat.ino != afterStat.ino { + machineRoot.Close() + closeVacuumDirectories(directories) + return nil, fmt.Errorf("journal machine directory changed while opening") + } + directories = append(directories, &vacuumDirectory{path: basePath, root: machineRoot}) + } + return directories, nil +} + +func collectVacuumCandidates(directories []*vacuumDirectory) ([]vacuumCandidate, uint64, error) { + candidates := make([]vacuumCandidate, 0) + var archivedBytes uint64 + for _, directory := range directories { + handle, err := directory.root.Open(".") + if err != nil { + return nil, 0, fmt.Errorf("open pinned journal directory: %w", err) + } + entries, readErr := handle.ReadDir(maxJournalFiles + 1) + closeErr := handle.Close() + if readErr != nil && readErr != io.EOF { + return nil, 0, fmt.Errorf("read pinned journal directory: %w", readErr) + } + if closeErr != nil { + return nil, 0, fmt.Errorf("close pinned journal directory: %w", closeErr) + } + if len(entries) > maxJournalFiles { + return nil, 0, fmt.Errorf("journal directory has too many entries (maximum %d)", maxJournalFiles) + } + for _, entry := range entries { + if !isArchivedJournalName(entry.Name()) || entry.Type()&fs.ModeSymlink != 0 { + continue + } + info, err := directory.root.Lstat(entry.Name()) + if err != nil { + return nil, 0, fmt.Errorf("inspect archived journal: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&fs.ModeSymlink != 0 { + continue + } + stat, err := journalStat(info) + if err != nil { + return nil, 0, fmt.Errorf("inspect archived journal allocation: %w", err) + } + if stat.nlink != 1 { + continue + } + if archivedBytes > math.MaxUint64-stat.allocated { + return nil, 0, fmt.Errorf("archived journal allocation total overflow") + } + archivedBytes += stat.allocated + candidates = append(candidates, vacuumCandidate{ + directory: directory, + name: entry.Name(), + modTime: info.ModTime(), + size: info.Size(), + stat: stat, + }) + if len(candidates) > maxJournalFiles { + return nil, 0, fmt.Errorf("systemd target has too many archived journal files (maximum %d)", maxJournalFiles) + } + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].modTime.Equal(candidates[j].modTime) { + if candidates[i].directory.path == candidates[j].directory.path { + return candidates[i].name < candidates[j].name + } + return candidates[i].directory.path < candidates[j].directory.path + } + return candidates[i].modTime.Before(candidates[j].modTime) + }) + return candidates, archivedBytes, nil +} + +func revalidateVacuumCandidate(candidate vacuumCandidate) error { + info, err := candidate.directory.root.Lstat(candidate.name) + if err != nil { + return fmt.Errorf("revalidate archived journal: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&fs.ModeSymlink != 0 || info.Size() != candidate.size || !info.ModTime().Equal(candidate.modTime) { + return fmt.Errorf("archived journal changed before deletion") + } + stat, err := journalStat(info) + if err != nil { + return fmt.Errorf("revalidate archived journal allocation: %w", err) + } + if stat.dev != candidate.stat.dev || stat.ino != candidate.stat.ino || stat.nlink != 1 || stat.blocks != candidate.stat.blocks { + return fmt.Errorf("archived journal identity changed before deletion") + } + return nil +} + +func isArchivedJournalName(name string) bool { + if strings.IndexByte(name, '/') >= 0 || strings.IndexByte(name, 0) >= 0 || !strings.HasSuffix(name, ".journal") { + return false + } + stem := strings.TrimSuffix(name, ".journal") + separator := strings.LastIndexByte(stem, '@') + if separator <= 0 || separator == len(stem)-1 { + return false + } + parts := strings.Split(stem[separator+1:], "-") + return len(parts) == 3 && validHexLength(parts[0], 32) && validHexLength(parts[1], 16) && validHexLength(parts[2], 16) +} + +func validHexLength(value string, length int) bool { + if len(value) != length { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} + +func closeVacuumDirectories(directories []*vacuumDirectory) { + for _, directory := range directories { + directory.root.Close() + } +} + +func vacuumPartialError(result builtins.JournalVacuumResult, err error) error { + if result.Files == 0 { + return err + } + return fmt.Errorf("journal vacuum stopped after deleting %d files (%d bytes): %w", result.Files, result.Bytes, err) +} diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go new file mode 100644 index 00000000..36ee84de --- /dev/null +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -0,0 +1,235 @@ +//go:build linux || darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +const testMachineID = "0123456789abcdef0123456789abcdef" + +func archivedJournalName(index int) string { + return fmt.Sprintf("system@abcdef0123456789abcdef0123456789-%016x-%016x.journal", index, index) +} + +func newVacuumTestClient(t *testing.T, policy *JournalVacuumPolicy) (*Client, string) { + t.Helper() + root := t.TempDir() + machineIDPath := filepath.Join(root, "machine-id") + journalRoot := filepath.Join(root, "journal") + machineDir := filepath.Join(journalRoot, testMachineID) + require.NoError(t, os.WriteFile(machineIDPath, []byte(testMachineID+"\n"), 0o600)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + return NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}, policy), machineDir +} + +func writeVacuumFile(t *testing.T, directory, name string, modTime time.Time) string { + t.Helper() + path := filepath.Join(directory, name) + require.NoError(t, os.WriteFile(path, make([]byte, 8192), 0o600)) + require.NoError(t, os.Chtimes(path, modTime, modTime)) + return path +} + +func testVacuumPolicy() *JournalVacuumPolicy { + return &JournalVacuumPolicy{ + MinRetentionAge: 24 * time.Hour, + MinRetainedFiles: 1, + MaxDeletedFiles: 10, + MaxDeletedBytes: 1 << 30, + } +} + +func TestVacuumJournalDeletesOldestArchivesWithinPolicy(t *testing.T) { + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) + client, directory := newVacuumTestClient(t, testVacuumPolicy()) + active := writeVacuumFile(t, directory, "system.journal", now.Add(-30*24*time.Hour)) + namespaceActive := writeVacuumFile(t, directory, "system@tenant.journal", now.Add(-30*24*time.Hour)) + malformed := writeVacuumFile(t, directory, "system@old.journal", now.Add(-30*24*time.Hour)) + oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + old := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-5*24*time.Hour)) + recent := writeVacuumFile(t, directory, archivedJournalName(3), now.Add(-time.Hour)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Equal(t, 2, result.Files) + assert.Greater(t, result.Bytes, uint64(0)) + assert.NoFileExists(t, oldest) + assert.NoFileExists(t, old) + assert.FileExists(t, recent) + assert.FileExists(t, active) + assert.FileExists(t, namespaceActive) + assert.FileExists(t, malformed) +} + +func TestVacuumJournalDryRunDoesNotDelete(t *testing.T) { + now := time.Now() + client, directory := newVacuumTestClient(t, testVacuumPolicy()) + archive := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-7*24*time.Hour)) + writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-6*24*time.Hour)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + DryRun: true, + }) + require.NoError(t, err) + assert.Equal(t, 1, result.Files) + assert.FileExists(t, archive) +} + +func TestVacuumJournalEnforcesPerCallFileCeiling(t *testing.T) { + now := time.Now() + policy := testVacuumPolicy() + policy.MinRetainedFiles = 0 + policy.MaxDeletedFiles = 1 + client, directory := newVacuumTestClient(t, policy) + oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-9*24*time.Hour)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Equal(t, 1, result.Files) + assert.NoFileExists(t, oldest) + assert.FileExists(t, second) +} + +func TestVacuumJournalDoesNotUnderflowByteCeiling(t *testing.T) { + now := time.Now() + policy := testVacuumPolicy() + policy.MinRetainedFiles = 0 + policy.MaxDeletedBytes = 1 + client, directory := newVacuumTestClient(t, policy) + archive := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Zero(t, result.Files) + assert.Zero(t, result.Bytes) + assert.FileExists(t, archive) +} + +func TestVacuumJournalHonorsAllocatedSizeTarget(t *testing.T) { + now := time.Now() + policy := testVacuumPolicy() + policy.MinRetainedFiles = 0 + client, directory := newVacuumTestClient(t, policy) + oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-9*24*time.Hour)) + info, err := os.Lstat(oldest) + require.NoError(t, err) + stat, err := journalStat(info) + require.NoError(t, err) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + MaxBytes: stat.allocated, + }) + require.NoError(t, err) + assert.Equal(t, 1, result.Files) + assert.NoFileExists(t, oldest) + assert.FileExists(t, second) +} + +func TestVacuumJournalSkipsHardlinksAndSymlinks(t *testing.T) { + now := time.Now() + policy := testVacuumPolicy() + policy.MinRetainedFiles = 0 + client, directory := newVacuumTestClient(t, policy) + first := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + second := filepath.Join(directory, archivedJournalName(2)) + require.NoError(t, os.Link(first, second)) + symlink := filepath.Join(directory, archivedJournalName(3)) + require.NoError(t, os.Symlink(first, symlink)) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Zero(t, result.Files) + assert.FileExists(t, first) + assert.FileExists(t, second) + _, err = os.Lstat(symlink) + require.NoError(t, err) +} + +func TestVacuumJournalRejectsSymlinkedMachineDirectory(t *testing.T) { + now := time.Now() + root := t.TempDir() + journalRoot := filepath.Join(root, "journal") + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(journalRoot, 0o700)) + require.NoError(t, os.MkdirAll(outside, 0o700)) + archive := writeVacuumFile(t, outside, archivedJournalName(1), now.Add(-10*24*time.Hour)) + require.NoError(t, os.Symlink(outside, filepath.Join(journalRoot, testMachineID))) + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(testMachineID+"\n"), 0o600)) + client := NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}, testVacuumPolicy()) + + _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a real directory") + assert.FileExists(t, archive) +} + +func TestVacuumJournalRequiresPolicyAndBounds(t *testing.T) { + now := time.Now() + client, _ := newVacuumTestClient(t, nil) + _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, Before: now.Add(-time.Hour)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "policy is not configured") + + client, _ = newVacuumTestClient(t, testVacuumPolicy()) + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now}) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a size or time limit") +} + +func TestIsArchivedJournalName(t *testing.T) { + for _, name := range []string{ + archivedJournalName(1), + "system@tenant@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal", + "user-1000@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal", + } { + assert.True(t, isArchivedJournalName(name), name) + } + for _, name := range []string{ + "system.journal", + "user-1000.journal", + "system@tenant.journal", + "system@old.journal", + "system@abcdef0123456789abcdef0123456789-1-1.journal", + "system@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal~", + "../" + archivedJournalName(1), + } { + assert.False(t, isArchivedJournalName(name), name) + } +} diff --git a/internal/systemd/journal_vacuum_unsupported.go b/internal/systemd/journal_vacuum_unsupported.go new file mode 100644 index 00000000..4faa5f77 --- /dev/null +++ b/internal/systemd/journal_vacuum_unsupported.go @@ -0,0 +1,20 @@ +//go:build !linux && !darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// VacuumJournal requires rooted Unix filesystem operations. +func (c *Client) VacuumJournal(context.Context, builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + return builtins.JournalVacuumResult{}, fmt.Errorf("%w: journal vacuum requires Linux or macOS", builtins.ErrSystemdUnsupported) +} diff --git a/interp/api.go b/interp/api.go index 08974836..5b1b85cc 100644 --- a/interp/api.go +++ b/interp/api.go @@ -337,6 +337,7 @@ func New(opts ...RunnerOption) (*Runner, error) { r.systemd = &builtins.SystemdServices{ Journal: systemdClient, JournalStorage: systemdClient, + JournalCleaner: systemdClient, } r.proc = builtins.NewProcProvider(r.procPath) return r, nil From 4ba0f0dfefa97fa341e23e35bcb312f68d90d4bb Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:24:49 -0400 Subject: [PATCH 10/53] feat(journalctl): add bounded journal vacuum --- analysis/symbols_builtins.go | 2 + builtins/journalctl/journalctl.go | 118 +++++++++++++++--- builtins/journalctl/journalctl_test.go | 113 +++++++++++++++++ interp/journal_vacuum_integration_test.go | 59 +++++++++ .../cmd/journalctl/errors/vacuum_denied.yaml | 12 ++ .../journalctl/errors/vacuum_read_only.yaml | 11 ++ 6 files changed, 298 insertions(+), 17 deletions(-) create mode 100644 interp/journal_vacuum_integration_test.go create mode 100644 tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml create mode 100644 tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml diff --git a/analysis/symbols_builtins.go b/analysis/symbols_builtins.go index 7935f952..6f11d710 100644 --- a/analysis/symbols_builtins.go +++ b/analysis/symbols_builtins.go @@ -209,6 +209,8 @@ var builtinPerCommandSymbols = map[string][]string{ "unicode.IsGraphic", // 🟢 identifies non-graphic runes that must be escaped at the output boundary; pure function, no I/O. "unicode/utf8.DecodeRuneInString", // 🟢 decodes output text so control and malformed bytes can be escaped; pure function, no I/O. "unicode/utf8.RuneError", // 🟢 replacement rune constant used to detect malformed UTF-8; no side effects. + // Note: builtins/internal/sizeparse symbols are exempt from this + // allowlist (internal packages are checked separately). }, "logrotate": { "context.Context", // 🟢 deadline/cancellation plumbing; pure interface, no side effects. diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index a0dedf1d..027a9e31 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -22,6 +22,9 @@ // YYYY-MM-DD HH:MM:SS timestamp, or lookback duration // -o, --output=FORMAT output format: short (default) or cat // --disk-usage show allocated journal storage and exit +// --vacuum-size=SIZE remove oldest archives toward allocated SIZE +// --vacuum-time=AGE remove archives older than Go duration AGE +// --dry-run report cleanup without deleting archives // -h, --help print usage and exit package journalctl @@ -36,6 +39,7 @@ import ( "unicode/utf8" "github.com/DataDog/rshell/builtins" + "github.com/DataDog/rshell/builtins/internal/sizeparse" ) const ( @@ -51,26 +55,32 @@ var Cmd = builtins.Command{ } type flags struct { - units *[]string - kernel *bool - boot *bool - lines *string - since *string - output *string - usage *bool - help *bool + units *[]string + kernel *bool + boot *bool + lines *string + since *string + output *string + usage *bool + vacuumSize *string + vacuumTime *string + dryRun *bool + help *bool } func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { options := flags{ - units: fs.StringArrayP("unit", "u", nil, "show logs for exact UNIT (repeatable)"), - kernel: fs.BoolP("dmesg", "k", false, "show current-boot kernel messages"), - boot: fs.BoolP("boot", "b", false, "show messages from the current boot"), - lines: fs.StringP("lines", "n", "100", "show at most COUNT entries (maximum 1000)"), - since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), - output: fs.StringP("output", "o", "short", "output format: short or cat"), - usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), - help: fs.BoolP("help", "h", false, "print usage and exit"), + units: fs.StringArrayP("unit", "u", nil, "show logs for exact UNIT (repeatable)"), + kernel: fs.BoolP("dmesg", "k", false, "show current-boot kernel messages"), + boot: fs.BoolP("boot", "b", false, "show messages from the current boot"), + lines: fs.StringP("lines", "n", "100", "show at most COUNT entries (maximum 1000)"), + since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), + output: fs.StringP("output", "o", "short", "output format: short or cat"), + usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), + vacuumSize: fs.String("vacuum-size", "", "remove oldest archives toward allocated SIZE"), + vacuumTime: fs.String("vacuum-time", "", "remove archives older than Go duration AGE"), + dryRun: fs.Bool("dry-run", false, "report cleanup without deleting archives"), + help: fs.BoolP("help", "h", false, "print usage and exit"), } return options.run(fs) } @@ -92,6 +102,9 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { if *options.usage { return runDiskUsage(ctx, callCtx, fs) } + if fs.Changed("vacuum-size") || fs.Changed("vacuum-time") || *options.dryRun { + return options.runVacuum(ctx, callCtx, fs) + } if len(*options.units) > builtins.MaxJournalQueryUnits { callCtx.Errf("journalctl: too many unit scopes (maximum %d)\n", builtins.MaxJournalQueryUnits) return builtins.Result{Code: 1} @@ -173,7 +186,7 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { } func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { - for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output"} { + for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "vacuum-size", "vacuum-time", "dry-run"} { if fs.Changed(flagName) { callCtx.Errf("journalctl: --disk-usage cannot be combined with --%s\n", flagName) return builtins.Result{Code: 1} @@ -206,6 +219,77 @@ func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builti return builtins.Result{} } +func (options flags) runVacuum(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { + for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "disk-usage"} { + if fs.Changed(flagName) { + callCtx.Errf("journalctl: journal vacuum cannot be combined with --%s\n", flagName) + return builtins.Result{Code: 1} + } + } + sizeSet := fs.Changed("vacuum-size") + timeSet := fs.Changed("vacuum-time") + if !sizeSet && !timeSet { + callCtx.Errf("journalctl: --dry-run requires --vacuum-size or --vacuum-time\n") + return builtins.Result{Code: 1} + } + if callCtx.Now.IsZero() { + callCtx.Errf("journalctl: journal vacuum requires a runner reference time\n") + return builtins.Result{Code: 1} + } + + request := builtins.JournalVacuumRequest{Now: callCtx.Now, DryRun: *options.dryRun} + if sizeSet { + size, err := sizeparse.Parse(*options.vacuumSize) + if err != nil || size <= 0 { + callCtx.Errf("journalctl: invalid --vacuum-size value %q (must be greater than zero)\n", *options.vacuumSize) + return builtins.Result{Code: 1} + } + request.MaxBytes = uint64(size) + } + if timeSet { + age, err := time.ParseDuration(*options.vacuumTime) + if err != nil || age <= 0 { + callCtx.Errf("journalctl: invalid --vacuum-time value %q (use a positive Go duration such as 168h)\n", *options.vacuumTime) + return builtins.Result{Code: 1} + } + request.Before = callCtx.Now.Add(-age) + } + + if callCtx.AuthorizeSystemd == nil { + callCtx.Errf("journalctl: systemd authorization capability is not available\n") + return builtins.Result{Code: 1} + } + err := callCtx.AuthorizeSystemd(builtins.SystemdOperation{ + Resource: builtins.SystemdResourceJournalStorage, + Action: builtins.SystemdActionClean, + }) + if err != nil { + callCtx.Errf("journalctl: %s\n", err) + return builtins.Result{Code: 1} + } + if callCtx.Systemd == nil || callCtx.Systemd.JournalCleaner == nil { + callCtx.Errf("journalctl: systemd journal cleaner capability is not available\n") + return builtins.Result{Code: 1} + } + result, err := callCtx.Systemd.JournalCleaner.VacuumJournal(ctx, request) + if err != nil { + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } + verb := "done, freed" + if request.DryRun { + verb = "would free" + } + noun := "files" + if result.Files == 1 { + noun = "file" + } + callCtx.Outf("Vacuuming %s %s from %d archived journal %s.\n", verb, formatUsage(result.Bytes), result.Files, noun) + return builtins.Result{} +} + func formatUsage(bytes uint64) string { const unit = 1024 if bytes < unit { diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index a2d9ef06..5d76aa10 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -34,6 +34,17 @@ type fakeJournalStorage struct { calls int } +type fakeJournalCleaner struct { + result builtins.JournalVacuumResult + err error + requests []builtins.JournalVacuumRequest +} + +func (c *fakeJournalCleaner) VacuumJournal(_ context.Context, request builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + c.requests = append(c.requests, request) + return c.result, c.err +} + func (s *fakeJournalStorage) JournalDiskUsage(context.Context) (builtins.JournalUsage, error) { s.calls++ return s.usage, s.err @@ -183,6 +194,108 @@ func TestJournalctlDiskUsageIsExclusive(t *testing.T) { } } +func TestJournalctlVacuumAuthorizesCleanAndBuildsRequest(t *testing.T) { + now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) + cleaner := &fakeJournalCleaner{result: builtins.JournalVacuumResult{Files: 2, Bytes: 3 * 1024 * 1024}} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"--vacuum-size", "64M", "--vacuum-time", "168h", "--dry-run"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: now, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Vacuuming would free 3.0M from 2 archived journal files.\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Resource: builtins.SystemdResourceJournalStorage, + Action: builtins.SystemdActionClean, + }}, authorized) + require.Len(t, cleaner.requests, 1) + assert.Equal(t, builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-168 * time.Hour), + MaxBytes: 64 * 1024 * 1024, + DryRun: true, + }, cleaner.requests[0]) +} + +func TestJournalctlVacuumFormatsSingularResult(t *testing.T) { + cleaner := &fakeJournalCleaner{result: builtins.JournalVacuumResult{Files: 1, Bytes: 1024}} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Vacuuming done, freed 1.0K from 1 archived journal file.\n", stdout.String()) +} + +func TestJournalctlVacuumRejectsInvalidOrMixedOptionsBeforeAuthorization(t *testing.T) { + tests := [][]string{ + {"--dry-run"}, + {"--vacuum-size", "0"}, + {"--vacuum-size", "-1"}, + {"--vacuum-time", "0s"}, + {"--vacuum-time", "-1h"}, + {"--vacuum-time", "7d"}, + {"--vacuum-size", "1M", "-u", "api.service"}, + {"--vacuum-time", "1h", "-k"}, + {"--vacuum-size", "1M", "-n", "10"}, + } + for _, args := range tests { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + authorized := 0 + result := runJournalctl(t, args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + authorized++ + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.NotEmpty(t, stderr.String()) + assert.Zero(t, authorized) + assert.Empty(t, cleaner.requests) + }) + } +} + +func TestJournalctlVacuumDenialPreventsCleanup(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return errors.New("clean denied") + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "clean denied") + assert.Empty(t, cleaner.requests) +} + func TestFormatUsage(t *testing.T) { assert.Equal(t, "0B", formatUsage(0)) assert.Equal(t, "1023B", formatUsage(1023)) diff --git a/interp/journal_vacuum_integration_test.go b/interp/journal_vacuum_integration_test.go new file mode 100644 index 00000000..aa1f4152 --- /dev/null +++ b/interp/journal_vacuum_integration_test.go @@ -0,0 +1,59 @@ +//go:build linux || darwin + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package interp + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalctlVacuumEndToEnd(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineDir := filepath.Join(root, "var", "log", "journal", machineID) + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte(machineID+"\n"), 0o600)) + archive := filepath.Join(machineDir, "system@abcdef0123456789abcdef0123456789-0000000000000001-0000000000000001.journal") + require.NoError(t, os.WriteFile(archive, make([]byte, 8192), 0o600)) + old := time.Now().Add(-7 * 24 * time.Hour) + require.NoError(t, os.Chtimes(archive, old, old)) + + var stdout, stderr bytes.Buffer + runner, err := New( + StdIO(nil, &stdout, &stderr), + AllowedCommands([]string{"rshell:journalctl"}), + AllowedSystemd([]SystemdControlGrant{{ + Resource: SystemdResourceJournalStorage, + Actions: []SystemdAction{SystemdActionClean}, + }}), + WithMode(ModeRemediation), + WithSystemdTarget(SystemdTargetConfig{Root: root}), + WithJournalVacuumPolicy(JournalVacuumPolicy{ + MinRetentionAge: 24 * time.Hour, + MaxDeletedFiles: 1, + MaxDeletedBytes: 1 << 20, + }), + ) + require.NoError(t, err) + defer runner.Close() + + program, err := ParseScript("journalctl --vacuum-time=48h", "") + require.NoError(t, err) + require.NoError(t, runner.Run(context.Background(), program)) + assert.Empty(t, stderr.String()) + assert.Contains(t, stdout.String(), "Vacuuming done, freed") + assert.NoFileExists(t, archive) +} diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml new file mode 100644 index 00000000..9f5d2485 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml @@ -0,0 +1,12 @@ +# Remediation mode alone does not bypass the shared systemd allowlist. +skip_assert_against_bash: true +description: journalctl vacuum requires a storage clean grant +input: + mode: remediation + script: |+ + journalctl --vacuum-size=64M +expect: + stdout: |+ + stderr: |+ + journalctl: systemd resource "journal:storage" is not allowed for action "clean" + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml new file mode 100644 index 00000000..480420ba --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_read_only.yaml @@ -0,0 +1,11 @@ +# Cleanup is intentionally unavailable outside remediation mode, even in dry-run mode. +skip_assert_against_bash: true +description: journalctl vacuum requires remediation mode +input: + script: |+ + journalctl --vacuum-time=168h --dry-run +expect: + stdout: |+ + stderr: |+ + journalctl: systemd action "clean" requires remediation mode + exit_code: 1 From d83c65f009369cc297b0c730b2ad7b7dc51b0385 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:35:00 -0400 Subject: [PATCH 11/53] feat(systemd): add synchronous journal rotation --- builtins/systemd.go | 7 + docs/RULES.md | 8 + internal/systemd/journal_rotate_linux.go | 41 ++++ .../systemd/journal_rotate_unsupported.go | 20 ++ internal/systemd/journal_varlink.go | 187 ++++++++++++++++++ internal/systemd/journal_varlink_test.go | 138 +++++++++++++ interp/api.go | 1 + 7 files changed, 402 insertions(+) create mode 100644 internal/systemd/journal_rotate_linux.go create mode 100644 internal/systemd/journal_rotate_unsupported.go create mode 100644 internal/systemd/journal_varlink.go create mode 100644 internal/systemd/journal_varlink_test.go diff --git a/builtins/systemd.go b/builtins/systemd.go index 22c4344d..189063a0 100644 --- a/builtins/systemd.go +++ b/builtins/systemd.go @@ -81,6 +81,12 @@ type JournalCleaner interface { VacuumJournal(ctx context.Context, request JournalVacuumRequest) (JournalVacuumResult, error) } +// JournalRotator synchronously archives the active journals for the selected +// target. Implementations return only after journald reports completion. +type JournalRotator interface { + RotateJournal(ctx context.Context) error +} + // SystemdServices contains the trusted backends available to systemd-aware // builtins. Additional manager and journal-maintenance interfaces can be added // here without exposing transports to command implementations. @@ -88,4 +94,5 @@ type SystemdServices struct { Journal JournalReader JournalStorage JournalStorageReader JournalCleaner JournalCleaner + JournalRotator JournalRotator } diff --git a/docs/RULES.md b/docs/RULES.md index b74df18a..0f007503 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -96,6 +96,14 @@ files must never be deleted. Every cleanup call has both file-count and byte ceilings in addition to the shared `journal:storage/clean` authorization and remediation-mode requirement. +`JournalRotator.RotateJournal` is the only journal-daemon mutation exception. +It may call only the fixed `io.systemd.Journal.Rotate` Varlink method through +the configured journal control socket. The backend validates the target machine +ID before connecting, rejects symlink and non-socket endpoints, verifies socket +identity across the connection, and applies fixed response-size and execution +time bounds. A generic Varlink method or parameter interface must not be exposed +to builtins. + --- ## Implementation Rules diff --git a/internal/systemd/journal_rotate_linux.go b/internal/systemd/journal_rotate_linux.go new file mode 100644 index 00000000..78c891a0 --- /dev/null +++ b/internal/systemd/journal_rotate_linux.go @@ -0,0 +1,41 @@ +//go:build linux + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "errors" + "fmt" + "time" +) + +const journalRotationTimeout = 30 * time.Second + +// RotateJournal asks journald to rotate active journals and waits for the +// daemon to report completion. +func (c *Client) RotateJournal(ctx context.Context) error { + if c.target.MachineIDPath == "" { + return fmt.Errorf("systemd target machine ID path is unavailable") + } + if _, err := readMachineID(c.target.MachineIDPath); err != nil { + return fmt.Errorf("validate systemd target machine ID: %w", err) + } + if c.target.JournalControlSocket == "" { + return fmt.Errorf("systemd target journal control socket is unavailable") + } + + rotationCtx, cancel := context.WithTimeout(ctx, journalRotationTimeout) + defer cancel() + if err := rotateJournalControl(rotationCtx, c.target.JournalControlSocket); err != nil { + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + return fmt.Errorf("journal rotation timed out after %s", journalRotationTimeout) + } + return fmt.Errorf("rotate journal: %w", err) + } + return nil +} diff --git a/internal/systemd/journal_rotate_unsupported.go b/internal/systemd/journal_rotate_unsupported.go new file mode 100644 index 00000000..b59a9498 --- /dev/null +++ b/internal/systemd/journal_rotate_unsupported.go @@ -0,0 +1,20 @@ +//go:build !linux + +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// RotateJournal requires a Linux systemd-journald control endpoint. +func (c *Client) RotateJournal(context.Context) error { + return fmt.Errorf("%w: journal rotation requires Linux", builtins.ErrSystemdUnsupported) +} diff --git a/internal/systemd/journal_varlink.go b/internal/systemd/journal_varlink.go new file mode 100644 index 00000000..c622da4f --- /dev/null +++ b/internal/systemd/journal_varlink.go @@ -0,0 +1,187 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "time" +) + +const ( + journalRotateMethod = "io.systemd.Journal.Rotate" + maxVarlinkMessageSize = 64 * 1024 +) + +type varlinkRequest struct { + Method string `json:"method"` +} + +type varlinkReply struct { + Parameters json.RawMessage `json:"parameters"` + Error string `json:"error"` + Continues bool `json:"continues"` +} + +func rotateJournalControl(ctx context.Context, path string) error { + before, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect journal control socket: %w", err) + } + if before.Mode()&os.ModeSymlink != 0 || before.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("journal control endpoint is not a Unix socket") + } + + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "unix", path) + if err != nil { + return contextIOError(ctx, "connect to journal control socket", err) + } + defer conn.Close() + + after, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("reinspect journal control socket: %w", err) + } + if !os.SameFile(before, after) { + return fmt.Errorf("journal control socket changed while connecting") + } + + return callJournalRotateVarlink(ctx, conn) +} + +func callJournalRotateVarlink(ctx context.Context, conn net.Conn) error { + stopCancel := context.AfterFunc(ctx, func() { + _ = conn.SetDeadline(time.Now()) + }) + defer stopCancel() + if deadline, ok := ctx.Deadline(); ok { + if err := conn.SetDeadline(deadline); err != nil { + return fmt.Errorf("set journal control deadline: %w", err) + } + } + + request, err := json.Marshal(varlinkRequest{Method: journalRotateMethod}) + if err != nil { + return fmt.Errorf("encode journal rotation request: %w", err) + } + request = append(request, 0) + if err := writeAll(conn, request); err != nil { + return contextIOError(ctx, "write journal rotation request", err) + } + + message, err := readVarlinkMessage(conn) + if err != nil { + return contextIOError(ctx, "read journal rotation response", err) + } + trimmed := bytes.TrimSpace(message) + if len(trimmed) < 2 || trimmed[0] != '{' || trimmed[len(trimmed)-1] != '}' { + return fmt.Errorf("journal control returned a malformed response") + } + + var reply varlinkReply + if err := json.Unmarshal(trimmed, &reply); err != nil { + return fmt.Errorf("decode journal rotation response: %w", err) + } + if reply.Continues { + return fmt.Errorf("journal control returned an unexpected streaming response") + } + if reply.Error != "" { + if !validVarlinkIdentifier(reply.Error) { + return fmt.Errorf("journald rejected journal rotation with an invalid protocol error") + } + return fmt.Errorf("journald rejected journal rotation: %s", reply.Error) + } + if len(reply.Parameters) > 0 { + var parameters map[string]json.RawMessage + if err := json.Unmarshal(reply.Parameters, ¶meters); err != nil || parameters == nil { + return fmt.Errorf("journal control returned malformed response parameters") + } + if len(parameters) != 0 { + return fmt.Errorf("journal control returned unexpected response parameters") + } + } + return nil +} + +func writeAll(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if err != nil { + return err + } + if n <= 0 || n > len(data) { + return io.ErrShortWrite + } + data = data[n:] + } + return nil +} + +func readVarlinkMessage(reader io.Reader) ([]byte, error) { + message := make([]byte, 0, 512) + buffer := make([]byte, 4096) + for { + n, err := reader.Read(buffer) + if n > 0 { + chunk := buffer[:n] + if terminator := bytes.IndexByte(chunk, 0); terminator >= 0 { + if len(message)+terminator > maxVarlinkMessageSize { + return nil, fmt.Errorf("Varlink response exceeds %d bytes", maxVarlinkMessageSize) + } + message = append(message, chunk[:terminator]...) + if len(chunk[terminator+1:]) > 0 { + return nil, fmt.Errorf("Varlink response contains trailing data") + } + if len(message) == 0 { + return nil, fmt.Errorf("Varlink response is empty") + } + return message, nil + } + if len(message)+len(chunk) > maxVarlinkMessageSize { + return nil, fmt.Errorf("Varlink response exceeds %d bytes", maxVarlinkMessageSize) + } + message = append(message, chunk...) + } + if err != nil { + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("Varlink response ended without a terminator") + } + return nil, err + } + } +} + +func validVarlinkIdentifier(value string) bool { + if value == "" || len(value) > 256 { + return false + } + for _, char := range []byte(value) { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '.' || char == '_' { + continue + } + return false + } + return true +} + +func contextIOError(ctx context.Context, operation string, err error) error { + if contextErr := ctx.Err(); contextErr != nil { + return contextErr + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if deadline, hasDeadline := ctx.Deadline(); hasDeadline && !time.Now().Before(deadline) { + return context.DeadlineExceeded + } + } + return fmt.Errorf("%s: %w", operation, err) +} diff --git a/internal/systemd/journal_varlink_test.go b/internal/systemd/journal_varlink_test.go new file mode 100644 index 00000000..86bec11d --- /dev/null +++ b/internal/systemd/journal_varlink_test.go @@ -0,0 +1,138 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func serveVarlinkResponse(conn net.Conn, response []byte, gate <-chan struct{}) (<-chan []byte, <-chan error) { + request := make(chan []byte, 1) + finished := make(chan error, 1) + go func() { + defer conn.Close() + message, err := readVarlinkMessage(conn) + if err != nil { + finished <- err + return + } + request <- message + if gate != nil { + <-gate + } + response = append(append([]byte(nil), response...), 0) + finished <- writeAll(conn, response) + }() + return request, finished +} + +func TestRotateJournalControlUsesFixedSynchronousMethod(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + gate := make(chan struct{}) + request, finished := serveVarlinkResponse(server, []byte(`{"parameters":{}}`), gate) + + rotationDone := make(chan error, 1) + go func() { + rotationDone <- callJournalRotateVarlink(context.Background(), client) + }() + + var decoded varlinkRequest + require.NoError(t, json.Unmarshal(<-request, &decoded)) + assert.Equal(t, journalRotateMethod, decoded.Method) + select { + case err := <-rotationDone: + t.Fatalf("rotation returned before journald replied: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(gate) + require.NoError(t, <-rotationDone) + require.NoError(t, <-finished) +} + +func TestRotateJournalControlReportsSafeProtocolErrors(t *testing.T) { + for _, test := range []struct { + name string + response string + needle string + }{ + {name: "daemon error", response: `{"error":"io.systemd.Journal.NotSupported","parameters":{}}`, needle: "io.systemd.Journal.NotSupported"}, + {name: "unsafe daemon error", response: "{\"error\":\"bad\\nterminal\"}", needle: "invalid protocol error"}, + {name: "streaming", response: `{"parameters":{},"continues":true}`, needle: "streaming response"}, + {name: "unexpected parameters", response: `{"parameters":{"path":"/host"}}`, needle: "unexpected response parameters"}, + {name: "malformed", response: `[]`, needle: "malformed response"}, + } { + t.Run(test.name, func(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + _, finished := serveVarlinkResponse(server, []byte(test.response), nil) + + err := callJournalRotateVarlink(context.Background(), client) + require.Error(t, err) + assert.Contains(t, err.Error(), test.needle) + if test.name == "unsafe daemon error" { + assert.NotContains(t, err.Error(), "terminal") + } + require.NoError(t, <-finished) + }) + } +} + +func TestRotateJournalControlHonorsCancellation(t *testing.T) { + client, server := net.Pipe() + gate := make(chan struct{}) + _, finished := serveVarlinkResponse(server, []byte(`{"parameters":{}}`), gate) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + err := callJournalRotateVarlink(ctx, client) + assert.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, client.Close()) + close(gate) + serverErr := <-finished + if serverErr != nil { + assert.True(t, errors.Is(serverErr, net.ErrClosed) || errors.Is(serverErr, io.ErrClosedPipe) || strings.Contains(serverErr.Error(), "broken pipe"), serverErr.Error()) + } +} + +func TestRotateJournalControlRejectsNonSocketEndpoints(t *testing.T) { + regular := filepath.Join(t.TempDir(), "journal.sock") + require.NoError(t, os.WriteFile(regular, []byte("not a socket"), 0o600)) + err := rotateJournalControl(context.Background(), regular) + assert.ErrorContains(t, err, "not a Unix socket") + + if runtime.GOOS == "windows" { + return + } + symlink := filepath.Join(t.TempDir(), "journal-link.sock") + require.NoError(t, os.Symlink(regular, symlink)) + err = rotateJournalControl(context.Background(), symlink) + assert.ErrorContains(t, err, "not a Unix socket") +} + +func TestReadVarlinkMessageBoundsResponses(t *testing.T) { + _, err := readVarlinkMessage(strings.NewReader(strings.Repeat("x", maxVarlinkMessageSize+1) + "\x00")) + assert.ErrorContains(t, err, "exceeds") + + _, err = readVarlinkMessage(strings.NewReader("{}")) + assert.ErrorContains(t, err, "without a terminator") + + _, err = readVarlinkMessage(strings.NewReader("{}\x00trailing")) + assert.ErrorContains(t, err, "trailing data") +} diff --git a/interp/api.go b/interp/api.go index 5b1b85cc..f2627344 100644 --- a/interp/api.go +++ b/interp/api.go @@ -338,6 +338,7 @@ func New(opts ...RunnerOption) (*Runner, error) { Journal: systemdClient, JournalStorage: systemdClient, JournalCleaner: systemdClient, + JournalRotator: systemdClient, } r.proc = builtins.NewProcProvider(r.procPath) return r, nil From 4755350785f1270005e0f8e401ed1641926f1620 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:38:44 -0400 Subject: [PATCH 12/53] feat(journalctl): add bounded journal rotation --- builtins/journalctl/journalctl.go | 48 ++++-- builtins/journalctl/journalctl_test.go | 142 +++++++++++++++++- .../journalctl/errors/rotate_read_only.yaml | 11 ++ tests/scenarios/cmd/journalctl/help/help.yaml | 1 + 4 files changed, 191 insertions(+), 11 deletions(-) create mode 100644 tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index 027a9e31..eb916e39 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -22,6 +22,7 @@ // YYYY-MM-DD HH:MM:SS timestamp, or lookback duration // -o, --output=FORMAT output format: short (default) or cat // --disk-usage show allocated journal storage and exit +// --rotate archive active journal files before returning // --vacuum-size=SIZE remove oldest archives toward allocated SIZE // --vacuum-time=AGE remove archives older than Go duration AGE // --dry-run report cleanup without deleting archives @@ -50,7 +51,7 @@ const ( // Cmd is the journalctl builtin command descriptor. var Cmd = builtins.Command{ Name: "journalctl", - Description: "query bounded systemd journal logs", + Description: "query and maintain bounded systemd journals", MakeFlags: makeFlags, } @@ -62,6 +63,7 @@ type flags struct { since *string output *string usage *bool + rotate *bool vacuumSize *string vacuumTime *string dryRun *bool @@ -77,6 +79,7 @@ func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), output: fs.StringP("output", "o", "short", "output format: short or cat"), usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), + rotate: fs.Bool("rotate", false, "archive active journal files and wait for completion"), vacuumSize: fs.String("vacuum-size", "", "remove oldest archives toward allocated SIZE"), vacuumTime: fs.String("vacuum-time", "", "remove archives older than Go duration AGE"), dryRun: fs.Bool("dry-run", false, "report cleanup without deleting archives"), @@ -102,8 +105,8 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { if *options.usage { return runDiskUsage(ctx, callCtx, fs) } - if fs.Changed("vacuum-size") || fs.Changed("vacuum-time") || *options.dryRun { - return options.runVacuum(ctx, callCtx, fs) + if *options.rotate || fs.Changed("vacuum-size") || fs.Changed("vacuum-time") || *options.dryRun { + return options.runMaintenance(ctx, callCtx, fs) } if len(*options.units) > builtins.MaxJournalQueryUnits { callCtx.Errf("journalctl: too many unit scopes (maximum %d)\n", builtins.MaxJournalQueryUnits) @@ -186,7 +189,7 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { } func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { - for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "vacuum-size", "vacuum-time", "dry-run"} { + for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "rotate", "vacuum-size", "vacuum-time", "dry-run"} { if fs.Changed(flagName) { callCtx.Errf("journalctl: --disk-usage cannot be combined with --%s\n", flagName) return builtins.Result{Code: 1} @@ -219,25 +222,34 @@ func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builti return builtins.Result{} } -func (options flags) runVacuum(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { +func (options flags) runMaintenance(ctx context.Context, callCtx *builtins.CallContext, fs *builtins.FlagSet) builtins.Result { for _, flagName := range []string{"unit", "dmesg", "boot", "lines", "since", "output", "disk-usage"} { if fs.Changed(flagName) { - callCtx.Errf("journalctl: journal vacuum cannot be combined with --%s\n", flagName) + callCtx.Errf("journalctl: journal maintenance cannot be combined with --%s\n", flagName) return builtins.Result{Code: 1} } } sizeSet := fs.Changed("vacuum-size") timeSet := fs.Changed("vacuum-time") - if !sizeSet && !timeSet { + vacuumSet := sizeSet || timeSet + if *options.dryRun && *options.rotate { + callCtx.Errf("journalctl: --dry-run cannot be combined with --rotate\n") + return builtins.Result{Code: 1} + } + if *options.dryRun && !vacuumSet { callCtx.Errf("journalctl: --dry-run requires --vacuum-size or --vacuum-time\n") return builtins.Result{Code: 1} } - if callCtx.Now.IsZero() { + if vacuumSet && callCtx.Now.IsZero() { callCtx.Errf("journalctl: journal vacuum requires a runner reference time\n") return builtins.Result{Code: 1} } - request := builtins.JournalVacuumRequest{Now: callCtx.Now, DryRun: *options.dryRun} + request := builtins.JournalVacuumRequest{} + if vacuumSet { + request.Now = callCtx.Now + request.DryRun = *options.dryRun + } if sizeSet { size, err := sizeparse.Parse(*options.vacuumSize) if err != nil || size <= 0 { @@ -267,10 +279,26 @@ func (options flags) runVacuum(ctx context.Context, callCtx *builtins.CallContex callCtx.Errf("journalctl: %s\n", err) return builtins.Result{Code: 1} } - if callCtx.Systemd == nil || callCtx.Systemd.JournalCleaner == nil { + if *options.rotate && (callCtx.Systemd == nil || callCtx.Systemd.JournalRotator == nil) { + callCtx.Errf("journalctl: systemd journal rotation capability is not available\n") + return builtins.Result{Code: 1} + } + if vacuumSet && (callCtx.Systemd == nil || callCtx.Systemd.JournalCleaner == nil) { callCtx.Errf("journalctl: systemd journal cleaner capability is not available\n") return builtins.Result{Code: 1} } + if *options.rotate { + if err := callCtx.Systemd.JournalRotator.RotateJournal(ctx); err != nil { + if ctx.Err() == nil { + callCtx.Errf("journalctl: %s\n", err) + } + return builtins.Result{Code: 1} + } + callCtx.Out("Journal rotation completed.\n") + } + if !vacuumSet { + return builtins.Result{} + } result, err := callCtx.Systemd.JournalCleaner.VacuumJournal(ctx, request) if err != nil { if ctx.Err() == nil { diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index 5d76aa10..adc69e9a 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -38,13 +38,31 @@ type fakeJournalCleaner struct { result builtins.JournalVacuumResult err error requests []builtins.JournalVacuumRequest + order *[]string +} + +type fakeJournalRotator struct { + err error + calls int + order *[]string } func (c *fakeJournalCleaner) VacuumJournal(_ context.Context, request builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { + if c.order != nil { + *c.order = append(*c.order, "vacuum") + } c.requests = append(c.requests, request) return c.result, c.err } +func (r *fakeJournalRotator) RotateJournal(context.Context) error { + if r.order != nil { + *r.order = append(*r.order, "rotate") + } + r.calls++ + return r.err +} + func (s *fakeJournalStorage) JournalDiskUsage(context.Context) (builtins.JournalUsage, error) { s.calls++ return s.usage, s.err @@ -177,6 +195,7 @@ func TestJournalctlDiskUsageIsExclusive(t *testing.T) { {"--disk-usage", "-n", "10"}, {"--disk-usage", "--since", "1h"}, {"--disk-usage", "-o", "cat"}, + {"--disk-usage", "--rotate"}, } { t.Run(strings.Join(args, "_"), func(t *testing.T) { storage := &fakeJournalStorage{} @@ -194,6 +213,108 @@ func TestJournalctlDiskUsageIsExclusive(t *testing.T) { } } +func TestJournalctlRotateUsesStorageCleanCapability(t *testing.T) { + rotator := &fakeJournalRotator{} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, []string{"--rotate"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Journal rotation completed.\n", stdout.String()) + assert.Equal(t, []builtins.SystemdOperation{{ + Resource: builtins.SystemdResourceJournalStorage, + Action: builtins.SystemdActionClean, + }}, authorized) + assert.Equal(t, 1, rotator.calls) +} + +func TestJournalctlRotateRunsBeforeVacuum(t *testing.T) { + order := []string{} + rotator := &fakeJournalRotator{order: &order} + cleaner := &fakeJournalCleaner{order: &order, result: builtins.JournalVacuumResult{Files: 1, Bytes: 1024}} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator, JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, []string{"rotate", "vacuum"}, order) + assert.Equal(t, "Journal rotation completed.\nVacuuming done, freed 1.0K from 1 archived journal file.\n", stdout.String()) +} + +func TestJournalctlRotationFailurePreventsVacuum(t *testing.T) { + rotator := &fakeJournalRotator{err: errors.New("rotation failed")} + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator, JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Contains(t, stderr.String(), "rotation failed") + assert.Equal(t, 1, rotator.calls) + assert.Empty(t, cleaner.requests) +} + +func TestJournalctlCombinedMaintenanceChecksCapabilitiesBeforeMutation(t *testing.T) { + t.Run("missing cleaner", func(t *testing.T) { + rotator := &fakeJournalRotator{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "cleaner capability is not available") + assert.Zero(t, rotator.calls) + }) + + t.Run("missing rotator", func(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate", "--vacuum-time", "1h"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "rotation capability is not available") + assert.Empty(t, cleaner.requests) + }) +} + func TestJournalctlVacuumAuthorizesCleanAndBuildsRequest(t *testing.T) { now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) cleaner := &fakeJournalCleaner{result: builtins.JournalVacuumResult{Files: 2, Bytes: 3 * 1024 * 1024}} @@ -243,7 +364,7 @@ func TestJournalctlVacuumFormatsSingularResult(t *testing.T) { assert.Equal(t, "Vacuuming done, freed 1.0K from 1 archived journal file.\n", stdout.String()) } -func TestJournalctlVacuumRejectsInvalidOrMixedOptionsBeforeAuthorization(t *testing.T) { +func TestJournalctlMaintenanceRejectsInvalidOrMixedOptionsBeforeAuthorization(t *testing.T) { tests := [][]string{ {"--dry-run"}, {"--vacuum-size", "0"}, @@ -251,6 +372,8 @@ func TestJournalctlVacuumRejectsInvalidOrMixedOptionsBeforeAuthorization(t *test {"--vacuum-time", "0s"}, {"--vacuum-time", "-1h"}, {"--vacuum-time", "7d"}, + {"--rotate", "--dry-run", "--vacuum-time", "1h"}, + {"--rotate", "-u", "api.service"}, {"--vacuum-size", "1M", "-u", "api.service"}, {"--vacuum-time", "1h", "-k"}, {"--vacuum-size", "1M", "-n", "10"}, @@ -279,6 +402,22 @@ func TestJournalctlVacuumRejectsInvalidOrMixedOptionsBeforeAuthorization(t *test } } +func TestJournalctlRotateDenialPreventsMutation(t *testing.T) { + rotator := &fakeJournalRotator{} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"--rotate"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return errors.New("clean denied") + }, + Systemd: &builtins.SystemdServices{JournalRotator: rotator}, + }) + assert.Equal(t, uint8(1), result.Code) + assert.Contains(t, stderr.String(), "clean denied") + assert.Zero(t, rotator.calls) +} + func TestJournalctlVacuumDenialPreventsCleanup(t *testing.T) { cleaner := &fakeJournalCleaner{} var stdout, stderr bytes.Buffer @@ -410,6 +549,7 @@ func TestJournalctlHelpDoesNotRequireSystemdCapability(t *testing.T) { assert.Equal(t, uint8(0), result.Code) assert.Contains(t, stdout.String(), "Usage: journalctl") assert.Contains(t, stdout.String(), "--unit") + assert.Contains(t, stdout.String(), "--rotate") assert.Empty(t, stderr.String()) } diff --git a/tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml b/tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml new file mode 100644 index 00000000..a11d7083 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/rotate_read_only.yaml @@ -0,0 +1,11 @@ +# Rotation mutates active journals and is unavailable outside remediation mode. +skip_assert_against_bash: true +description: journalctl rotation requires remediation mode +input: + script: |+ + journalctl --rotate +expect: + stdout: |+ + stderr: |+ + journalctl: systemd action "clean" requires remediation mode + exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/help/help.yaml b/tests/scenarios/cmd/journalctl/help/help.yaml index f9020b5b..7c74a3c6 100644 --- a/tests/scenarios/cmd/journalctl/help/help.yaml +++ b/tests/scenarios/cmd/journalctl/help/help.yaml @@ -10,5 +10,6 @@ expect: - "--unit" - "--dmesg" - "--lines" + - "--rotate" stderr: |+ exit_code: 0 From d1c231abbb47f1dd44b33d99140e534466ff16b5 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:40:30 -0400 Subject: [PATCH 13/53] refactor(systemd): remove unused runtime target path --- cmd/rshell/main.go | 5 +---- cmd/rshell/main_test.go | 1 - internal/systemd/target.go | 9 ++------- internal/systemd/target_test.go | 3 --- interp/systemd_target.go | 2 -- 5 files changed, 3 insertions(+), 17 deletions(-) diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index df6f24c1..ede636b5 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -49,7 +49,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. machineIDPath string journalSocket string systemBusSocket string - journalRuntime string vacuumMinAge time.Duration vacuumMinFiles int vacuumMinBytes uint64 @@ -115,7 +114,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. if journalDirs != "" { configuredJournalDirs = strings.Split(journalDirs, ",") } - systemdTargetSet := systemdRoot != "" || journalDirs != "" || machineIDPath != "" || journalSocket != "" || systemBusSocket != "" || journalRuntime != "" + systemdTargetSet := systemdRoot != "" || journalDirs != "" || machineIDPath != "" || journalSocket != "" || systemBusSocket != "" vacuumPolicySet := cmd.Flags().Changed("journal-vacuum-min-age") || cmd.Flags().Changed("journal-vacuum-min-files") || cmd.Flags().Changed("journal-vacuum-min-bytes") || @@ -145,7 +144,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. MachineIDPath: machineIDPath, JournalControlSocket: journalSocket, SystemBusSocket: systemBusSocket, - JournalRuntimeDir: journalRuntime, }, systemdTargetSet: systemdTargetSet, vacuumPolicy: vacuumPolicy, @@ -209,7 +207,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().StringVar(&machineIDPath, "systemd-machine-id-path", "", "machine-id file for an explicit systemd target") cmd.Flags().StringVar(&journalSocket, "systemd-journal-socket", "", "journald Varlink socket for an explicit systemd target") cmd.Flags().StringVar(&systemBusSocket, "systemd-bus-socket", "", "system D-Bus socket for an explicit systemd target") - cmd.Flags().StringVar(&journalRuntime, "systemd-runtime-dir", "", "journald runtime directory for legacy control acknowledgements") cmd.Flags().DurationVar(&vacuumMinAge, "journal-vacuum-min-age", 0, "minimum age of archived journals eligible for cleanup (required to enable cleanup)") cmd.Flags().IntVar(&vacuumMinFiles, "journal-vacuum-min-files", 0, "minimum archived journal files retained across cleanup") cmd.Flags().Uint64Var(&vacuumMinBytes, "journal-vacuum-min-bytes", 0, "minimum allocated archived journal bytes retained across cleanup") diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index 6434e647..8a6a8c8f 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -191,7 +191,6 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "--systemd-machine-id-path") assert.Contains(t, stdout, "--systemd-journal-socket") assert.Contains(t, stdout, "--systemd-bus-socket") - assert.Contains(t, stdout, "--systemd-runtime-dir") assert.Contains(t, stdout, "--journal-vacuum-min-age") assert.Contains(t, stdout, "--journal-vacuum-max-delete-files") assert.Contains(t, stdout, "--journal-vacuum-max-delete-bytes") diff --git a/internal/systemd/target.go b/internal/systemd/target.go index b39f2b29..2d1b3355 100644 --- a/internal/systemd/target.go +++ b/internal/systemd/target.go @@ -24,7 +24,6 @@ type Target struct { MachineIDPath string JournalControlSocket string SystemBusSocket string - JournalRuntimeDir string } // LocalTarget returns the standard paths for the local systemd host. @@ -38,7 +37,7 @@ func LocalTarget() Target { // fields remain empty and never fall back to local paths. func ResolveTarget(target Target) (Target, error) { if target.Root != "" { - if len(target.JournalDirs) > 0 || target.MachineIDPath != "" || target.JournalControlSocket != "" || target.SystemBusSocket != "" || target.JournalRuntimeDir != "" { + if len(target.JournalDirs) > 0 || target.MachineIDPath != "" || target.JournalControlSocket != "" || target.SystemBusSocket != "" { return Target{}, fmt.Errorf("systemd target root cannot be combined with explicit paths") } root, err := validateAbsolutePath("root", target.Root) @@ -48,7 +47,7 @@ func ResolveTarget(target Target) (Target, error) { return targetFromRoot(root), nil } - if len(target.JournalDirs) == 0 && target.MachineIDPath == "" && target.JournalControlSocket == "" && target.SystemBusSocket == "" && target.JournalRuntimeDir == "" { + if len(target.JournalDirs) == 0 && target.MachineIDPath == "" && target.JournalControlSocket == "" && target.SystemBusSocket == "" { return LocalTarget(), nil } if len(target.JournalDirs) > MaxJournalDirs { @@ -82,9 +81,6 @@ func ResolveTarget(target Target) (Target, error) { if resolved.SystemBusSocket, err = validateOptionalAbsolutePath("system bus socket", target.SystemBusSocket); err != nil { return Target{}, err } - if resolved.JournalRuntimeDir, err = validateOptionalAbsolutePath("journal runtime directory", target.JournalRuntimeDir); err != nil { - return Target{}, err - } return resolved, nil } @@ -97,7 +93,6 @@ func targetFromRoot(root string) Target { MachineIDPath: filepath.Join(root, "etc", "machine-id"), JournalControlSocket: filepath.Join(root, "run", "systemd", "journal", "io.systemd.journal"), SystemBusSocket: filepath.Join(root, "run", "dbus", "system_bus_socket"), - JournalRuntimeDir: filepath.Join(root, "run", "systemd", "journal"), } } diff --git a/internal/systemd/target_test.go b/internal/systemd/target_test.go index a9288a64..c3a37782 100644 --- a/internal/systemd/target_test.go +++ b/internal/systemd/target_test.go @@ -23,7 +23,6 @@ func TestResolveTargetDefaultsToLocalPaths(t *testing.T) { assert.Equal(t, "/etc/machine-id", target.MachineIDPath) assert.Equal(t, "/run/systemd/journal/io.systemd.journal", target.JournalControlSocket) assert.Equal(t, "/run/dbus/system_bus_socket", target.SystemBusSocket) - assert.Equal(t, "/run/systemd/journal", target.JournalRuntimeDir) } func TestResolveTargetDerivesMountedRootPaths(t *testing.T) { @@ -34,7 +33,6 @@ func TestResolveTargetDerivesMountedRootPaths(t *testing.T) { assert.Equal(t, filepath.FromSlash("/host/etc/machine-id"), target.MachineIDPath) assert.Equal(t, filepath.FromSlash("/host/run/systemd/journal/io.systemd.journal"), target.JournalControlSocket) assert.Equal(t, filepath.FromSlash("/host/run/dbus/system_bus_socket"), target.SystemBusSocket) - assert.Equal(t, filepath.FromSlash("/host/run/systemd/journal"), target.JournalRuntimeDir) } func TestResolveTargetUsesOnlyExplicitPaths(t *testing.T) { @@ -48,7 +46,6 @@ func TestResolveTargetUsesOnlyExplicitPaths(t *testing.T) { assert.Equal(t, "/mnt/etc/machine-id", target.MachineIDPath) assert.Empty(t, target.JournalControlSocket) assert.Empty(t, target.SystemBusSocket) - assert.Empty(t, target.JournalRuntimeDir) } func TestResolveTargetRejectsInvalidConfiguration(t *testing.T) { diff --git a/interp/systemd_target.go b/interp/systemd_target.go index ca4a6364..a12abad7 100644 --- a/interp/systemd_target.go +++ b/interp/systemd_target.go @@ -22,7 +22,6 @@ type SystemdTargetConfig struct { MachineIDPath string JournalControlSocket string SystemBusSocket string - JournalRuntimeDir string } // WithSystemdTarget configures the trusted systemd target. Scripts cannot @@ -35,7 +34,6 @@ func WithSystemdTarget(config SystemdTargetConfig) RunnerOption { MachineIDPath: config.MachineIDPath, JournalControlSocket: config.JournalControlSocket, SystemBusSocket: config.SystemBusSocket, - JournalRuntimeDir: config.JournalRuntimeDir, }) if err != nil { return fmt.Errorf("WithSystemdTarget: %w", err) From b0849c66bb6f940bb4bf093f0f8c31377c993a9d Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:42:01 -0400 Subject: [PATCH 14/53] docs(journalctl): document restricted systemd support --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++-- SHELL_FEATURES.md | 4 +++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b7b8222c..e499b8ff 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,49 @@ interp.AllowedSystemd([]interp.SystemdControlGrant{ }) ``` -The development CLI accepts equivalent grants through `--allowed-systemd unit:mysql.service:restart+reload+read,journal:storage:read+clean`. The older `AllowedSystemServices` API and `--allowed-services` flag remain as unit-only compatibility shorthands backed by the same allowlist. The policy and authorization capability are implemented, but `systemctl` and `journalctl` builtins are not yet available. +The development CLI accepts equivalent grants through `--allowed-systemd unit:mysql.service:restart+reload+read,journal:storage:read+clean`. The older `AllowedSystemServices` API and `--allowed-services` flag remain as unit-only compatibility shorthands backed by the same allowlist. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. -**SystemdTargetConfig** selects which Linux host those builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, system bus socket, and journald runtime directory. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target so later backends can reject mixed-host configurations. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. +**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. + +The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Journal entries are checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. + +| Operation | Required target access | +|-----------|------------------------| +| Journal query, disk usage, or vacuum | `/etc/machine-id` and one or both of `/var/log/journal`, `/run/log/journal` | +| Journal rotation | `/etc/machine-id` and `/run/systemd/journal/io.systemd.journal` | +| Future `systemctl` manager operations | `/etc/machine-id` and `/run/dbus/system_bus_socket` | + +Root targets derive those paths below `Root`; explicit targets may map them to arbitrary absolute container paths. A command fails closed when one of its required paths was omitted. + +### Restricted journalctl + +`journalctl` provides bounded investigation and disk-cleanup operations without executing the host binary or accepting user-selected files, directories, journal matches, cursors, machines, or namespaces. + +| Operation | Supported flags | Required systemd grant | +|-----------|-----------------|------------------------| +| Exact unit logs | `-u UNIT` (repeatable), `-b`, `-n COUNT`, `--since TIME`, `-o short\|cat` | Exact `unit:UNIT/read` for every unit | +| Current-boot kernel logs | `-k`, `-n COUNT`, `--since TIME`, `-o short\|cat` | `journal:kernel/read` | +| Allocated journal usage | `--disk-usage` | `journal:storage/read` | +| Synchronous active-file rotation | `--rotate` | `journal:storage/clean` plus remediation mode | +| Archived-file cleanup | `--vacuum-size SIZE`, `--vacuum-time AGE`, `--dry-run` | `journal:storage/clean` plus remediation mode | + +Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. + +Log reading requires Linux, cgo, and `libsystemd`. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--rotate` may be combined with vacuum flags; rotation completes first so the newly archived files are eligible for cleanup. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. + +Vacuuming is disabled unless the trusted caller also supplies `WithJournalVacuumPolicy`. Script thresholds can only narrow this policy. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, malformed names, or files younger than the retention floor. It also enforces retained-file and retained-byte floors plus per-invocation deletion ceilings. + +```go +interp.WithJournalVacuumPolicy(interp.JournalVacuumPolicy{ + MinRetentionAge: 24 * time.Hour, + MinRetainedFiles: 4, + MinRetainedBytes: 256 * 1024 * 1024, + MaxDeletedFiles: 32, + MaxDeletedBytes: 512 * 1024 * 1024, +}) +``` + +The development CLI exposes the same policy through `--journal-vacuum-min-age`, `--journal-vacuum-min-files`, `--journal-vacuum-min-bytes`, `--journal-vacuum-max-delete-files`, and `--journal-vacuum-max-delete-bytes`. The byte-valued policy flags accept raw byte counts; builtin `--vacuum-size` accepts size suffixes. **AllowedPaths** restricts all file operations to specified directories using Go's `os.Root` API for reads and openat-based write handling for writes. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 4f92a6f0..9c29652b 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,6 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported +- ✅ `journalctl (-u UNIT...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system units or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 unit scopes; unit reads require exact `unit:UNIT/read` grants and kernel reads require `journal:kernel/read`; also supports `--disk-usage` with `journal:storage/read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `journal:storage/clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; vacuum deletion additionally requires a trusted `JournalVacuumPolicy`; bare/unrestricted reads, globbed units, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads require Linux+cgo+libsystemd, rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected @@ -122,7 +123,8 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed - ✅ AllowedSystemServices policy — one shared default-deny capability map for resources with generic `read`, `clean`, `reload`, and `restart` actions; invalid resource/action pairs are rejected, unit names are case-sensitive and are not normalized, `read` works in read-only mode, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure through `interp.AllowedSystemd` or CLI `--allowed-systemd RESOURCE:ACTION[+ACTION...]`; `AllowedSystemServices` and `--allowed-services` remain unit-only compatibility shorthands -- ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives a complete mounted-host target, while explicit journal, machine-id, Varlink, D-Bus, and runtime paths support split mount layouts without falling back to local endpoints +- ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host +- ✅ JournalVacuumPolicy — trusted retention floors and per-invocation file/byte deletion ceilings required for `journalctl --vacuum-*`; cleanup remains disabled without `interp.WithJournalVacuumPolicy` even when `journal:storage/clean` is granted - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command - ✅ ProcPath — overrides the proc filesystem path used by `ps` (default `/proc`; Linux-only; useful for testing/container environments); `ps` does not read `/proc//cmdline` From 2ce3a3f7551f5674e1e80d46c2b761c1fc372044 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 13:47:22 -0400 Subject: [PATCH 15/53] chore(compliance): register systemd dependency --- LICENSE-3rdparty.csv | 1 + internal/systemd/journal_file_unix.go | 4 ++-- internal/systemd/journal_reader_linux_cgo.go | 4 ++-- internal/systemd/journal_reader_unsupported.go | 4 ++-- internal/systemd/journal_rotate_linux.go | 4 ++-- internal/systemd/journal_rotate_unsupported.go | 4 ++-- internal/systemd/journal_usage_unix.go | 4 ++-- internal/systemd/journal_usage_unix_test.go | 4 ++-- internal/systemd/journal_usage_unsupported.go | 4 ++-- internal/systemd/journal_vacuum_unix.go | 4 ++-- internal/systemd/journal_vacuum_unix_test.go | 4 ++-- internal/systemd/journal_vacuum_unsupported.go | 4 ++-- interp/journal_vacuum_integration_test.go | 4 ++-- 13 files changed, 25 insertions(+), 24 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index e5324d09..571464e6 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -4,6 +4,7 @@ github.com/DataDog/datadog-agent/pkg/template,https://github.com/DataDog/datadog github.com/DataDog/datadog-agent/pkg/util/log,https://github.com/DataDog/datadog-agent,Apache-2.0,"Copyright 2016-present Datadog, Inc." github.com/DataDog/datadog-agent/pkg/util/scrubber,https://github.com/DataDog/datadog-agent,Apache-2.0,"Copyright 2016-present Datadog, Inc." github.com/DataDog/datadog-agent/pkg/version,https://github.com/DataDog/datadog-agent,Apache-2.0,"Copyright 2016-present Datadog, Inc." +github.com/coreos/go-systemd/v22,https://github.com/coreos/go-systemd,Apache-2.0,"Copyright 2018 CoreOS, Inc" github.com/davecgh/go-spew,https://github.com/davecgh/go-spew,ISC,Copyright (c) 2012-2016 Dave Collins github.com/ebitengine/purego,https://github.com/ebitengine/purego,Apache-2.0,Copyright 2022 The Ebitengine Authors github.com/go-ole/go-ole,https://github.com/go-ole/go-ole,MIT,"Copyright (c) 2013-2017 Yasuhiro Matsumoto" diff --git a/internal/systemd/journal_file_unix.go b/internal/systemd/journal_file_unix.go index baee3514..8e77a7a1 100644 --- a/internal/systemd/journal_file_unix.go +++ b/internal/systemd/journal_file_unix.go @@ -1,10 +1,10 @@ -//go:build linux || darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux || darwin + package systemd import ( diff --git a/internal/systemd/journal_reader_linux_cgo.go b/internal/systemd/journal_reader_linux_cgo.go index ebbe1f63..4fcef4eb 100644 --- a/internal/systemd/journal_reader_linux_cgo.go +++ b/internal/systemd/journal_reader_linux_cgo.go @@ -1,10 +1,10 @@ -//go:build linux && cgo - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux && cgo + package systemd import ( diff --git a/internal/systemd/journal_reader_unsupported.go b/internal/systemd/journal_reader_unsupported.go index a1c9f4bf..fcd2e14a 100644 --- a/internal/systemd/journal_reader_unsupported.go +++ b/internal/systemd/journal_reader_unsupported.go @@ -1,10 +1,10 @@ -//go:build !linux || !cgo - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build !linux || !cgo + package systemd import ( diff --git a/internal/systemd/journal_rotate_linux.go b/internal/systemd/journal_rotate_linux.go index 78c891a0..30660f4c 100644 --- a/internal/systemd/journal_rotate_linux.go +++ b/internal/systemd/journal_rotate_linux.go @@ -1,10 +1,10 @@ -//go:build linux - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux + package systemd import ( diff --git a/internal/systemd/journal_rotate_unsupported.go b/internal/systemd/journal_rotate_unsupported.go index b59a9498..fedca45c 100644 --- a/internal/systemd/journal_rotate_unsupported.go +++ b/internal/systemd/journal_rotate_unsupported.go @@ -1,10 +1,10 @@ -//go:build !linux - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build !linux + package systemd import ( diff --git a/internal/systemd/journal_usage_unix.go b/internal/systemd/journal_usage_unix.go index 085664e9..ffbdad76 100644 --- a/internal/systemd/journal_usage_unix.go +++ b/internal/systemd/journal_usage_unix.go @@ -1,10 +1,10 @@ -//go:build linux || darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux || darwin + package systemd import ( diff --git a/internal/systemd/journal_usage_unix_test.go b/internal/systemd/journal_usage_unix_test.go index 83bf84e9..875517c6 100644 --- a/internal/systemd/journal_usage_unix_test.go +++ b/internal/systemd/journal_usage_unix_test.go @@ -1,10 +1,10 @@ -//go:build linux || darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux || darwin + package systemd import ( diff --git a/internal/systemd/journal_usage_unsupported.go b/internal/systemd/journal_usage_unsupported.go index 79e9308c..e93ff990 100644 --- a/internal/systemd/journal_usage_unsupported.go +++ b/internal/systemd/journal_usage_unsupported.go @@ -1,10 +1,10 @@ -//go:build !linux && !darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build !linux && !darwin + package systemd import ( diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go index 614769a6..56fb5ac6 100644 --- a/internal/systemd/journal_vacuum_unix.go +++ b/internal/systemd/journal_vacuum_unix.go @@ -1,10 +1,10 @@ -//go:build linux || darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux || darwin + package systemd import ( diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go index 36ee84de..7cc23713 100644 --- a/internal/systemd/journal_vacuum_unix_test.go +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -1,10 +1,10 @@ -//go:build linux || darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux || darwin + package systemd import ( diff --git a/internal/systemd/journal_vacuum_unsupported.go b/internal/systemd/journal_vacuum_unsupported.go index 4faa5f77..f9b6c402 100644 --- a/internal/systemd/journal_vacuum_unsupported.go +++ b/internal/systemd/journal_vacuum_unsupported.go @@ -1,10 +1,10 @@ -//go:build !linux && !darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build !linux && !darwin + package systemd import ( diff --git a/interp/journal_vacuum_integration_test.go b/interp/journal_vacuum_integration_test.go index aa1f4152..6ede6622 100644 --- a/interp/journal_vacuum_integration_test.go +++ b/interp/journal_vacuum_integration_test.go @@ -1,10 +1,10 @@ -//go:build linux || darwin - // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026-present Datadog, Inc. +//go:build linux || darwin + package interp import ( From b0245f09b7550240cd298ca245932a6512242df9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 15:10:59 -0400 Subject: [PATCH 16/53] refactor(systemd): restore service allowlist name --- README.md | 6 +-- SHELL_FEATURES.md | 2 +- cmd/rshell/main.go | 66 +++++------------------ cmd/rshell/main_test.go | 15 +++--- interp/api.go | 8 +-- interp/journal_vacuum_integration_test.go | 2 +- interp/system_services.go | 47 ++++++++-------- interp/system_services_test.go | 16 +++--- 8 files changed, 62 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index e499b8ff..59de7aa0 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Every access path is default-deny: | Resource | Default | Opt-in | |----------------------|-------------------------------------|----------------------------------------------| | Command execution | All commands blocked (exit code 127)| `AllowedCommands` with namespaced command list (e.g. `rshell:cat`) | -| Systemd resources | All resources and actions blocked | `AllowedSystemd` with exact resource/action grants | +| Systemd resources | All resources and actions blocked | `AllowedSystemServices` with exact resource/action grants | | External commands | Blocked (exit code 127) | Provide an `ExecHandler` | | Filesystem access | Blocked | Configure `AllowedPaths` with `PATH[:ro|:rw]` root specs | | Environment variables| Empty (no host env inherited) | Pass variables via the `Env` option | @@ -72,7 +72,7 @@ Every access path is default-deny: **AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact resource (`unit:NAME`, `journal:all`, `journal:kernel`, `journal:storage`, or `manager`) with generic actions (`read`, `clean`, `reload`, or `restart`). Invalid resource/action combinations are rejected. Unit names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `unit:mysql` and `unit:mysql.service` are different resources. Empty unit names, whitespace, path-like names, and glob patterns are rejected. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go -interp.AllowedSystemd([]interp.SystemdControlGrant{ +interp.AllowedSystemServices([]interp.SystemdControlGrant{ { Resource: interp.SystemdUnitResource("mysql.service"), Actions: []interp.SystemdAction{ @@ -88,7 +88,7 @@ interp.AllowedSystemd([]interp.SystemdControlGrant{ }) ``` -The development CLI accepts equivalent grants through `--allowed-systemd unit:mysql.service:restart+reload+read,journal:storage:read+clean`. The older `AllowedSystemServices` API and `--allowed-services` flag remain as unit-only compatibility shorthands backed by the same allowlist. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. +The development CLI accepts equivalent grants through `--allowed-services unit:mysql.service:restart+reload+read,journal:storage:read+clean`. Bare service selectors remain shorthand for exact unit resources, so `mysql.service:read` is equivalent to `unit:mysql.service:read`. Existing `SystemServiceControlGrant{Service: ...}` values remain supported by the same allowlist. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. **SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 9c29652b..34728dbb 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -122,7 +122,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c ## Execution - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed -- ✅ AllowedSystemServices policy — one shared default-deny capability map for resources with generic `read`, `clean`, `reload`, and `restart` actions; invalid resource/action pairs are rejected, unit names are case-sensitive and are not normalized, `read` works in read-only mode, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure through `interp.AllowedSystemd` or CLI `--allowed-systemd RESOURCE:ACTION[+ACTION...]`; `AllowedSystemServices` and `--allowed-services` remain unit-only compatibility shorthands +- ✅ AllowedSystemServices policy — one shared default-deny capability map for resources with generic `read`, `clean`, `reload`, and `restart` actions; invalid resource/action pairs are skipped, unit names are case-sensitive and are not normalized, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure through `interp.AllowedSystemServices` or CLI `--allowed-services RESOURCE:ACTION[+ACTION...]`; bare service selectors remain exact unit-resource shorthands - ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host - ✅ JournalVacuumPolicy — trusted retention floors and per-invocation file/byte deletion ceilings required for `journalctl --vacuum-*`; cleanup remains disabled without `interp.WithJournalVacuumPolicy` even when `journal:storage/clean` is granted - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index ede636b5..e1ef49e9 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -40,7 +40,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowedPaths string allowedCommands string allowedServices string - allowedSystemd string allowAllCmds bool timeout time.Duration procPath string @@ -100,11 +99,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. if err != nil { return err } - systemdGrants, err := parseAllowedSystemd(allowedSystemd) - if err != nil { - return err - } - parsedMode := interp.Mode(mode) if parsedMode != interp.ModeReadOnly && parsedMode != interp.ModeRemediation { return fmt.Errorf("--mode must be one of: read-only, remediation") @@ -135,7 +129,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. allowedPaths: paths, allowedCommands: cmds, allowedServices: serviceGrants, - allowedSystemd: systemdGrants, allowAllCommands: allowAllCmds, procPath: procPath, systemdTarget: interp.SystemdTargetConfig{ @@ -197,8 +190,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().MarkHidden("command") //nolint:errcheck // flag is guaranteed to exist cmd.Flags().StringVarP(&allowedPaths, "allowed-paths", "p", "", "comma-separated list of PATH[:ro|:rw] directories the shell is allowed to access; entries without a suffix are read-only") cmd.Flags().StringVar(&allowedCommands, "allowed-commands", "", "comma-separated list of namespaced commands (e.g. rshell:cat,rshell:find)") - cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated exact unit grants in SERVICE:ACTION[+ACTION...] form; shorthand for unit: resources") - cmd.Flags().StringVar(&allowedSystemd, "allowed-systemd", "", "comma-separated systemd grants in RESOURCE:ACTION[+ACTION...] form; resources: unit:NAME, journal:all, journal:kernel, journal:storage, manager") + cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated systemd grants in RESOURCE:ACTION[+ACTION...] form; bare service names are unit resources") cmd.Flags().BoolVar(&allowAllCmds, "allow-all-commands", false, "allow execution of all commands (builtins and external)") cmd.Flags().DurationVar(&timeout, "timeout", 0, "maximum execution time for the entire shell run (e.g. 100ms, 5s, 1m)") cmd.Flags().StringVar(&procPath, "proc-path", "", "path to the proc filesystem used by ps (default \"/proc\")") @@ -280,8 +272,7 @@ func rejectLongCommand(rawArgs []string) error { type executeOpts struct { allowedPaths []string allowedCommands []string - allowedServices []interp.SystemServiceControlGrant - allowedSystemd []interp.SystemdControlGrant + allowedServices []interp.SystemdControlGrant allowAllCommands bool procPath string systemdTarget interp.SystemdTargetConfig @@ -311,15 +302,8 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i } else if len(opts.allowedCommands) > 0 { runOpts = append(runOpts, interp.AllowedCommands(opts.allowedCommands)) } - if len(opts.allowedServices) > 0 || len(opts.allowedSystemd) > 0 { - grants := append([]interp.SystemdControlGrant(nil), opts.allowedSystemd...) - for _, grant := range opts.allowedServices { - grants = append(grants, interp.SystemdControlGrant{ - Resource: interp.SystemdUnitResource(grant.Service), - Actions: append([]interp.SystemdAction(nil), grant.Actions...), - }) - } - runOpts = append(runOpts, interp.AllowedSystemd(grants)) + if len(opts.allowedServices) > 0 { + runOpts = append(runOpts, interp.AllowedSystemServices(opts.allowedServices)) } if opts.procPath != "" { runOpts = append(runOpts, interp.ProcPath(opts.procPath)) @@ -343,33 +327,7 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i return runner.Run(ctx, prog) } -func parseAllowedServices(value string) ([]interp.SystemServiceControlGrant, error) { - if value == "" { - return nil, nil - } - - entries := strings.Split(value, ",") - grants := make([]interp.SystemServiceControlGrant, 0, len(entries)) - for _, entry := range entries { - separator := strings.LastIndexByte(entry, ':') - if separator <= 0 || separator == len(entry)-1 { - return nil, fmt.Errorf("--allowed-services: invalid grant %q (expected SERVICE:ACTION[+ACTION...])", entry) - } - - actionNames := strings.Split(entry[separator+1:], "+") - actions := make([]interp.SystemServiceAction, len(actionNames)) - for i, action := range actionNames { - actions[i] = interp.SystemServiceAction(action) - } - grants = append(grants, interp.SystemServiceControlGrant{ - Service: entry[:separator], - Actions: actions, - }) - } - return grants, nil -} - -func parseAllowedSystemd(value string) ([]interp.SystemdControlGrant, error) { +func parseAllowedServices(value string) ([]interp.SystemdControlGrant, error) { if value == "" { return nil, nil } @@ -379,7 +337,7 @@ func parseAllowedSystemd(value string) ([]interp.SystemdControlGrant, error) { for _, entry := range entries { separator := strings.LastIndexByte(entry, ':') if separator <= 0 || separator == len(entry)-1 { - return nil, fmt.Errorf("--allowed-systemd: invalid grant %q (expected RESOURCE:ACTION[+ACTION...])", entry) + return nil, fmt.Errorf("--allowed-services: invalid grant %q (expected RESOURCE:ACTION[+ACTION...])", entry) } actionNames := strings.Split(entry[separator+1:], "+") @@ -387,10 +345,14 @@ func parseAllowedSystemd(value string) ([]interp.SystemdControlGrant, error) { for i, action := range actionNames { actions[i] = interp.SystemdAction(action) } - grants = append(grants, interp.SystemdControlGrant{ - Resource: interp.SystemdResource(entry[:separator]), - Actions: actions, - }) + selector := entry[:separator] + grant := interp.SystemdControlGrant{Actions: actions} + if selector == string(interp.SystemdResourceManager) || strings.HasPrefix(selector, "unit:") || strings.HasPrefix(selector, "journal:") { + grant.Resource = interp.SystemdResource(selector) + } else { + grant.Service = selector + } + grants = append(grants, grant) } return grants, nil } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index 8a6a8c8f..6edd50c1 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -180,9 +180,8 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "entries without a suffix are read-only") assert.Contains(t, stdout, "--allowed-commands") assert.Contains(t, stdout, "--allowed-services") - assert.Contains(t, stdout, "SERVICE:ACTION[+ACTION...]") - assert.Contains(t, stdout, "--allowed-systemd") assert.Contains(t, stdout, "RESOURCE:ACTION[+ACTION...]") + assert.NotContains(t, stdout, "--allowed-systemd") assert.Contains(t, stdout, "--allow-all-commands") assert.Contains(t, stdout, "file-target output redirections within :rw AllowedPaths roots") assert.Contains(t, stdout, "--timeout") @@ -349,10 +348,10 @@ func TestParseAllowedServicesUsesLastColon(t *testing.T) { assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemServiceReload}, grants[0].Actions) } -func TestAllowedSystemdFlag(t *testing.T) { +func TestAllowedServicesFlagAcceptsSystemdResources(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-systemd", "unit:mysql.service:read+restart,journal:kernel:read,journal:storage:read+clean,manager:reload", + "--allowed-services", "unit:mysql.service:read+restart,journal:kernel:read,journal:storage:read+clean,manager:reload", "--mode", "remediation", "-c", `echo hello`, ) @@ -361,18 +360,18 @@ func TestAllowedSystemdFlag(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedSystemdFlagRejectsInvalidCombination(t *testing.T) { +func TestAllowedServicesFlagRejectsInvalidSystemdCombination(t *testing.T) { code, _, stderr := runCLI(t, "--allow-all-commands", - "--allowed-systemd", "journal:storage:restart", + "--allowed-services", "journal:storage:restart", "-c", `echo hello`, ) assert.Equal(t, 1, code) assert.Contains(t, stderr, `unsupported operation "restart" on "journal:storage"`) } -func TestParseAllowedSystemdUsesLastColon(t *testing.T) { - grants, err := parseAllowedSystemd("unit:tenant:mysql.service:read+reload") +func TestParseAllowedServicesRecognizesExplicitResource(t *testing.T) { + grants, err := parseAllowedServices("unit:tenant:mysql.service:read+reload") require.NoError(t, err) require.Len(t, grants, 1) assert.Equal(t, interp.SystemdResource("unit:tenant:mysql.service"), grants[0].Resource) diff --git a/interp/api.go b/interp/api.go index f2627344..a01db5d2 100644 --- a/interp/api.go +++ b/interp/api.go @@ -81,10 +81,10 @@ type runnerConfig struct { // command. Intended for testing convenience. allowAllCommands bool - // allowedSystemd maps exact resources to their permitted actions. It is - // independent of allowAllCommands and defaults to denying every systemd - // operation. - allowedSystemd systemdGrants + // allowedSystemServices maps exact systemd resources to their permitted + // actions. It is independent of allowAllCommands and defaults to denying + // every systemd operation. + allowedSystemServices systemdGrants // systemdTarget identifies the local or mounted host used by systemd-aware // builtins. It is resolved once during construction and shared by diff --git a/interp/journal_vacuum_integration_test.go b/interp/journal_vacuum_integration_test.go index 6ede6622..3e9331c0 100644 --- a/interp/journal_vacuum_integration_test.go +++ b/interp/journal_vacuum_integration_test.go @@ -35,7 +35,7 @@ func TestJournalctlVacuumEndToEnd(t *testing.T) { runner, err := New( StdIO(nil, &stdout, &stderr), AllowedCommands([]string{"rshell:journalctl"}), - AllowedSystemd([]SystemdControlGrant{{ + AllowedSystemServices([]SystemdControlGrant{{ Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionClean}, }}), diff --git a/interp/system_services.go b/interp/system_services.go index ab81364b..9a061a03 100644 --- a/interp/system_services.go +++ b/interp/system_services.go @@ -42,13 +42,7 @@ func SystemdUnitResource(name string) SystemdResource { // SystemdOperation is one resource/action pair checked by the shared policy. type SystemdOperation = builtins.SystemdOperation -// SystemdControlGrant grants Actions for one exact Resource. -type SystemdControlGrant struct { - Resource SystemdResource - Actions []SystemdAction -} - -// Deprecated compatibility aliases for the original service-only policy. +// Compatibility aliases for the original service-only policy. type SystemServiceAction = builtins.SystemServiceAction const ( @@ -57,13 +51,19 @@ const ( SystemServiceRestart = builtins.SystemServiceRestart ) -// SystemServiceControlGrant grants Actions for one exact Service spelling. -// Service names are never normalized, expanded, or resolved as aliases. +// SystemServiceControlGrant grants Actions for one exact systemd resource. +// Service is shorthand for an exact unit resource; callers must set exactly +// one of Resource or Service. type SystemServiceControlGrant struct { - Service string - Actions []SystemServiceAction + Service string + Actions []SystemServiceAction + Resource SystemdResource } +// SystemdControlGrant is a resource-oriented alias for the original grant +// type used by AllowedSystemServices. +type SystemdControlGrant = SystemServiceControlGrant + type systemdGrants map[SystemdResource]map[SystemdAction]struct{} // AllowedSystemServices configures the system services and actions that @@ -78,7 +78,7 @@ type systemdGrants map[SystemdResource]map[SystemdAction]struct{} // // When not set (default), or when passed an empty slice, every systemd // operation is denied. This policy is not bypassed by allowing all commands. -func AllowedSystemd(grants []SystemdControlGrant) RunnerOption { +func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { return func(r *Runner) error { allowed := make(systemdGrants, len(grants)) for i, grant := range grants { @@ -105,23 +105,20 @@ func AllowedSystemd(grants []SystemdControlGrant) RunnerOption { actions[action] = struct{}{} } } - r.allowedSystemd = allowed + r.allowedSystemServices = allowed return nil } } -// AllowedSystemServices is a compatibility wrapper that adds the unit: prefix -// to each exact service name and stores the grants in the shared systemd -// allowlist. -func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { - systemdGrants := make([]SystemdControlGrant, len(grants)) - for i, grant := range grants { - systemdGrants[i] = SystemdControlGrant{ - Resource: SystemdUnitResource(grant.Service), - Actions: append([]SystemdAction(nil), grant.Actions...), - } +func systemdGrantResource(grant SystemServiceControlGrant) (SystemdResource, error) { + switch { + case grant.Resource != "" && grant.Service != "": + return "", fmt.Errorf("must not set both Resource and Service") + case grant.Resource != "": + return grant.Resource, nil + default: + return SystemdUnitResource(grant.Service), nil } - return AllowedSystemd(systemdGrants) } func validSystemdOperation(operation SystemdOperation) bool { @@ -188,7 +185,7 @@ func (r *Runner) authorizeSystemd(operations ...SystemdOperation) error { if operation.Action != SystemdActionRead && !r.remediationMode { return fmt.Errorf("systemd action %q requires remediation mode", operation.Action) } - actions := r.allowedSystemd[operation.Resource] + actions := r.allowedSystemServices[operation.Resource] if _, ok := actions[operation.Action]; !ok { return fmt.Errorf("systemd resource %q is not allowed for action %q", operation.Resource, operation.Action) } diff --git a/interp/system_services_test.go b/interp/system_services_test.go index 47b65986..caa36012 100644 --- a/interp/system_services_test.go +++ b/interp/system_services_test.go @@ -76,10 +76,12 @@ func TestAllowedSystemServicesAllowsReadOutsideRemediationMode(t *testing.T) { assert.Contains(t, err.Error(), `action "restart" requires remediation mode`) } -func TestAllowedSystemdAuthorizesJournalAndManagerResources(t *testing.T) { +func TestAllowedSystemServicesAuthorizesJournalAndManagerResources(t *testing.T) { runner, err := New( WithMode(ModeRemediation), - AllowedSystemd([]SystemdControlGrant{ + AllowedSystemServices([]SystemdControlGrant{ + {Service: "mysql.service", Actions: []SystemdAction{SystemdActionRead}}, + {Resource: SystemdUnitResource("mysql.service"), Actions: []SystemdAction{SystemdActionRestart}}, {Resource: SystemdResourceJournalKernel, Actions: []SystemdAction{SystemdActionRead}}, {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead, SystemdActionClean}}, {Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionReload}}, @@ -89,6 +91,8 @@ func TestAllowedSystemdAuthorizesJournalAndManagerResources(t *testing.T) { defer runner.Close() require.NoError(t, runner.authorizeSystemd( + SystemdOperation{Resource: SystemdUnitResource("mysql.service"), Action: SystemdActionRead}, + SystemdOperation{Resource: SystemdUnitResource("mysql.service"), Action: SystemdActionRestart}, SystemdOperation{Resource: SystemdResourceJournalKernel, Action: SystemdActionRead}, SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionClean}, SystemdOperation{Resource: SystemdResourceManager, Action: SystemdActionReload}, @@ -99,8 +103,8 @@ func TestAllowedSystemdAuthorizesJournalAndManagerResources(t *testing.T) { assert.Contains(t, err.Error(), `resource "journal:all" is not allowed`) } -func TestAllowedSystemdReadDoesNotEnableMutation(t *testing.T) { - runner, err := New(AllowedSystemd([]SystemdControlGrant{ +func TestAllowedSystemServicesReadDoesNotEnableMutation(t *testing.T) { + runner, err := New(AllowedSystemServices([]SystemdControlGrant{ {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead}}, })) require.NoError(t, err) @@ -200,7 +204,7 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { assert.Contains(t, warningOutput.String(), `AllowedSystemServices: skipping unsupported action "enable" in grant 1 for "ignored.service"`) } -func TestAllowedSystemdRejectsInvalidResourceActionCombinations(t *testing.T) { +func TestAllowedSystemServicesRejectsInvalidResourceActionCombinations(t *testing.T) { tests := []struct { name string grant SystemdControlGrant @@ -225,7 +229,7 @@ func TestAllowedSystemdRejectsInvalidResourceActionCombinations(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - runner, err := New(AllowedSystemd([]SystemdControlGrant{test.grant})) + runner, err := New(AllowedSystemServices([]SystemdControlGrant{test.grant})) if runner != nil { runner.Close() } From 5c22a5572b847aafff9e1063270adc0bef3c4d6b Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 15:48:47 -0400 Subject: [PATCH 17/53] fix(systemd): restore generic grants after rebase --- README.md | 2 +- cmd/rshell/main_test.go | 9 +++--- interp/system_services.go | 36 ++++++++++++--------- interp/system_services_test.go | 57 ++++++++++++++++++++++------------ 4 files changed, 66 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 59de7aa0..f1d07d99 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Every access path is default-deny: **AllowedCommands** restricts which commands (builtins or external) the interpreter may execute. Commands must be specified with the `rshell:` namespace prefix (e.g. `rshell:cat`, `rshell:echo`). If not set, no commands are allowed. -**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact resource (`unit:NAME`, `journal:all`, `journal:kernel`, `journal:storage`, or `manager`) with generic actions (`read`, `clean`, `reload`, or `restart`). Invalid resource/action combinations are rejected. Unit names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `unit:mysql` and `unit:mysql.service` are different resources. Empty unit names, whitespace, path-like names, and glob patterns are rejected. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. +**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact resource (`unit:NAME`, `journal:all`, `journal:kernel`, `journal:storage`, or `manager`) with generic actions (`read`, `clean`, `reload`, or `restart`). Invalid resources and unsupported resource/action combinations are skipped with warnings. Unit names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `unit:mysql` and `unit:mysql.service` are different resources. Empty unit names, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go interp.AllowedSystemServices([]interp.SystemdControlGrant{ diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index 6edd50c1..e762bae6 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -360,14 +360,15 @@ func TestAllowedServicesFlagAcceptsSystemdResources(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedServicesFlagRejectsInvalidSystemdCombination(t *testing.T) { - code, _, stderr := runCLI(t, +func TestAllowedServicesFlagWarnsAndSkipsInvalidSystemdCombination(t *testing.T) { + code, stdout, stderr := runCLI(t, "--allow-all-commands", "--allowed-services", "journal:storage:restart", "-c", `echo hello`, ) - assert.Equal(t, 1, code) - assert.Contains(t, stderr, `unsupported operation "restart" on "journal:storage"`) + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Contains(t, stderr, `skipping unsupported action "restart"`) } func TestParseAllowedServicesRecognizesExplicitResource(t *testing.T) { diff --git a/interp/system_services.go b/interp/system_services.go index 9a061a03..6df12394 100644 --- a/interp/system_services.go +++ b/interp/system_services.go @@ -66,15 +66,15 @@ type SystemdControlGrant = SystemServiceControlGrant type systemdGrants map[SystemdResource]map[SystemdAction]struct{} -// AllowedSystemServices configures the system services and actions that -// system-service builtins may use. A grant matches its Service exactly: for -// example, "mysql" and "mysql.service" are different service names. +// AllowedSystemServices configures the resources and actions that systemd-aware +// builtins may use. Unit names are matched exactly: for example, "mysql" and +// "mysql.service" are different unit resources. // -// Grants without actions are ignored. Empty service names and names containing -// whitespace, control characters, path separators, or glob patterns are -// skipped with a warning. Supported actions are read, reload, and restart; -// unsupported actions are skipped with a warning. Duplicate services and -// actions are accepted and combined idempotently. +// Grants without actions are ignored. Invalid resources and unsupported +// resource/action pairs are skipped with a warning. Supported resources are +// exact units, journal:all, journal:kernel, journal:storage, and manager. +// Supported actions are read, clean, reload, and restart. Duplicate resources +// and actions are accepted and combined idempotently. // // When not set (default), or when passed an empty slice, every systemd // operation is denied. This policy is not bypassed by allowing all commands. @@ -85,22 +85,30 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { if len(grant.Actions) == 0 { continue } - if err := validateSystemServiceName(grant.Service); err != nil { + resource, err := systemdGrantResource(grant) + if err == nil { + err = validateSystemdResource(resource) + } + if err != nil { warning := fmt.Sprintf("AllowedSystemServices: skipping grant %d: %v\n", i, err) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } - actions := allowed[grant.Service] + selector := grant.Service + if selector == "" { + selector = string(resource) + } + actions := allowed[resource] for _, action := range grant.Actions { - if !validSystemServiceAction(action) { - warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, grant.Service) + if !validSystemdOperation(SystemdOperation{Resource: resource, Action: action}) { + warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, selector) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } if actions == nil { - actions = make(map[SystemServiceAction]struct{}, len(grant.Actions)) - allowed[grant.Service] = actions + actions = make(map[SystemdAction]struct{}, len(grant.Actions)) + allowed[resource] = actions } actions[action] = struct{}{} } diff --git a/interp/system_services_test.go b/interp/system_services_test.go index caa36012..6909f857 100644 --- a/interp/system_services_test.go +++ b/interp/system_services_test.go @@ -163,7 +163,7 @@ func TestAllowedSystemServicesSkipsEmptyAndInvalidGrants(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) assert.Len(t, runner.allowedSystemServices, 1) - assert.NotContains(t, runner.allowedSystemServices, "ignored.service") + assert.NotContains(t, runner.allowedSystemServices, SystemdUnitResource("ignored.service")) warnings := runner.Warnings() require.Len(t, warnings, 8) @@ -196,7 +196,7 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceReload, "mysql.service")) - assert.NotContains(t, runner.allowedSystemServices, "ignored.service") + assert.NotContains(t, runner.allowedSystemServices, SystemdUnitResource("ignored.service")) warnings := runner.Warnings() require.Len(t, warnings, 2) @@ -204,37 +204,56 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { assert.Contains(t, warningOutput.String(), `AllowedSystemServices: skipping unsupported action "enable" in grant 1 for "ignored.service"`) } -func TestAllowedSystemServicesRejectsInvalidResourceActionCombinations(t *testing.T) { +func TestAllowedSystemServicesSkipsInvalidResourceActionCombinations(t *testing.T) { tests := []struct { - name string - grant SystemdControlGrant + name string + grant SystemdControlGrant + needle string }{ { - name: "unknown resource", - grant: SystemdControlGrant{Resource: "journal:namespace", Actions: []SystemdAction{SystemdActionRead}}, + name: "unknown resource", + grant: SystemdControlGrant{Resource: "journal:namespace", Actions: []SystemdAction{SystemdActionRead}}, + needle: `unsupported systemd resource "journal:namespace"`, }, { - name: "clean unit", - grant: SystemdControlGrant{Resource: SystemdUnitResource("mysql.service"), Actions: []SystemdAction{SystemdActionClean}}, + name: "clean unit", + grant: SystemdControlGrant{Resource: SystemdUnitResource("mysql.service"), Actions: []SystemdAction{SystemdActionClean}}, + needle: `skipping unsupported action "clean"`, }, { - name: "restart journal", - grant: SystemdControlGrant{Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRestart}}, + name: "restart journal", + grant: SystemdControlGrant{Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRestart}}, + needle: `skipping unsupported action "restart"`, }, { - name: "clean manager", - grant: SystemdControlGrant{Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionClean}}, + name: "clean manager", + grant: SystemdControlGrant{Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionClean}}, + needle: `skipping unsupported action "clean"`, + }, + { + name: "resource and service", + grant: SystemdControlGrant{ + Service: "mysql.service", + Resource: SystemdResourceManager, + Actions: []SystemdAction{SystemdActionRead}, + }, + needle: "must not set both Resource and Service", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - runner, err := New(AllowedSystemServices([]SystemdControlGrant{test.grant})) - if runner != nil { - runner.Close() - } - require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported") + var warningOutput bytes.Buffer + runner, err := New( + WarningsWriter(&warningOutput), + AllowedSystemServices([]SystemdControlGrant{test.grant}), + ) + require.NoError(t, err) + defer runner.Close() + + assert.Empty(t, runner.allowedSystemServices) + require.Len(t, runner.Warnings(), 1) + assert.Contains(t, warningOutput.String(), test.needle) }) } } From 6e4f7ce92cfb12e394917d94d0efff9d1a8e04ec Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 16:53:49 -0400 Subject: [PATCH 18/53] refactor(systemd): use service action grants --- README.md | 10 +- SHELL_FEATURES.md | 4 +- builtins/builtins.go | 21 ++- builtins/journalctl/journalctl.go | 4 +- builtins/journalctl/journalctl_test.go | 4 +- cmd/rshell/main.go | 9 +- cmd/rshell/main_test.go | 35 +++-- interp/api.go | 6 +- interp/system_services.go | 136 ++++++++++-------- interp/system_services_test.go | 25 ++-- .../cmd/journalctl/errors/denied_unit.yaml | 2 +- 11 files changed, 140 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index f1d07d99..d44b185d 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Every access path is default-deny: | Resource | Default | Opt-in | |----------------------|-------------------------------------|----------------------------------------------| | Command execution | All commands blocked (exit code 127)| `AllowedCommands` with namespaced command list (e.g. `rshell:cat`) | -| Systemd resources | All resources and actions blocked | `AllowedSystemServices` with exact resource/action grants | +| Systemd services | All services and actions blocked | `AllowedSystemServices` with exact service/action grants and fixed journal/manager capabilities | | External commands | Blocked (exit code 127) | Provide an `ExecHandler` | | Filesystem access | Blocked | Configure `AllowedPaths` with `PATH[:ro|:rw]` root specs | | Environment variables| Empty (no host env inherited) | Pass variables via the `Env` option | @@ -69,12 +69,12 @@ Every access path is default-deny: **AllowedCommands** restricts which commands (builtins or external) the interpreter may execute. Commands must be specified with the `rshell:` namespace prefix (e.g. `rshell:cat`, `rshell:echo`). If not set, no commands are allowed. -**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact resource (`unit:NAME`, `journal:all`, `journal:kernel`, `journal:storage`, or `manager`) with generic actions (`read`, `clean`, `reload`, or `restart`). Invalid resources and unsupported resource/action combinations are skipped with warnings. Unit names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `unit:mysql` and `unit:mysql.service` are different resources. Empty unit names, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. +**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact service with generic actions (`read`, `reload`, or `restart`), using `SERVICE:ACTION[+ACTION...]` syntax. Fixed non-service capabilities use the reserved selectors `journal:all`, `journal:kernel`, `journal:storage`, and `manager`, with supported actions such as `read`, `clean`, and `reload`. Invalid selectors and unsupported action combinations are skipped with warnings. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different services. Empty service names, service names containing `:`, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go interp.AllowedSystemServices([]interp.SystemdControlGrant{ { - Resource: interp.SystemdUnitResource("mysql.service"), + Service: "mysql.service", Actions: []interp.SystemdAction{ interp.SystemdActionRestart, interp.SystemdActionReload, @@ -88,7 +88,7 @@ interp.AllowedSystemServices([]interp.SystemdControlGrant{ }) ``` -The development CLI accepts equivalent grants through `--allowed-services unit:mysql.service:restart+reload+read,journal:storage:read+clean`. Bare service selectors remain shorthand for exact unit resources, so `mysql.service:read` is equivalent to `unit:mysql.service:read`. Existing `SystemServiceControlGrant{Service: ...}` values remain supported by the same allowlist. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. +The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,journal:storage:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. **SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. @@ -108,7 +108,7 @@ Root targets derive those paths below `Root`; explicit targets may map them to a | Operation | Supported flags | Required systemd grant | |-----------|-----------------|------------------------| -| Exact unit logs | `-u UNIT` (repeatable), `-b`, `-n COUNT`, `--since TIME`, `-o short\|cat` | Exact `unit:UNIT/read` for every unit | +| Exact service logs | `-u SERVICE` (repeatable), `-b`, `-n COUNT`, `--since TIME`, `-o short\|cat` | Exact `SERVICE:read` grant for every service | | Current-boot kernel logs | `-k`, `-n COUNT`, `--since TIME`, `-o short\|cat` | `journal:kernel/read` | | Allocated journal usage | `--disk-usage` | `journal:storage/read` | | Synchronous active-file rotation | `--rotate` | `journal:storage/clean` plus remediation mode | diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 34728dbb..809a35d4 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u UNIT...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system units or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 unit scopes; unit reads require exact `unit:UNIT/read` grants and kernel reads require `journal:kernel/read`; also supports `--disk-usage` with `journal:storage/read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `journal:storage/clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; vacuum deletion additionally requires a trusted `JournalVacuumPolicy`; bare/unrestricted reads, globbed units, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads require Linux+cgo+libsystemd, rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `journal:kernel/read`; also supports `--disk-usage` with `journal:storage/read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `journal:storage/clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; vacuum deletion additionally requires a trusted `JournalVacuumPolicy`; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads require Linux+cgo+libsystemd, rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected @@ -122,7 +122,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c ## Execution - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed -- ✅ AllowedSystemServices policy — one shared default-deny capability map for resources with generic `read`, `clean`, `reload`, and `restart` actions; invalid resource/action pairs are skipped, unit names are case-sensitive and are not normalized, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure through `interp.AllowedSystemServices` or CLI `--allowed-services RESOURCE:ACTION[+ACTION...]`; bare service selectors remain exact unit-resource shorthands +- ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services plus fixed journal/manager capabilities with generic `read`, `clean`, `reload`, and `restart` actions; invalid selector/action pairs are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` - ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host - ✅ JournalVacuumPolicy — trusted retention floors and per-invocation file/byte deletion ceilings required for `journalctl --vacuum-*`; cleanup remains disabled without `interp.WithJournalVacuumPolicy` even when `journal:storage/clean` is granted - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk diff --git a/builtins/builtins.go b/builtins/builtins.go index cff26da5..8e3304ed 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -58,7 +58,7 @@ type AllowedPath struct { } // SystemdAction identifies an operation that a builtin may perform on an -// explicitly configured systemd resource. +// explicitly configured systemd service or fixed capability. type SystemdAction string const ( @@ -68,10 +68,9 @@ const ( SystemdActionRestart SystemdAction = "restart" ) -// SystemdResource identifies an exact resource in the shared systemd -// capability policy. Unit resources use the "unit:" prefix. Journal and -// manager resources are fixed constants so builtins cannot authorize -// arbitrary filesystem paths or D-Bus objects. +// SystemdResource identifies a fixed non-service resource in the shared +// systemd capability policy. These constants prevent builtins from +// authorizing arbitrary filesystem paths or D-Bus objects. type SystemdResource string const ( @@ -81,15 +80,11 @@ const ( SystemdResourceManager SystemdResource = "manager" ) -// SystemdUnitResource returns the policy resource for one exact unit name. -// The interpreter validates the name before accepting or authorizing it. -func SystemdUnitResource(name string) SystemdResource { - return SystemdResource("unit:" + name) -} - -// SystemdOperation is one resource/action pair that must be authorized before -// a builtin interacts with systemd. +// SystemdOperation is one service or fixed-resource action that must be +// authorized before a builtin interacts with systemd. Exactly one of Service +// or Resource must be set. type SystemdOperation struct { + Service string Resource SystemdResource Action SystemdAction } diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index eb916e39..23892fbe 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -150,8 +150,8 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { } else { for _, unit := range units { operations = append(operations, builtins.SystemdOperation{ - Resource: builtins.SystemdUnitResource(unit), - Action: builtins.SystemdActionRead, + Service: unit, + Action: builtins.SystemdActionRead, }) } } diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index adc69e9a..c94e5ae2 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -117,8 +117,8 @@ func TestJournalctlBuildsBoundedUnitQuery(t *testing.T) { assert.Empty(t, stderr.String()) assert.Equal(t, "ready\n", stdout.String()) assert.Equal(t, []builtins.SystemdOperation{ - {Resource: builtins.SystemdUnitResource("api.service"), Action: builtins.SystemdActionRead}, - {Resource: builtins.SystemdUnitResource("worker.service"), Action: builtins.SystemdActionRead}, + {Service: "api.service", Action: builtins.SystemdActionRead}, + {Service: "worker.service", Action: builtins.SystemdActionRead}, }, authorized) require.Len(t, reader.queries, 1) assert.Equal(t, builtins.JournalQuery{ diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index e1ef49e9..6826f699 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -190,7 +190,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().MarkHidden("command") //nolint:errcheck // flag is guaranteed to exist cmd.Flags().StringVarP(&allowedPaths, "allowed-paths", "p", "", "comma-separated list of PATH[:ro|:rw] directories the shell is allowed to access; entries without a suffix are read-only") cmd.Flags().StringVar(&allowedCommands, "allowed-commands", "", "comma-separated list of namespaced commands (e.g. rshell:cat,rshell:find)") - cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated systemd grants in RESOURCE:ACTION[+ACTION...] form; bare service names are unit resources") + cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated systemd grants in SERVICE:ACTION[+ACTION...] form; journal and manager selectors grant fixed capabilities") cmd.Flags().BoolVar(&allowAllCmds, "allow-all-commands", false, "allow execution of all commands (builtins and external)") cmd.Flags().DurationVar(&timeout, "timeout", 0, "maximum execution time for the entire shell run (e.g. 100ms, 5s, 1m)") cmd.Flags().StringVar(&procPath, "proc-path", "", "path to the proc filesystem used by ps (default \"/proc\")") @@ -337,7 +337,7 @@ func parseAllowedServices(value string) ([]interp.SystemdControlGrant, error) { for _, entry := range entries { separator := strings.LastIndexByte(entry, ':') if separator <= 0 || separator == len(entry)-1 { - return nil, fmt.Errorf("--allowed-services: invalid grant %q (expected RESOURCE:ACTION[+ACTION...])", entry) + return nil, fmt.Errorf("--allowed-services: invalid grant %q (expected SERVICE:ACTION[+ACTION...])", entry) } actionNames := strings.Split(entry[separator+1:], "+") @@ -347,9 +347,12 @@ func parseAllowedServices(value string) ([]interp.SystemdControlGrant, error) { } selector := entry[:separator] grant := interp.SystemdControlGrant{Actions: actions} - if selector == string(interp.SystemdResourceManager) || strings.HasPrefix(selector, "unit:") || strings.HasPrefix(selector, "journal:") { + if selector == string(interp.SystemdResourceManager) || strings.HasPrefix(selector, "journal:") { grant.Resource = interp.SystemdResource(selector) } else { + if strings.ContainsRune(selector, ':') { + return nil, fmt.Errorf("--allowed-services: invalid service selector %q (service names must not contain ':')", selector) + } grant.Service = selector } grants = append(grants, grant) diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index e762bae6..d2288d6f 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -180,7 +180,7 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "entries without a suffix are read-only") assert.Contains(t, stdout, "--allowed-commands") assert.Contains(t, stdout, "--allowed-services") - assert.Contains(t, stdout, "RESOURCE:ACTION[+ACTION...]") + assert.Contains(t, stdout, "SERVICE:ACTION[+ACTION...]") assert.NotContains(t, stdout, "--allowed-systemd") assert.Contains(t, stdout, "--allow-all-commands") assert.Contains(t, stdout, "file-target output redirections within :rw AllowedPaths roots") @@ -340,18 +340,24 @@ func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { assert.Contains(t, stderr, "glob pattern") } -func TestParseAllowedServicesUsesLastColon(t *testing.T) { - grants, err := parseAllowedServices("tenant:mysql.service:read+reload") +func TestParseAllowedServicesKeepsReservedResourceSelector(t *testing.T) { + grants, err := parseAllowedServices("journal:storage:read+clean") require.NoError(t, err) require.Len(t, grants, 1) - assert.Equal(t, "tenant:mysql.service", grants[0].Service) - assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemServiceReload}, grants[0].Actions) + assert.Equal(t, interp.SystemdResourceJournalStorage, grants[0].Resource) + assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemdActionClean}, grants[0].Actions) } -func TestAllowedServicesFlagAcceptsSystemdResources(t *testing.T) { +func TestParseAllowedServicesRejectsColonInServiceSelector(t *testing.T) { + _, err := parseAllowedServices("tenant:mysql.service:read") + require.Error(t, err) + assert.Contains(t, err.Error(), `invalid service selector "tenant:mysql.service"`) +} + +func TestAllowedServicesFlagAcceptsServicesAndFixedResources(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "unit:mysql.service:read+restart,journal:kernel:read,journal:storage:read+clean,manager:reload", + "--allowed-services", "mysql.service:read+restart,journal:kernel:read,journal:storage:read+clean,manager:reload", "--mode", "remediation", "-c", `echo hello`, ) @@ -371,12 +377,15 @@ func TestAllowedServicesFlagWarnsAndSkipsInvalidSystemdCombination(t *testing.T) assert.Contains(t, stderr, `skipping unsupported action "restart"`) } -func TestParseAllowedServicesRecognizesExplicitResource(t *testing.T) { - grants, err := parseAllowedServices("unit:tenant:mysql.service:read+reload") - require.NoError(t, err) - require.Len(t, grants, 1) - assert.Equal(t, interp.SystemdResource("unit:tenant:mysql.service"), grants[0].Resource) - assert.Equal(t, []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionReload}, grants[0].Actions) +func TestAllowedServicesFlagRejectsColonInServiceSelector(t *testing.T) { + code, stdout, stderr := runCLI(t, + "--allow-all-commands", + "--allowed-services", "tenant:mysql.service:read", + "-c", `echo hello`, + ) + assert.Equal(t, 1, code) + assert.Empty(t, stdout) + assert.Contains(t, stderr, "service names must not contain ':'") } func TestSystemdRootFlag(t *testing.T) { diff --git a/interp/api.go b/interp/api.go index a01db5d2..2cc330ac 100644 --- a/interp/api.go +++ b/interp/api.go @@ -81,9 +81,9 @@ type runnerConfig struct { // command. Intended for testing convenience. allowAllCommands bool - // allowedSystemServices maps exact systemd resources to their permitted - // actions. It is independent of allowAllCommands and defaults to denying - // every systemd operation. + // allowedSystemServices maps exact services and fixed systemd resources to + // their permitted actions. It is independent of allowAllCommands and + // defaults to denying every systemd operation. allowedSystemServices systemdGrants // systemdTarget identifies the local or mounted host used by systemd-aware diff --git a/interp/system_services.go b/interp/system_services.go index 6df12394..d8e56692 100644 --- a/interp/system_services.go +++ b/interp/system_services.go @@ -14,7 +14,7 @@ import ( ) // SystemdAction identifies an operation that may be granted for a systemd -// resource. +// service or fixed capability. type SystemdAction = builtins.SystemdAction const ( @@ -24,7 +24,8 @@ const ( SystemdActionRestart = builtins.SystemdActionRestart ) -// SystemdResource identifies an exact resource in the shared systemd policy. +// SystemdResource identifies a fixed non-service capability in the shared +// systemd policy. type SystemdResource = builtins.SystemdResource const ( @@ -34,12 +35,8 @@ const ( SystemdResourceManager = builtins.SystemdResourceManager ) -// SystemdUnitResource returns the policy resource for one exact unit name. -func SystemdUnitResource(name string) SystemdResource { - return builtins.SystemdUnitResource(name) -} - -// SystemdOperation is one resource/action pair checked by the shared policy. +// SystemdOperation is one service or fixed-resource action checked by the +// shared policy. type SystemdOperation = builtins.SystemdOperation // Compatibility aliases for the original service-only policy. @@ -51,30 +48,33 @@ const ( SystemServiceRestart = builtins.SystemServiceRestart ) -// SystemServiceControlGrant grants Actions for one exact systemd resource. -// Service is shorthand for an exact unit resource; callers must set exactly -// one of Resource or Service. +// SystemServiceControlGrant grants Actions for one exact Service or one fixed +// non-service Resource. Callers must set exactly one of Service or Resource. type SystemServiceControlGrant struct { Service string Actions []SystemServiceAction Resource SystemdResource } -// SystemdControlGrant is a resource-oriented alias for the original grant -// type used by AllowedSystemServices. +// SystemdControlGrant is an alias for the shared systemd policy grant type. type SystemdControlGrant = SystemServiceControlGrant -type systemdGrants map[SystemdResource]map[SystemdAction]struct{} +type systemdGrantTarget struct { + service string + resource SystemdResource +} + +type systemdGrants map[systemdGrantTarget]map[SystemdAction]struct{} -// AllowedSystemServices configures the resources and actions that systemd-aware -// builtins may use. Unit names are matched exactly: for example, "mysql" and -// "mysql.service" are different unit resources. +// AllowedSystemServices configures the services, fixed resources, and actions +// that systemd-aware builtins may use. Service names are matched exactly: for +// example, "mysql" and "mysql.service" are different services. // -// Grants without actions are ignored. Invalid resources and unsupported -// resource/action pairs are skipped with a warning. Supported resources are -// exact units, journal:all, journal:kernel, journal:storage, and manager. -// Supported actions are read, clean, reload, and restart. Duplicate resources -// and actions are accepted and combined idempotently. +// Grants without actions are ignored. Invalid targets and unsupported +// target/action pairs are skipped with a warning. Supported fixed resources +// are journal:all, journal:kernel, journal:storage, and manager. Supported +// actions are read, clean, reload, and restart. Duplicate targets and actions +// are accepted and combined idempotently. // // When not set (default), or when passed an empty slice, every systemd // operation is denied. This policy is not bypassed by allowing all commands. @@ -85,30 +85,23 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { if len(grant.Actions) == 0 { continue } - resource, err := systemdGrantResource(grant) - if err == nil { - err = validateSystemdResource(resource) - } + target, err := makeSystemdGrantTarget(grant.Service, grant.Resource) if err != nil { warning := fmt.Sprintf("AllowedSystemServices: skipping grant %d: %v\n", i, err) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } - selector := grant.Service - if selector == "" { - selector = string(resource) - } - actions := allowed[resource] + actions := allowed[target] for _, action := range grant.Actions { - if !validSystemdOperation(SystemdOperation{Resource: resource, Action: action}) { - warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, selector) + if !validSystemdOperation(target, action) { + warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, target.selector()) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } if actions == nil { actions = make(map[SystemdAction]struct{}, len(grant.Actions)) - allowed[resource] = actions + allowed[target] = actions } actions[action] = struct{}{} } @@ -118,39 +111,56 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { } } -func systemdGrantResource(grant SystemServiceControlGrant) (SystemdResource, error) { +func makeSystemdGrantTarget(service string, resource SystemdResource) (systemdGrantTarget, error) { switch { - case grant.Resource != "" && grant.Service != "": - return "", fmt.Errorf("must not set both Resource and Service") - case grant.Resource != "": - return grant.Resource, nil + case resource != "" && service != "": + return systemdGrantTarget{}, fmt.Errorf("must not set both Resource and Service") + case service != "": + if err := validateSystemServiceName(service); err != nil { + return systemdGrantTarget{}, err + } + return systemdGrantTarget{service: service}, nil + case resource != "": + if err := validateSystemdResource(resource); err != nil { + return systemdGrantTarget{}, err + } + return systemdGrantTarget{resource: resource}, nil default: - return SystemdUnitResource(grant.Service), nil + return systemdGrantTarget{}, validateSystemServiceName("") } } -func validSystemdOperation(operation SystemdOperation) bool { +func (target systemdGrantTarget) selector() string { + if target.service != "" { + return target.service + } + return string(target.resource) +} + +func (target systemdGrantTarget) description() string { + if target.service != "" { + return fmt.Sprintf("system service %q", target.service) + } + return fmt.Sprintf("systemd resource %q", target.resource) +} + +func validSystemdOperation(target systemdGrantTarget, action SystemdAction) bool { switch { - case strings.HasPrefix(string(operation.Resource), "unit:"): - return operation.Action == SystemdActionRead || operation.Action == SystemdActionReload || operation.Action == SystemdActionRestart - case operation.Resource == SystemdResourceJournalAll, - operation.Resource == SystemdResourceJournalKernel: - return operation.Action == SystemdActionRead - case operation.Resource == SystemdResourceJournalStorage: - return operation.Action == SystemdActionRead || operation.Action == SystemdActionClean - case operation.Resource == SystemdResourceManager: - return operation.Action == SystemdActionRead || operation.Action == SystemdActionReload + case target.service != "": + return action == SystemdActionRead || action == SystemdActionReload || action == SystemdActionRestart + case target.resource == SystemdResourceJournalAll, + target.resource == SystemdResourceJournalKernel: + return action == SystemdActionRead + case target.resource == SystemdResourceJournalStorage: + return action == SystemdActionRead || action == SystemdActionClean + case target.resource == SystemdResourceManager: + return action == SystemdActionRead || action == SystemdActionReload default: return false } } func validateSystemdResource(resource SystemdResource) error { - const unitPrefix = "unit:" - resourceName := string(resource) - if strings.HasPrefix(resourceName, unitPrefix) { - return validateSystemServiceName(resourceName[len(unitPrefix):]) - } switch resource { case SystemdResourceJournalAll, SystemdResourceJournalKernel, SystemdResourceJournalStorage, SystemdResourceManager: return nil @@ -166,6 +176,9 @@ func validateSystemServiceName(service string) error { if strings.ContainsRune(service, '/') || strings.ContainsRune(service, '\\') { return fmt.Errorf("system service name %q must not contain a path separator", service) } + if strings.ContainsRune(service, ':') { + return fmt.Errorf("system service name %q must not contain ':'", service) + } for _, r := range service { switch r { case '*', '?', '[', ']': @@ -184,18 +197,19 @@ func (r *Runner) authorizeSystemd(operations ...SystemdOperation) error { } for _, operation := range operations { - if err := validateSystemdResource(operation.Resource); err != nil { + target, err := makeSystemdGrantTarget(operation.Service, operation.Resource) + if err != nil { return err } - if !validSystemdOperation(operation) { - return fmt.Errorf("unsupported systemd operation %q on %q", operation.Action, operation.Resource) + if !validSystemdOperation(target, operation.Action) { + return fmt.Errorf("unsupported systemd action %q for %s", operation.Action, target.description()) } if operation.Action != SystemdActionRead && !r.remediationMode { return fmt.Errorf("systemd action %q requires remediation mode", operation.Action) } - actions := r.allowedSystemServices[operation.Resource] + actions := r.allowedSystemServices[target] if _, ok := actions[operation.Action]; !ok { - return fmt.Errorf("systemd resource %q is not allowed for action %q", operation.Resource, operation.Action) + return fmt.Errorf("%s is not allowed for action %q", target.description(), operation.Action) } } return nil @@ -207,7 +221,7 @@ func (r *Runner) authorizeSystemServices(action SystemServiceAction, services .. } operations := make([]SystemdOperation, len(services)) for i, service := range services { - operations[i] = SystemdOperation{Resource: SystemdUnitResource(service), Action: action} + operations[i] = SystemdOperation{Service: service, Action: action} } return r.authorizeSystemd(operations...) } diff --git a/interp/system_services_test.go b/interp/system_services_test.go index 6909f857..821d7d2b 100644 --- a/interp/system_services_test.go +++ b/interp/system_services_test.go @@ -41,12 +41,12 @@ func TestAllowedSystemServicesAuthorizesExactServiceAndAction(t *testing.T) { err = runner.authorizeSystemServices(SystemServiceRestart, "nginx.service") require.Error(t, err) - assert.Contains(t, err.Error(), `systemd resource "unit:nginx.service" is not allowed for action "restart"`) + assert.Contains(t, err.Error(), `system service "nginx.service" is not allowed for action "restart"`) for _, service := range []string{"mysql", "MYSQL.service"} { err = runner.authorizeSystemServices(SystemServiceRead, service) require.Error(t, err) - assert.Contains(t, err.Error(), `systemd resource "unit:`+service+`" is not allowed`) + assert.Contains(t, err.Error(), `system service "`+service+`" is not allowed`) } } @@ -81,7 +81,7 @@ func TestAllowedSystemServicesAuthorizesJournalAndManagerResources(t *testing.T) WithMode(ModeRemediation), AllowedSystemServices([]SystemdControlGrant{ {Service: "mysql.service", Actions: []SystemdAction{SystemdActionRead}}, - {Resource: SystemdUnitResource("mysql.service"), Actions: []SystemdAction{SystemdActionRestart}}, + {Service: "mysql.service", Actions: []SystemdAction{SystemdActionRestart}}, {Resource: SystemdResourceJournalKernel, Actions: []SystemdAction{SystemdActionRead}}, {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead, SystemdActionClean}}, {Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionReload}}, @@ -91,8 +91,8 @@ func TestAllowedSystemServicesAuthorizesJournalAndManagerResources(t *testing.T) defer runner.Close() require.NoError(t, runner.authorizeSystemd( - SystemdOperation{Resource: SystemdUnitResource("mysql.service"), Action: SystemdActionRead}, - SystemdOperation{Resource: SystemdUnitResource("mysql.service"), Action: SystemdActionRestart}, + SystemdOperation{Service: "mysql.service", Action: SystemdActionRead}, + SystemdOperation{Service: "mysql.service", Action: SystemdActionRestart}, SystemdOperation{Resource: SystemdResourceJournalKernel, Action: SystemdActionRead}, SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionClean}, SystemdOperation{Resource: SystemdResourceManager, Action: SystemdActionReload}, @@ -152,6 +152,7 @@ func TestAllowedSystemServicesSkipsEmptyAndInvalidGrants(t *testing.T) { {Service: "/etc/systemd/system/mysql.service", Actions: []SystemServiceAction{SystemServiceRead}}, {Service: "C:\\svc", Actions: []SystemServiceAction{SystemServiceRead}}, {Service: "..\\svc", Actions: []SystemServiceAction{SystemServiceRead}}, + {Service: "tenant:mysql.service", Actions: []SystemServiceAction{SystemServiceRead}}, {Service: "mysql*.service", Actions: []SystemServiceAction{SystemServiceRead}}, }), // Applying AllowedPaths after AllowedSystemServices verifies that one @@ -163,16 +164,17 @@ func TestAllowedSystemServicesSkipsEmptyAndInvalidGrants(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) assert.Len(t, runner.allowedSystemServices, 1) - assert.NotContains(t, runner.allowedSystemServices, SystemdUnitResource("ignored.service")) + assert.NotContains(t, runner.allowedSystemServices, systemdGrantTarget{service: "ignored.service"}) warnings := runner.Warnings() - require.Len(t, warnings, 8) + require.Len(t, warnings, 9) for _, needle := range []string{ "AllowedSystemServices: skipping grant 2: system service name must not be empty", "whitespace or control characters", `AllowedSystemServices: skipping grant 6: system service name "C:\\svc" must not contain a path separator`, `AllowedSystemServices: skipping grant 7: system service name "..\\svc" must not contain a path separator`, "path separator", + "must not contain ':'", "glob pattern", "AllowedPaths: skipping", } { @@ -196,7 +198,7 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceReload, "mysql.service")) - assert.NotContains(t, runner.allowedSystemServices, SystemdUnitResource("ignored.service")) + assert.NotContains(t, runner.allowedSystemServices, systemdGrantTarget{service: "ignored.service"}) warnings := runner.Warnings() require.Len(t, warnings, 2) @@ -216,8 +218,8 @@ func TestAllowedSystemServicesSkipsInvalidResourceActionCombinations(t *testing. needle: `unsupported systemd resource "journal:namespace"`, }, { - name: "clean unit", - grant: SystemdControlGrant{Resource: SystemdUnitResource("mysql.service"), Actions: []SystemdAction{SystemdActionClean}}, + name: "clean service", + grant: SystemdControlGrant{Service: "mysql.service", Actions: []SystemdAction{SystemdActionClean}}, needle: `skipping unsupported action "clean"`, }, { @@ -274,8 +276,9 @@ func TestAuthorizeSystemServicesRejectsInvalidRequests(t *testing.T) { services []string needle string }{ - {name: "unknown action", action: "stop", services: []string{"mysql.service"}, needle: "unsupported systemd operation"}, + {name: "unknown action", action: "stop", services: []string{"mysql.service"}, needle: "unsupported systemd action"}, {name: "no services", action: SystemServiceRead, needle: "at least one system service"}, + {name: "runtime resource separator", action: SystemServiceRead, services: []string{"tenant:mysql.service"}, needle: "must not contain ':'"}, {name: "runtime glob", action: SystemServiceRead, services: []string{"mysql*.service"}, needle: "glob pattern"}, {name: "runtime backslash path", action: SystemServiceRead, services: []string{"..\\mysql.service"}, needle: "path separator"}, } diff --git a/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml b/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml index 6b07d136..34d462d7 100644 --- a/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml +++ b/tests/scenarios/cmd/journalctl/errors/denied_unit.yaml @@ -7,5 +7,5 @@ input: expect: stdout: |+ stderr: |+ - journalctl: systemd resource "unit:api.service" is not allowed for action "read" + journalctl: system service "api.service" is not allowed for action "read" exit_code: 1 From a73bf2535e7068cad5f358adac1b8a7615950863 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Tue, 14 Jul 2026 17:13:20 -0400 Subject: [PATCH 19/53] refactor(systemd): use service-only grants --- README.md | 32 ++--- SHELL_FEATURES.md | 5 +- builtins/builtins.go | 25 ++-- builtins/journalctl/journalctl.go | 12 +- builtins/journalctl/journalctl_test.go | 16 +-- cmd/rshell/main.go | 44 +----- cmd/rshell/main_test.go | 49 ++----- internal/systemd/client.go | 12 +- internal/systemd/journal_files_test.go | 4 +- internal/systemd/journal_usage_unix_test.go | 6 +- internal/systemd/journal_vacuum_unix.go | 22 +-- internal/systemd/journal_vacuum_unix_test.go | 82 +++-------- internal/systemd/vacuum_policy.go | 40 ------ interp/api.go | 7 +- interp/journal_vacuum.go | 29 ---- interp/journal_vacuum_integration_test.go | 9 +- interp/journal_vacuum_test.go | 69 ---------- interp/system_services.go | 130 ++++-------------- interp/system_services_test.go | 95 ++----------- .../cmd/journalctl/errors/denied_storage.yaml | 4 +- .../cmd/journalctl/errors/vacuum_denied.yaml | 4 +- 21 files changed, 126 insertions(+), 570 deletions(-) delete mode 100644 internal/systemd/vacuum_policy.go delete mode 100644 interp/journal_vacuum.go delete mode 100644 interp/journal_vacuum_test.go diff --git a/README.md b/README.md index d44b185d..3ebad34a 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Every access path is default-deny: | Resource | Default | Opt-in | |----------------------|-------------------------------------|----------------------------------------------| | Command execution | All commands blocked (exit code 127)| `AllowedCommands` with namespaced command list (e.g. `rshell:cat`) | -| Systemd services | All services and actions blocked | `AllowedSystemServices` with exact service/action grants and fixed journal/manager capabilities | +| Systemd services | All services and actions blocked | `AllowedSystemServices` with exact service/action grants | | External commands | Blocked (exit code 127) | Provide an `ExecHandler` | | Filesystem access | Blocked | Configure `AllowedPaths` with `PATH[:ro|:rw]` root specs | | Environment variables| Empty (no host env inherited) | Pass variables via the `Env` option | @@ -69,7 +69,7 @@ Every access path is default-deny: **AllowedCommands** restricts which commands (builtins or external) the interpreter may execute. Commands must be specified with the `rshell:` namespace prefix (e.g. `rshell:cat`, `rshell:echo`). If not set, no commands are allowed. -**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact service with generic actions (`read`, `reload`, or `restart`), using `SERVICE:ACTION[+ACTION...]` syntax. Fixed non-service capabilities use the reserved selectors `journal:all`, `journal:kernel`, `journal:storage`, and `manager`, with supported actions such as `read`, `clean`, and `reload`. Invalid selectors and unsupported action combinations are skipped with warnings. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different services. Empty service names, service names containing `:`, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. +**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact service with generic actions (`read`, `clean`, `reload`, or `restart`), using `SERVICE:ACTION[+ACTION...]` syntax. Invalid services and unsupported actions are skipped with warnings. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different services. Empty service names, service names containing `:`, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go interp.AllowedSystemServices([]interp.SystemdControlGrant{ @@ -82,13 +82,13 @@ interp.AllowedSystemServices([]interp.SystemdControlGrant{ }, }, { - Resource: interp.SystemdResourceJournalStorage, - Actions: []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionClean}, + Service: "systemd-journald.service", + Actions: []interp.SystemdAction{interp.SystemdActionRead, interp.SystemdActionClean}, }, }) ``` -The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,journal:storage:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. +The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,systemd-journald.service:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. **SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. @@ -109,28 +109,16 @@ Root targets derive those paths below `Root`; explicit targets may map them to a | Operation | Supported flags | Required systemd grant | |-----------|-----------------|------------------------| | Exact service logs | `-u SERVICE` (repeatable), `-b`, `-n COUNT`, `--since TIME`, `-o short\|cat` | Exact `SERVICE:read` grant for every service | -| Current-boot kernel logs | `-k`, `-n COUNT`, `--since TIME`, `-o short\|cat` | `journal:kernel/read` | -| Allocated journal usage | `--disk-usage` | `journal:storage/read` | -| Synchronous active-file rotation | `--rotate` | `journal:storage/clean` plus remediation mode | -| Archived-file cleanup | `--vacuum-size SIZE`, `--vacuum-time AGE`, `--dry-run` | `journal:storage/clean` plus remediation mode | +| Current-boot kernel logs | `-k`, `-n COUNT`, `--since TIME`, `-o short\|cat` | `systemd-journald.service:read` | +| Allocated journal usage | `--disk-usage` | `systemd-journald.service:read` | +| Synchronous active-file rotation | `--rotate` | `systemd-journald.service:clean` plus remediation mode | +| Archived-file cleanup | `--vacuum-size SIZE`, `--vacuum-time AGE`, `--dry-run` | `systemd-journald.service:clean` plus remediation mode | Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. Log reading requires Linux, cgo, and `libsystemd`. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--rotate` may be combined with vacuum flags; rotation completes first so the newly archived files are eligible for cleanup. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. -Vacuuming is disabled unless the trusted caller also supplies `WithJournalVacuumPolicy`. Script thresholds can only narrow this policy. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, malformed names, or files younger than the retention floor. It also enforces retained-file and retained-byte floors plus per-invocation deletion ceilings. - -```go -interp.WithJournalVacuumPolicy(interp.JournalVacuumPolicy{ - MinRetentionAge: 24 * time.Hour, - MinRetainedFiles: 4, - MinRetainedBytes: 256 * 1024 * 1024, - MaxDeletedFiles: 32, - MaxDeletedBytes: 512 * 1024 * 1024, -}) -``` - -The development CLI exposes the same policy through `--journal-vacuum-min-age`, `--journal-vacuum-min-files`, `--journal-vacuum-min-bytes`, `--journal-vacuum-max-delete-files`, and `--journal-vacuum-max-delete-bytes`. The byte-valued policy flags accept raw byte counts; builtin `--vacuum-size` accepts size suffixes. +Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. **AllowedPaths** restricts all file operations to specified directories using Go's `os.Root` API for reads and openat-based write handling for writes. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 809a35d4..86b51d92 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `journal:kernel/read`; also supports `--disk-usage` with `journal:storage/read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `journal:storage/clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; vacuum deletion additionally requires a trusted `JournalVacuumPolicy`; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads require Linux+cgo+libsystemd, rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads require Linux+cgo+libsystemd, rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected @@ -122,9 +122,8 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c ## Execution - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed -- ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services plus fixed journal/manager capabilities with generic `read`, `clean`, `reload`, and `restart` actions; invalid selector/action pairs are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` +- ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services with generic `read`, `clean`, `reload`, and `restart` actions; invalid services/actions are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` - ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host -- ✅ JournalVacuumPolicy — trusted retention floors and per-invocation file/byte deletion ceilings required for `journalctl --vacuum-*`; cleanup remains disabled without `interp.WithJournalVacuumPolicy` even when `journal:storage/clean` is granted - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command - ✅ ProcPath — overrides the proc filesystem path used by `ps` (default `/proc`; Linux-only; useful for testing/container environments); `ps` does not read `/proc//cmdline` diff --git a/builtins/builtins.go b/builtins/builtins.go index 8e3304ed..f536f537 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -58,7 +58,7 @@ type AllowedPath struct { } // SystemdAction identifies an operation that a builtin may perform on an -// explicitly configured systemd service or fixed capability. +// explicitly configured systemd service. type SystemdAction string const ( @@ -68,25 +68,17 @@ const ( SystemdActionRestart SystemdAction = "restart" ) -// SystemdResource identifies a fixed non-service resource in the shared -// systemd capability policy. These constants prevent builtins from -// authorizing arbitrary filesystem paths or D-Bus objects. -type SystemdResource string - const ( - SystemdResourceJournalAll SystemdResource = "journal:all" - SystemdResourceJournalKernel SystemdResource = "journal:kernel" - SystemdResourceJournalStorage SystemdResource = "journal:storage" - SystemdResourceManager SystemdResource = "manager" + // SystemdJournaldService is the exact service name used for journal-wide + // operations such as kernel log reads, disk usage, rotation, and vacuuming. + SystemdJournaldService = "systemd-journald.service" ) -// SystemdOperation is one service or fixed-resource action that must be -// authorized before a builtin interacts with systemd. Exactly one of Service -// or Resource must be set. +// SystemdOperation is one service action that must be authorized before a +// builtin interacts with systemd. type SystemdOperation struct { - Service string - Resource SystemdResource - Action SystemdAction + Service string + Action SystemdAction } // Deprecated compatibility aliases for the service-only policy introduced @@ -95,6 +87,7 @@ type SystemServiceAction = SystemdAction const ( SystemServiceRead = SystemdActionRead + SystemServiceClean = SystemdActionClean SystemServiceReload = SystemdActionReload SystemServiceRestart = SystemdActionRestart ) diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index 23892fbe..ccac388e 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -144,8 +144,8 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { operations := make([]builtins.SystemdOperation, 0, len(units)+1) if *options.kernel { operations = append(operations, builtins.SystemdOperation{ - Resource: builtins.SystemdResourceJournalKernel, - Action: builtins.SystemdActionRead, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, }) } else { for _, unit := range units { @@ -200,8 +200,8 @@ func runDiskUsage(ctx context.Context, callCtx *builtins.CallContext, fs *builti return builtins.Result{Code: 1} } err := callCtx.AuthorizeSystemd(builtins.SystemdOperation{ - Resource: builtins.SystemdResourceJournalStorage, - Action: builtins.SystemdActionRead, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, }) if err != nil { callCtx.Errf("journalctl: %s\n", err) @@ -272,8 +272,8 @@ func (options flags) runMaintenance(ctx context.Context, callCtx *builtins.CallC return builtins.Result{Code: 1} } err := callCtx.AuthorizeSystemd(builtins.SystemdOperation{ - Resource: builtins.SystemdResourceJournalStorage, - Action: builtins.SystemdActionClean, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionClean, }) if err != nil { callCtx.Errf("journalctl: %s\n", err) diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index c94e5ae2..f818efe9 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -153,8 +153,8 @@ func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { assert.Empty(t, stderr.String()) assert.Equal(t, "Jul 14 12:34:56 host\\tname kernel\\x1b[\\xff]: cafe\u0301\\nline\\x00\\u202e\n", stdout.String()) assert.Equal(t, []builtins.SystemdOperation{{ - Resource: builtins.SystemdResourceJournalKernel, - Action: builtins.SystemdActionRead, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, }}, authorized) require.Len(t, reader.queries, 1) assert.True(t, reader.queries[0].Kernel) @@ -180,8 +180,8 @@ func TestJournalctlDiskUsageUsesStorageReadCapability(t *testing.T) { assert.Empty(t, stderr.String()) assert.Equal(t, "Archived and active journals take up 5.0M in the file system.\n", stdout.String()) assert.Equal(t, []builtins.SystemdOperation{{ - Resource: builtins.SystemdResourceJournalStorage, - Action: builtins.SystemdActionRead, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionRead, }}, authorized) assert.Equal(t, 1, storage.calls) assert.Empty(t, reader.queries) @@ -231,8 +231,8 @@ func TestJournalctlRotateUsesStorageCleanCapability(t *testing.T) { assert.Empty(t, stderr.String()) assert.Equal(t, "Journal rotation completed.\n", stdout.String()) assert.Equal(t, []builtins.SystemdOperation{{ - Resource: builtins.SystemdResourceJournalStorage, - Action: builtins.SystemdActionClean, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionClean, }}, authorized) assert.Equal(t, 1, rotator.calls) } @@ -335,8 +335,8 @@ func TestJournalctlVacuumAuthorizesCleanAndBuildsRequest(t *testing.T) { assert.Empty(t, stderr.String()) assert.Equal(t, "Vacuuming would free 3.0M from 2 archived journal files.\n", stdout.String()) assert.Equal(t, []builtins.SystemdOperation{{ - Resource: builtins.SystemdResourceJournalStorage, - Action: builtins.SystemdActionClean, + Service: builtins.SystemdJournaldService, + Action: builtins.SystemdActionClean, }}, authorized) require.Len(t, cleaner.requests, 1) assert.Equal(t, builtins.JournalVacuumRequest{ diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index 6826f699..aa62ccd1 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -48,11 +48,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. machineIDPath string journalSocket string systemBusSocket string - vacuumMinAge time.Duration - vacuumMinFiles int - vacuumMinBytes uint64 - vacuumMaxFiles int - vacuumMaxBytes uint64 mode string ) @@ -109,21 +104,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. configuredJournalDirs = strings.Split(journalDirs, ",") } systemdTargetSet := systemdRoot != "" || journalDirs != "" || machineIDPath != "" || journalSocket != "" || systemBusSocket != "" - vacuumPolicySet := cmd.Flags().Changed("journal-vacuum-min-age") || - cmd.Flags().Changed("journal-vacuum-min-files") || - cmd.Flags().Changed("journal-vacuum-min-bytes") || - cmd.Flags().Changed("journal-vacuum-max-delete-files") || - cmd.Flags().Changed("journal-vacuum-max-delete-bytes") - var vacuumPolicy *interp.JournalVacuumPolicy - if vacuumPolicySet { - vacuumPolicy = &interp.JournalVacuumPolicy{ - MinRetentionAge: vacuumMinAge, - MinRetainedFiles: vacuumMinFiles, - MinRetainedBytes: vacuumMinBytes, - MaxDeletedFiles: vacuumMaxFiles, - MaxDeletedBytes: vacuumMaxBytes, - } - } execOpts := executeOpts{ allowedPaths: paths, @@ -139,7 +119,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. SystemBusSocket: systemBusSocket, }, systemdTargetSet: systemdTargetSet, - vacuumPolicy: vacuumPolicy, mode: parsedMode, } @@ -190,7 +169,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().MarkHidden("command") //nolint:errcheck // flag is guaranteed to exist cmd.Flags().StringVarP(&allowedPaths, "allowed-paths", "p", "", "comma-separated list of PATH[:ro|:rw] directories the shell is allowed to access; entries without a suffix are read-only") cmd.Flags().StringVar(&allowedCommands, "allowed-commands", "", "comma-separated list of namespaced commands (e.g. rshell:cat,rshell:find)") - cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated systemd grants in SERVICE:ACTION[+ACTION...] form; journal and manager selectors grant fixed capabilities") + cmd.Flags().StringVar(&allowedServices, "allowed-services", "", "comma-separated systemd service grants in SERVICE:ACTION[+ACTION...] form") cmd.Flags().BoolVar(&allowAllCmds, "allow-all-commands", false, "allow execution of all commands (builtins and external)") cmd.Flags().DurationVar(&timeout, "timeout", 0, "maximum execution time for the entire shell run (e.g. 100ms, 5s, 1m)") cmd.Flags().StringVar(&procPath, "proc-path", "", "path to the proc filesystem used by ps (default \"/proc\")") @@ -199,11 +178,6 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmd.Flags().StringVar(&machineIDPath, "systemd-machine-id-path", "", "machine-id file for an explicit systemd target") cmd.Flags().StringVar(&journalSocket, "systemd-journal-socket", "", "journald Varlink socket for an explicit systemd target") cmd.Flags().StringVar(&systemBusSocket, "systemd-bus-socket", "", "system D-Bus socket for an explicit systemd target") - cmd.Flags().DurationVar(&vacuumMinAge, "journal-vacuum-min-age", 0, "minimum age of archived journals eligible for cleanup (required to enable cleanup)") - cmd.Flags().IntVar(&vacuumMinFiles, "journal-vacuum-min-files", 0, "minimum archived journal files retained across cleanup") - cmd.Flags().Uint64Var(&vacuumMinBytes, "journal-vacuum-min-bytes", 0, "minimum allocated archived journal bytes retained across cleanup") - cmd.Flags().IntVar(&vacuumMaxFiles, "journal-vacuum-max-delete-files", 0, "maximum archived journal files deleted per invocation (required to enable cleanup)") - cmd.Flags().Uint64Var(&vacuumMaxBytes, "journal-vacuum-max-delete-bytes", 0, "maximum allocated journal bytes deleted per invocation (required to enable cleanup)") cmd.Flags().StringVar(&mode, "mode", "read-only", "shell execution mode: read-only (default) or remediation (enables file-target output redirections within :rw AllowedPaths roots)") if err := cmd.ExecuteContext(ctx); err != nil { @@ -277,7 +251,6 @@ type executeOpts struct { procPath string systemdTarget interp.SystemdTargetConfig systemdTargetSet bool - vacuumPolicy *interp.JournalVacuumPolicy mode interp.Mode } @@ -311,9 +284,6 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i if opts.systemdTargetSet { runOpts = append(runOpts, interp.WithSystemdTarget(opts.systemdTarget)) } - if opts.vacuumPolicy != nil { - runOpts = append(runOpts, interp.WithJournalVacuumPolicy(*opts.vacuumPolicy)) - } if opts.mode != "" { runOpts = append(runOpts, interp.WithMode(opts.mode)) } @@ -346,16 +316,10 @@ func parseAllowedServices(value string) ([]interp.SystemdControlGrant, error) { actions[i] = interp.SystemdAction(action) } selector := entry[:separator] - grant := interp.SystemdControlGrant{Actions: actions} - if selector == string(interp.SystemdResourceManager) || strings.HasPrefix(selector, "journal:") { - grant.Resource = interp.SystemdResource(selector) - } else { - if strings.ContainsRune(selector, ':') { - return nil, fmt.Errorf("--allowed-services: invalid service selector %q (service names must not contain ':')", selector) - } - grant.Service = selector + if strings.ContainsRune(selector, ':') { + return nil, fmt.Errorf("--allowed-services: invalid service selector %q (service names must not contain ':')", selector) } - grants = append(grants, grant) + grants = append(grants, interp.SystemdControlGrant{Service: selector, Actions: actions}) } return grants, nil } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index d2288d6f..cb86b023 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -190,9 +190,7 @@ func TestHelp(t *testing.T) { assert.Contains(t, stdout, "--systemd-machine-id-path") assert.Contains(t, stdout, "--systemd-journal-socket") assert.Contains(t, stdout, "--systemd-bus-socket") - assert.Contains(t, stdout, "--journal-vacuum-min-age") - assert.Contains(t, stdout, "--journal-vacuum-max-delete-files") - assert.Contains(t, stdout, "--journal-vacuum-max-delete-bytes") + assert.NotContains(t, stdout, "--journal-vacuum-") assert.NotContains(t, stdout, "--command", "-c/--command should be hidden from help") } @@ -340,11 +338,11 @@ func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { assert.Contains(t, stderr, "glob pattern") } -func TestParseAllowedServicesKeepsReservedResourceSelector(t *testing.T) { - grants, err := parseAllowedServices("journal:storage:read+clean") +func TestParseAllowedServicesParsesServiceActions(t *testing.T) { + grants, err := parseAllowedServices("systemd-journald.service:read+clean") require.NoError(t, err) require.Len(t, grants, 1) - assert.Equal(t, interp.SystemdResourceJournalStorage, grants[0].Resource) + assert.Equal(t, "systemd-journald.service", grants[0].Service) assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemdActionClean}, grants[0].Actions) } @@ -354,10 +352,10 @@ func TestParseAllowedServicesRejectsColonInServiceSelector(t *testing.T) { assert.Contains(t, err.Error(), `invalid service selector "tenant:mysql.service"`) } -func TestAllowedServicesFlagAcceptsServicesAndFixedResources(t *testing.T) { +func TestAllowedServicesFlagAcceptsServiceGrants(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "mysql.service:read+restart,journal:kernel:read,journal:storage:read+clean,manager:reload", + "--allowed-services", "mysql.service:read+restart,systemd-journald.service:read+clean", "--mode", "remediation", "-c", `echo hello`, ) @@ -366,15 +364,15 @@ func TestAllowedServicesFlagAcceptsServicesAndFixedResources(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedServicesFlagWarnsAndSkipsInvalidSystemdCombination(t *testing.T) { +func TestAllowedServicesFlagRejectsColonInSelector(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--allowed-services", "journal:storage:restart", + "--allowed-services", "tenant:mysql.service:read", "-c", `echo hello`, ) - assert.Equal(t, 0, code) - assert.Equal(t, "hello\n", stdout) - assert.Contains(t, stderr, `skipping unsupported action "restart"`) + assert.Equal(t, 1, code) + assert.Empty(t, stdout) + assert.Contains(t, stderr, `invalid service selector "tenant:mysql.service"`) } func TestAllowedServicesFlagRejectsColonInServiceSelector(t *testing.T) { @@ -420,31 +418,6 @@ func TestExplicitSystemdTargetRequiresMachineIDPath(t *testing.T) { assert.Contains(t, stderr, "machine ID path is required") } -func TestJournalVacuumPolicyFlags(t *testing.T) { - code, stdout, stderr := runCLI(t, - "--allow-all-commands", - "--journal-vacuum-min-age", "24h", - "--journal-vacuum-min-files", "2", - "--journal-vacuum-min-bytes", "1048576", - "--journal-vacuum-max-delete-files", "4", - "--journal-vacuum-max-delete-bytes", "8388608", - "-c", `echo hello`, - ) - assert.Equal(t, 0, code) - assert.Equal(t, "hello\n", stdout) - assert.Empty(t, stderr) -} - -func TestJournalVacuumPolicyFlagsRejectIncompletePolicy(t *testing.T) { - code, _, stderr := runCLI(t, - "--allow-all-commands", - "--journal-vacuum-min-age", "24h", - "-c", `echo hello`, - ) - assert.Equal(t, 1, code) - assert.Contains(t, stderr, "maximum deleted files") -} - func TestAllowAllCommandsFlag(t *testing.T) { code, stdout, _ := runCLI(t, "--allow-all-commands", "-c", `echo hello`) assert.Equal(t, 0, code) diff --git a/internal/systemd/client.go b/internal/systemd/client.go index ec5b773e..49d2d876 100644 --- a/internal/systemd/client.go +++ b/internal/systemd/client.go @@ -9,17 +9,11 @@ package systemd // Target paths are copied at construction so runner configuration remains // immutable while commands execute. type Client struct { - target Target - vacuumPolicy *JournalVacuumPolicy + target Target } // NewClient creates a client for one resolved systemd target. -func NewClient(target Target, vacuumPolicy *JournalVacuumPolicy) *Client { +func NewClient(target Target) *Client { target.JournalDirs = append([]string(nil), target.JournalDirs...) - client := &Client{target: target} - if vacuumPolicy != nil { - policyCopy := *vacuumPolicy - client.vacuumPolicy = &policyCopy - } - return client + return &Client{target: target} } diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go index 5d3f46a1..ecb09111 100644 --- a/internal/systemd/journal_files_test.go +++ b/internal/systemd/journal_files_test.go @@ -62,7 +62,7 @@ func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { client := NewClient(Target{ JournalDirs: []string{secondBase, firstBase, filepath.Join(root, "missing")}, MachineIDPath: machineIDPath, - }, nil) + }) gotMachineID, files, err := client.journalFiles() require.NoError(t, err) assert.Equal(t, machineID, gotMachineID) @@ -70,7 +70,7 @@ func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { } func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { - client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}, nil) + client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}) _, _, err := client.journalFiles() require.Error(t, err) assert.Contains(t, err.Error(), "journal directories are not configured") diff --git a/internal/systemd/journal_usage_unix_test.go b/internal/systemd/journal_usage_unix_test.go index 875517c6..ce71fc46 100644 --- a/internal/systemd/journal_usage_unix_test.go +++ b/internal/systemd/journal_usage_unix_test.go @@ -28,7 +28,7 @@ func TestJournalDiskUsageCountsAllocatedJournalFiles(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), make([]byte, 8192), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(machineDir, "ignored.txt"), make([]byte, 8192), 0o600)) - client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}, nil) + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) usage, err := client.JournalDiskUsage(context.Background()) require.NoError(t, err) assert.Equal(t, 1, usage.Files) @@ -41,7 +41,7 @@ func TestJournalDiskUsageReturnsZeroForEmptyTarget(t *testing.T) { machineIDPath := filepath.Join(root, "machine-id") require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) - client := NewClient(Target{JournalDirs: []string{filepath.Join(root, "missing")}, MachineIDPath: machineIDPath}, nil) + client := NewClient(Target{JournalDirs: []string{filepath.Join(root, "missing")}, MachineIDPath: machineIDPath}) usage, err := client.JournalDiskUsage(context.Background()) require.NoError(t, err) assert.Zero(t, usage.Files) @@ -61,7 +61,7 @@ func TestJournalDiskUsageHonorsCancellation(t *testing.T) { require.NoError(t, os.MkdirAll(machineDir, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(machineDir, "system.journal"), []byte("data"), 0o600)) - client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}, nil) + client := NewClient(Target{JournalDirs: []string{journalDir}, MachineIDPath: machineIDPath}) _, err := client.JournalDiskUsage(ctx) require.ErrorIs(t, err, context.Canceled) } diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go index 56fb5ac6..084e5e86 100644 --- a/internal/systemd/journal_vacuum_unix.go +++ b/internal/systemd/journal_vacuum_unix.go @@ -34,15 +34,9 @@ type vacuumCandidate struct { stat journalFileStat } -// VacuumJournal deletes only strictly recognized archived journals while -// enforcing the trusted policy on every invocation. +// VacuumJournal deletes only strictly recognized archived journals from the +// configured target while honoring the requested size and time thresholds. func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacuumRequest) (builtins.JournalVacuumResult, error) { - if c.vacuumPolicy == nil { - return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum policy is not configured") - } - if err := c.vacuumPolicy.Validate(); err != nil { - return builtins.JournalVacuumResult{}, fmt.Errorf("invalid journal vacuum policy: %w", err) - } if request.Now.IsZero() { return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum reference time is required") } @@ -63,8 +57,6 @@ func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacu if err != nil { return builtins.JournalVacuumResult{}, err } - policyCutoff := request.Now.Add(-c.vacuumPolicy.MinRetentionAge) - remainingFiles := len(candidates) remainingBytes := archivedBytes result := builtins.JournalVacuumResult{} @@ -72,20 +64,11 @@ func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacu if err := ctx.Err(); err != nil { return result, vacuumPartialError(result, err) } - if candidate.modTime.After(policyCutoff) { - break - } expired := !request.Before.IsZero() && !candidate.modTime.After(request.Before) overSize := request.MaxBytes > 0 && remainingBytes > request.MaxBytes if !expired && !overSize { break } - if remainingFiles-1 < c.vacuumPolicy.MinRetainedFiles || remainingBytes-candidate.stat.allocated < c.vacuumPolicy.MinRetainedBytes { - break - } - if result.Files >= c.vacuumPolicy.MaxDeletedFiles || candidate.stat.allocated > c.vacuumPolicy.MaxDeletedBytes-result.Bytes { - break - } if err := revalidateVacuumCandidate(candidate); err != nil { return result, vacuumPartialError(result, err) } @@ -96,7 +79,6 @@ func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacu } result.Files++ result.Bytes += candidate.stat.allocated - remainingFiles-- remainingBytes -= candidate.stat.allocated } return result, nil diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go index 7cc23713..7520d418 100644 --- a/internal/systemd/journal_vacuum_unix_test.go +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -27,7 +27,7 @@ func archivedJournalName(index int) string { return fmt.Sprintf("system@abcdef0123456789abcdef0123456789-%016x-%016x.journal", index, index) } -func newVacuumTestClient(t *testing.T, policy *JournalVacuumPolicy) (*Client, string) { +func newVacuumTestClient(t *testing.T) (*Client, string) { t.Helper() root := t.TempDir() machineIDPath := filepath.Join(root, "machine-id") @@ -35,7 +35,7 @@ func newVacuumTestClient(t *testing.T, policy *JournalVacuumPolicy) (*Client, st machineDir := filepath.Join(journalRoot, testMachineID) require.NoError(t, os.WriteFile(machineIDPath, []byte(testMachineID+"\n"), 0o600)) require.NoError(t, os.MkdirAll(machineDir, 0o700)) - return NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}, policy), machineDir + return NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}), machineDir } func writeVacuumFile(t *testing.T, directory, name string, modTime time.Time) string { @@ -46,18 +46,9 @@ func writeVacuumFile(t *testing.T, directory, name string, modTime time.Time) st return path } -func testVacuumPolicy() *JournalVacuumPolicy { - return &JournalVacuumPolicy{ - MinRetentionAge: 24 * time.Hour, - MinRetainedFiles: 1, - MaxDeletedFiles: 10, - MaxDeletedBytes: 1 << 30, - } -} - -func TestVacuumJournalDeletesOldestArchivesWithinPolicy(t *testing.T) { +func TestVacuumJournalDeletesOldestArchivesWithinRequest(t *testing.T) { now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC) - client, directory := newVacuumTestClient(t, testVacuumPolicy()) + client, directory := newVacuumTestClient(t) active := writeVacuumFile(t, directory, "system.journal", now.Add(-30*24*time.Hour)) namespaceActive := writeVacuumFile(t, directory, "system@tenant.journal", now.Add(-30*24*time.Hour)) malformed := writeVacuumFile(t, directory, "system@old.journal", now.Add(-30*24*time.Hour)) @@ -82,9 +73,9 @@ func TestVacuumJournalDeletesOldestArchivesWithinPolicy(t *testing.T) { func TestVacuumJournalDryRunDoesNotDelete(t *testing.T) { now := time.Now() - client, directory := newVacuumTestClient(t, testVacuumPolicy()) + client, directory := newVacuumTestClient(t) archive := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-7*24*time.Hour)) - writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-6*24*time.Hour)) + second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-6*24*time.Hour)) result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ Now: now, @@ -92,52 +83,14 @@ func TestVacuumJournalDryRunDoesNotDelete(t *testing.T) { DryRun: true, }) require.NoError(t, err) - assert.Equal(t, 1, result.Files) + assert.Equal(t, 2, result.Files) assert.FileExists(t, archive) -} - -func TestVacuumJournalEnforcesPerCallFileCeiling(t *testing.T) { - now := time.Now() - policy := testVacuumPolicy() - policy.MinRetainedFiles = 0 - policy.MaxDeletedFiles = 1 - client, directory := newVacuumTestClient(t, policy) - oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) - second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-9*24*time.Hour)) - - result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ - Now: now, - Before: now.Add(-48 * time.Hour), - }) - require.NoError(t, err) - assert.Equal(t, 1, result.Files) - assert.NoFileExists(t, oldest) assert.FileExists(t, second) } -func TestVacuumJournalDoesNotUnderflowByteCeiling(t *testing.T) { - now := time.Now() - policy := testVacuumPolicy() - policy.MinRetainedFiles = 0 - policy.MaxDeletedBytes = 1 - client, directory := newVacuumTestClient(t, policy) - archive := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) - - result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ - Now: now, - Before: now.Add(-48 * time.Hour), - }) - require.NoError(t, err) - assert.Zero(t, result.Files) - assert.Zero(t, result.Bytes) - assert.FileExists(t, archive) -} - func TestVacuumJournalHonorsAllocatedSizeTarget(t *testing.T) { now := time.Now() - policy := testVacuumPolicy() - policy.MinRetainedFiles = 0 - client, directory := newVacuumTestClient(t, policy) + client, directory := newVacuumTestClient(t) oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-9*24*time.Hour)) info, err := os.Lstat(oldest) @@ -157,9 +110,7 @@ func TestVacuumJournalHonorsAllocatedSizeTarget(t *testing.T) { func TestVacuumJournalSkipsHardlinksAndSymlinks(t *testing.T) { now := time.Now() - policy := testVacuumPolicy() - policy.MinRetainedFiles = 0 - client, directory := newVacuumTestClient(t, policy) + client, directory := newVacuumTestClient(t) first := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) second := filepath.Join(directory, archivedJournalName(2)) require.NoError(t, os.Link(first, second)) @@ -189,7 +140,7 @@ func TestVacuumJournalRejectsSymlinkedMachineDirectory(t *testing.T) { require.NoError(t, os.Symlink(outside, filepath.Join(journalRoot, testMachineID))) machineIDPath := filepath.Join(root, "machine-id") require.NoError(t, os.WriteFile(machineIDPath, []byte(testMachineID+"\n"), 0o600)) - client := NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}, testVacuumPolicy()) + client := NewClient(Target{JournalDirs: []string{journalRoot}, MachineIDPath: machineIDPath}) _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ Now: now, @@ -200,17 +151,20 @@ func TestVacuumJournalRejectsSymlinkedMachineDirectory(t *testing.T) { assert.FileExists(t, archive) } -func TestVacuumJournalRequiresPolicyAndBounds(t *testing.T) { +func TestVacuumJournalRequiresRequestBounds(t *testing.T) { now := time.Now() - client, _ := newVacuumTestClient(t, nil) - _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, Before: now.Add(-time.Hour)}) + client, _ := newVacuumTestClient(t) + _, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Before: now.Add(-time.Hour)}) require.Error(t, err) - assert.Contains(t, err.Error(), "policy is not configured") + assert.Contains(t, err.Error(), "reference time") - client, _ = newVacuumTestClient(t, testVacuumPolicy()) _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now}) require.Error(t, err) assert.Contains(t, err.Error(), "requires a size or time limit") + + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, Before: now.Add(time.Hour)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be in the future") } func TestIsArchivedJournalName(t *testing.T) { diff --git a/internal/systemd/vacuum_policy.go b/internal/systemd/vacuum_policy.go deleted file mode 100644 index a80d784c..00000000 --- a/internal/systemd/vacuum_policy.go +++ /dev/null @@ -1,40 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -package systemd - -import ( - "fmt" - "time" -) - -// JournalVacuumPolicy is a trusted operator ceiling for journal cleanup. -// Script-provided vacuum thresholds can only narrow this policy. -type JournalVacuumPolicy struct { - MinRetentionAge time.Duration - MinRetainedFiles int - MinRetainedBytes uint64 - MaxDeletedFiles int - MaxDeletedBytes uint64 -} - -// Validate rejects policies that would leave repeated cleanup invocations -// unbounded. A positive age floor is mandatory even when file/byte floors are -// also configured. -func (policy JournalVacuumPolicy) Validate() error { - if policy.MinRetentionAge <= 0 { - return fmt.Errorf("minimum retention age must be greater than zero") - } - if policy.MinRetainedFiles < 0 || policy.MinRetainedFiles > maxJournalFiles { - return fmt.Errorf("minimum retained files must be between 0 and %d", maxJournalFiles) - } - if policy.MaxDeletedFiles <= 0 || policy.MaxDeletedFiles > maxJournalFiles { - return fmt.Errorf("maximum deleted files must be between 1 and %d", maxJournalFiles) - } - if policy.MaxDeletedBytes == 0 { - return fmt.Errorf("maximum deleted bytes must be greater than zero") - } - return nil -} diff --git a/interp/api.go b/interp/api.go index 2cc330ac..e94a72e9 100644 --- a/interp/api.go +++ b/interp/api.go @@ -81,8 +81,8 @@ type runnerConfig struct { // command. Intended for testing convenience. allowAllCommands bool - // allowedSystemServices maps exact services and fixed systemd resources to - // their permitted actions. It is independent of allowAllCommands and + // allowedSystemServices maps exact services to their permitted actions. It + // is independent of allowAllCommands and // defaults to denying every systemd operation. allowedSystemServices systemdGrants @@ -92,7 +92,6 @@ type runnerConfig struct { systemdTarget internalsystemd.Target systemdTargetConfigured bool systemd *builtins.SystemdServices - journalVacuumPolicy *internalsystemd.JournalVacuumPolicy // maxExecutionTime bounds the duration of each Run call. Zero disables // the limit. When non-zero, Run derives a child context with this timeout. @@ -333,7 +332,7 @@ func New(opts ...RunnerOption) (*Runner, error) { if !r.systemdTargetConfigured { r.systemdTarget = internalsystemd.LocalTarget() } - systemdClient := internalsystemd.NewClient(r.systemdTarget, r.journalVacuumPolicy) + systemdClient := internalsystemd.NewClient(r.systemdTarget) r.systemd = &builtins.SystemdServices{ Journal: systemdClient, JournalStorage: systemdClient, diff --git a/interp/journal_vacuum.go b/interp/journal_vacuum.go deleted file mode 100644 index b4b190b5..00000000 --- a/interp/journal_vacuum.go +++ /dev/null @@ -1,29 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -package interp - -import ( - "fmt" - - internalsystemd "github.com/DataDog/rshell/internal/systemd" -) - -// JournalVacuumPolicy is the trusted retention floor and per-invocation -// deletion ceiling used by journal cleanup operations. -type JournalVacuumPolicy = internalsystemd.JournalVacuumPolicy - -// WithJournalVacuumPolicy enables bounded journal cleanup. Without this -// option, cleanup remains disabled even when journal:storage/clean is granted. -func WithJournalVacuumPolicy(policy JournalVacuumPolicy) RunnerOption { - return func(r *Runner) error { - if err := policy.Validate(); err != nil { - return fmt.Errorf("WithJournalVacuumPolicy: %w", err) - } - policyCopy := policy - r.journalVacuumPolicy = &policyCopy - return nil - } -} diff --git a/interp/journal_vacuum_integration_test.go b/interp/journal_vacuum_integration_test.go index 3e9331c0..55a432d3 100644 --- a/interp/journal_vacuum_integration_test.go +++ b/interp/journal_vacuum_integration_test.go @@ -36,16 +36,11 @@ func TestJournalctlVacuumEndToEnd(t *testing.T) { StdIO(nil, &stdout, &stderr), AllowedCommands([]string{"rshell:journalctl"}), AllowedSystemServices([]SystemdControlGrant{{ - Resource: SystemdResourceJournalStorage, - Actions: []SystemdAction{SystemdActionClean}, + Service: "systemd-journald.service", + Actions: []SystemdAction{SystemdActionClean}, }}), WithMode(ModeRemediation), WithSystemdTarget(SystemdTargetConfig{Root: root}), - WithJournalVacuumPolicy(JournalVacuumPolicy{ - MinRetentionAge: 24 * time.Hour, - MaxDeletedFiles: 1, - MaxDeletedBytes: 1 << 20, - }), ) require.NoError(t, err) defer runner.Close() diff --git a/interp/journal_vacuum_test.go b/interp/journal_vacuum_test.go deleted file mode 100644 index 2a8b6a13..00000000 --- a/interp/journal_vacuum_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -package interp - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWithJournalVacuumPolicyCopiesValidatedPolicy(t *testing.T) { - policy := JournalVacuumPolicy{ - MinRetentionAge: 24 * time.Hour, - MinRetainedFiles: 2, - MinRetainedBytes: 64 * 1024 * 1024, - MaxDeletedFiles: 4, - MaxDeletedBytes: 128 * 1024 * 1024, - } - runner, err := New(WithJournalVacuumPolicy(policy)) - require.NoError(t, err) - defer runner.Close() - - policy.MinRetainedFiles = 0 - require.NotNil(t, runner.journalVacuumPolicy) - assert.Equal(t, 2, runner.journalVacuumPolicy.MinRetainedFiles) -} - -func TestWithJournalVacuumPolicyIsDisabledByDefault(t *testing.T) { - runner, err := New() - require.NoError(t, err) - defer runner.Close() - assert.Nil(t, runner.journalVacuumPolicy) -} - -func TestWithJournalVacuumPolicyRejectsUnboundedPolicies(t *testing.T) { - valid := JournalVacuumPolicy{ - MinRetentionAge: time.Hour, - MaxDeletedFiles: 1, - MaxDeletedBytes: 1, - } - tests := []struct { - name string - mutate func(*JournalVacuumPolicy) - needle string - }{ - {name: "no retention age", mutate: func(p *JournalVacuumPolicy) { p.MinRetentionAge = 0 }, needle: "retention age"}, - {name: "negative retained files", mutate: func(p *JournalVacuumPolicy) { p.MinRetainedFiles = -1 }, needle: "retained files"}, - {name: "no file ceiling", mutate: func(p *JournalVacuumPolicy) { p.MaxDeletedFiles = 0 }, needle: "deleted files"}, - {name: "no byte ceiling", mutate: func(p *JournalVacuumPolicy) { p.MaxDeletedBytes = 0 }, needle: "deleted bytes"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - policy := valid - test.mutate(&policy) - runner, err := New(WithJournalVacuumPolicy(policy)) - if runner != nil { - runner.Close() - } - require.Error(t, err) - assert.Contains(t, err.Error(), test.needle) - }) - } -} diff --git a/interp/system_services.go b/interp/system_services.go index d8e56692..88c70b99 100644 --- a/interp/system_services.go +++ b/interp/system_services.go @@ -14,7 +14,7 @@ import ( ) // SystemdAction identifies an operation that may be granted for a systemd -// service or fixed capability. +// service. type SystemdAction = builtins.SystemdAction const ( @@ -24,19 +24,7 @@ const ( SystemdActionRestart = builtins.SystemdActionRestart ) -// SystemdResource identifies a fixed non-service capability in the shared -// systemd policy. -type SystemdResource = builtins.SystemdResource - -const ( - SystemdResourceJournalAll = builtins.SystemdResourceJournalAll - SystemdResourceJournalKernel = builtins.SystemdResourceJournalKernel - SystemdResourceJournalStorage = builtins.SystemdResourceJournalStorage - SystemdResourceManager = builtins.SystemdResourceManager -) - -// SystemdOperation is one service or fixed-resource action checked by the -// shared policy. +// SystemdOperation is one service action checked by the shared policy. type SystemdOperation = builtins.SystemdOperation // Compatibility aliases for the original service-only policy. @@ -44,37 +32,30 @@ type SystemServiceAction = builtins.SystemServiceAction const ( SystemServiceRead = builtins.SystemServiceRead + SystemServiceClean = builtins.SystemServiceClean SystemServiceReload = builtins.SystemServiceReload SystemServiceRestart = builtins.SystemServiceRestart ) -// SystemServiceControlGrant grants Actions for one exact Service or one fixed -// non-service Resource. Callers must set exactly one of Service or Resource. +// SystemServiceControlGrant grants Actions for one exact Service. type SystemServiceControlGrant struct { - Service string - Actions []SystemServiceAction - Resource SystemdResource + Service string + Actions []SystemServiceAction } // SystemdControlGrant is an alias for the shared systemd policy grant type. type SystemdControlGrant = SystemServiceControlGrant -type systemdGrantTarget struct { - service string - resource SystemdResource -} - -type systemdGrants map[systemdGrantTarget]map[SystemdAction]struct{} +type systemdGrants map[string]map[SystemdAction]struct{} -// AllowedSystemServices configures the services, fixed resources, and actions -// that systemd-aware builtins may use. Service names are matched exactly: for -// example, "mysql" and "mysql.service" are different services. +// AllowedSystemServices configures the services and actions that systemd-aware +// builtins may use. Service names are matched exactly: for example, "mysql" +// and "mysql.service" are different services. // -// Grants without actions are ignored. Invalid targets and unsupported -// target/action pairs are skipped with a warning. Supported fixed resources -// are journal:all, journal:kernel, journal:storage, and manager. Supported -// actions are read, clean, reload, and restart. Duplicate targets and actions -// are accepted and combined idempotently. +// Grants without actions are ignored. Invalid services and unsupported actions +// are skipped with a warning. Supported actions are read, clean, reload, and +// restart. Duplicate services and actions are accepted and combined +// idempotently. // // When not set (default), or when passed an empty slice, every systemd // operation is denied. This policy is not bypassed by allowing all commands. @@ -85,23 +66,22 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { if len(grant.Actions) == 0 { continue } - target, err := makeSystemdGrantTarget(grant.Service, grant.Resource) - if err != nil { + if err := validateSystemServiceName(grant.Service); err != nil { warning := fmt.Sprintf("AllowedSystemServices: skipping grant %d: %v\n", i, err) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } - actions := allowed[target] + actions := allowed[grant.Service] for _, action := range grant.Actions { - if !validSystemdOperation(target, action) { - warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, target.selector()) + if !validSystemdAction(action) { + warning := fmt.Sprintf("AllowedSystemServices: skipping unsupported action %q in grant %d for %q\n", action, i, grant.Service) r.sandboxWarnings = append(r.sandboxWarnings, warning...) continue } if actions == nil { actions = make(map[SystemdAction]struct{}, len(grant.Actions)) - allowed[target] = actions + allowed[grant.Service] = actions } actions[action] = struct{}{} } @@ -111,62 +91,11 @@ func AllowedSystemServices(grants []SystemServiceControlGrant) RunnerOption { } } -func makeSystemdGrantTarget(service string, resource SystemdResource) (systemdGrantTarget, error) { - switch { - case resource != "" && service != "": - return systemdGrantTarget{}, fmt.Errorf("must not set both Resource and Service") - case service != "": - if err := validateSystemServiceName(service); err != nil { - return systemdGrantTarget{}, err - } - return systemdGrantTarget{service: service}, nil - case resource != "": - if err := validateSystemdResource(resource); err != nil { - return systemdGrantTarget{}, err - } - return systemdGrantTarget{resource: resource}, nil - default: - return systemdGrantTarget{}, validateSystemServiceName("") - } -} - -func (target systemdGrantTarget) selector() string { - if target.service != "" { - return target.service - } - return string(target.resource) -} - -func (target systemdGrantTarget) description() string { - if target.service != "" { - return fmt.Sprintf("system service %q", target.service) - } - return fmt.Sprintf("systemd resource %q", target.resource) -} - -func validSystemdOperation(target systemdGrantTarget, action SystemdAction) bool { - switch { - case target.service != "": - return action == SystemdActionRead || action == SystemdActionReload || action == SystemdActionRestart - case target.resource == SystemdResourceJournalAll, - target.resource == SystemdResourceJournalKernel: - return action == SystemdActionRead - case target.resource == SystemdResourceJournalStorage: - return action == SystemdActionRead || action == SystemdActionClean - case target.resource == SystemdResourceManager: - return action == SystemdActionRead || action == SystemdActionReload - default: - return false - } -} - -func validateSystemdResource(resource SystemdResource) error { - switch resource { - case SystemdResourceJournalAll, SystemdResourceJournalKernel, SystemdResourceJournalStorage, SystemdResourceManager: - return nil - default: - return fmt.Errorf("unsupported systemd resource %q", resource) - } +func validSystemdAction(action SystemdAction) bool { + return action == SystemdActionRead || + action == SystemdActionClean || + action == SystemdActionReload || + action == SystemdActionRestart } func validateSystemServiceName(service string) error { @@ -197,19 +126,18 @@ func (r *Runner) authorizeSystemd(operations ...SystemdOperation) error { } for _, operation := range operations { - target, err := makeSystemdGrantTarget(operation.Service, operation.Resource) - if err != nil { + if err := validateSystemServiceName(operation.Service); err != nil { return err } - if !validSystemdOperation(target, operation.Action) { - return fmt.Errorf("unsupported systemd action %q for %s", operation.Action, target.description()) + if !validSystemdAction(operation.Action) { + return fmt.Errorf("unsupported systemd action %q for system service %q", operation.Action, operation.Service) } if operation.Action != SystemdActionRead && !r.remediationMode { return fmt.Errorf("systemd action %q requires remediation mode", operation.Action) } - actions := r.allowedSystemServices[target] + actions := r.allowedSystemServices[operation.Service] if _, ok := actions[operation.Action]; !ok { - return fmt.Errorf("%s is not allowed for action %q", target.description(), operation.Action) + return fmt.Errorf("system service %q is not allowed for action %q", operation.Service, operation.Action) } } return nil diff --git a/interp/system_services_test.go b/interp/system_services_test.go index 821d7d2b..1cdea6b0 100644 --- a/interp/system_services_test.go +++ b/interp/system_services_test.go @@ -24,6 +24,7 @@ func TestAllowedSystemServicesAuthorizesExactServiceAndAction(t *testing.T) { SystemServiceRestart, SystemServiceReload, SystemServiceRead, + SystemServiceClean, }, }, { @@ -37,6 +38,7 @@ func TestAllowedSystemServicesAuthorizesExactServiceAndAction(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRestart, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceReload, "mysql.service")) + require.NoError(t, runner.authorizeSystemServices(SystemServiceClean, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service", "nginx.service")) err = runner.authorizeSystemServices(SystemServiceRestart, "nginx.service") @@ -64,7 +66,7 @@ func TestAllowedSystemServicesAllowsReadOutsideRemediationMode(t *testing.T) { runner, err := New(AllowedSystemServices([]SystemServiceControlGrant{ { Service: "mysql.service", - Actions: []SystemServiceAction{SystemServiceRead, SystemServiceRestart}, + Actions: []SystemServiceAction{SystemServiceRead, SystemServiceRestart, SystemServiceClean}, }, })) require.NoError(t, err) @@ -74,47 +76,24 @@ func TestAllowedSystemServicesAllowsReadOutsideRemediationMode(t *testing.T) { err = runner.authorizeSystemServices(SystemServiceRestart, "mysql.service") require.Error(t, err) assert.Contains(t, err.Error(), `action "restart" requires remediation mode`) -} -func TestAllowedSystemServicesAuthorizesJournalAndManagerResources(t *testing.T) { - runner, err := New( - WithMode(ModeRemediation), - AllowedSystemServices([]SystemdControlGrant{ - {Service: "mysql.service", Actions: []SystemdAction{SystemdActionRead}}, - {Service: "mysql.service", Actions: []SystemdAction{SystemdActionRestart}}, - {Resource: SystemdResourceJournalKernel, Actions: []SystemdAction{SystemdActionRead}}, - {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead, SystemdActionClean}}, - {Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionReload}}, - }), - ) - require.NoError(t, err) - defer runner.Close() - - require.NoError(t, runner.authorizeSystemd( - SystemdOperation{Service: "mysql.service", Action: SystemdActionRead}, - SystemdOperation{Service: "mysql.service", Action: SystemdActionRestart}, - SystemdOperation{Resource: SystemdResourceJournalKernel, Action: SystemdActionRead}, - SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionClean}, - SystemdOperation{Resource: SystemdResourceManager, Action: SystemdActionReload}, - )) - - err = runner.authorizeSystemd(SystemdOperation{Resource: SystemdResourceJournalAll, Action: SystemdActionRead}) + err = runner.authorizeSystemServices(SystemServiceClean, "mysql.service") require.Error(t, err) - assert.Contains(t, err.Error(), `resource "journal:all" is not allowed`) + assert.Contains(t, err.Error(), `action "clean" requires remediation mode`) } func TestAllowedSystemServicesReadDoesNotEnableMutation(t *testing.T) { runner, err := New(AllowedSystemServices([]SystemdControlGrant{ - {Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRead}}, + {Service: "systemd-journald.service", Actions: []SystemdAction{SystemdActionRead}}, })) require.NoError(t, err) defer runner.Close() require.NoError(t, runner.authorizeSystemd( - SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionRead}, + SystemdOperation{Service: "systemd-journald.service", Action: SystemdActionRead}, )) err = runner.authorizeSystemd( - SystemdOperation{Resource: SystemdResourceJournalStorage, Action: SystemdActionClean}, + SystemdOperation{Service: "systemd-journald.service", Action: SystemdActionClean}, ) require.Error(t, err) assert.Contains(t, err.Error(), `action "clean" requires remediation mode`) @@ -164,7 +143,7 @@ func TestAllowedSystemServicesSkipsEmptyAndInvalidGrants(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) assert.Len(t, runner.allowedSystemServices, 1) - assert.NotContains(t, runner.allowedSystemServices, systemdGrantTarget{service: "ignored.service"}) + assert.NotContains(t, runner.allowedSystemServices, "ignored.service") warnings := runner.Warnings() require.Len(t, warnings, 9) @@ -198,7 +177,7 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { require.NoError(t, runner.authorizeSystemServices(SystemServiceRead, "mysql.service")) require.NoError(t, runner.authorizeSystemServices(SystemServiceReload, "mysql.service")) - assert.NotContains(t, runner.allowedSystemServices, systemdGrantTarget{service: "ignored.service"}) + assert.NotContains(t, runner.allowedSystemServices, "ignored.service") warnings := runner.Warnings() require.Len(t, warnings, 2) @@ -206,60 +185,6 @@ func TestAllowedSystemServicesSkipsUnsupportedActions(t *testing.T) { assert.Contains(t, warningOutput.String(), `AllowedSystemServices: skipping unsupported action "enable" in grant 1 for "ignored.service"`) } -func TestAllowedSystemServicesSkipsInvalidResourceActionCombinations(t *testing.T) { - tests := []struct { - name string - grant SystemdControlGrant - needle string - }{ - { - name: "unknown resource", - grant: SystemdControlGrant{Resource: "journal:namespace", Actions: []SystemdAction{SystemdActionRead}}, - needle: `unsupported systemd resource "journal:namespace"`, - }, - { - name: "clean service", - grant: SystemdControlGrant{Service: "mysql.service", Actions: []SystemdAction{SystemdActionClean}}, - needle: `skipping unsupported action "clean"`, - }, - { - name: "restart journal", - grant: SystemdControlGrant{Resource: SystemdResourceJournalStorage, Actions: []SystemdAction{SystemdActionRestart}}, - needle: `skipping unsupported action "restart"`, - }, - { - name: "clean manager", - grant: SystemdControlGrant{Resource: SystemdResourceManager, Actions: []SystemdAction{SystemdActionClean}}, - needle: `skipping unsupported action "clean"`, - }, - { - name: "resource and service", - grant: SystemdControlGrant{ - Service: "mysql.service", - Resource: SystemdResourceManager, - Actions: []SystemdAction{SystemdActionRead}, - }, - needle: "must not set both Resource and Service", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var warningOutput bytes.Buffer - runner, err := New( - WarningsWriter(&warningOutput), - AllowedSystemServices([]SystemdControlGrant{test.grant}), - ) - require.NoError(t, err) - defer runner.Close() - - assert.Empty(t, runner.allowedSystemServices) - require.Len(t, runner.Warnings(), 1) - assert.Contains(t, warningOutput.String(), test.needle) - }) - } -} - func TestAuthorizeSystemServicesRejectsInvalidRequests(t *testing.T) { runner, err := New(WithMode(ModeRemediation), AllowedSystemServices([]SystemServiceControlGrant{ { diff --git a/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml b/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml index b6b8703a..bf56373e 100644 --- a/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml +++ b/tests/scenarios/cmd/journalctl/errors/denied_storage.yaml @@ -1,11 +1,11 @@ # Scenario runners do not grant systemd capabilities by default. skip_assert_against_bash: true -description: journalctl disk usage enforces the shared storage allowlist +description: journalctl disk usage enforces the journald service allowlist input: script: |+ journalctl --disk-usage expect: stdout: |+ stderr: |+ - journalctl: systemd resource "journal:storage" is not allowed for action "read" + journalctl: system service "systemd-journald.service" is not allowed for action "read" exit_code: 1 diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml index 9f5d2485..fc54ee24 100644 --- a/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml @@ -1,6 +1,6 @@ # Remediation mode alone does not bypass the shared systemd allowlist. skip_assert_against_bash: true -description: journalctl vacuum requires a storage clean grant +description: journalctl vacuum requires a journald clean grant input: mode: remediation script: |+ @@ -8,5 +8,5 @@ input: expect: stdout: |+ stderr: |+ - journalctl: systemd resource "journal:storage" is not allowed for action "clean" + journalctl: system service "systemd-journald.service" is not allowed for action "clean" exit_code: 1 From 71d7627d4a37df4ea61495a015c58d5cd95f1d2e Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 13:29:38 -0400 Subject: [PATCH 20/53] feat(systemd): add bounded journal format parser --- internal/systemd/journal_format.go | 372 ++++++++++++++++++++++++ internal/systemd/journal_format_test.go | 256 ++++++++++++++++ 2 files changed, 628 insertions(+) create mode 100644 internal/systemd/journal_format.go create mode 100644 internal/systemd/journal_format_test.go diff --git a/internal/systemd/journal_format.go b/internal/systemd/journal_format.go new file mode 100644 index 00000000..dca5a886 --- /dev/null +++ b/internal/systemd/journal_format.go @@ -0,0 +1,372 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "math" +) + +const ( + journalHeaderMinSize = 208 + journalHeaderCurrentSize = 272 + journalObjectHeaderSize = 16 + + journalStateOffline = 0 + journalStateOnline = 1 + journalStateArchived = 2 + + journalObjectData = 1 + journalObjectField = 2 + journalObjectEntry = 3 + journalObjectDataHashTable = 4 + journalObjectFieldHashTable = 5 + journalObjectEntryArray = 6 + journalObjectTag = 7 + + journalObjectCompressedXZ = 1 << 0 + journalObjectCompressedLZ4 = 1 << 1 + journalObjectCompressedZSTD = 1 << 2 + journalObjectCompressionMask = journalObjectCompressedXZ | + journalObjectCompressedLZ4 | + journalObjectCompressedZSTD + + journalHeaderIncompatibleCompressedXZ = 1 << 0 + journalHeaderIncompatibleCompressedLZ4 = 1 << 1 + journalHeaderIncompatibleKeyedHash = 1 << 2 + journalHeaderIncompatibleCompressedZSTD = 1 << 3 + journalHeaderIncompatibleCompact = 1 << 4 + journalHeaderKnownIncompatible = journalHeaderIncompatibleCompressedXZ | + journalHeaderIncompatibleCompressedLZ4 | + journalHeaderIncompatibleKeyedHash | + journalHeaderIncompatibleCompressedZSTD | + journalHeaderIncompatibleCompact +) + +var ( + errJournalCorrupt = errors.New("corrupt systemd journal") + errJournalUnsupported = errors.New("unsupported systemd journal format") + journalSignature = [8]byte{'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H'} +) + +type journalID [16]byte + +func (id journalID) String() string { + return hex.EncodeToString(id[:]) +} + +type journalHeader struct { + compatibleFlags uint32 + incompatibleFlags uint32 + state uint8 + fileID journalID + machineID journalID + tailEntryBootID journalID + seqnumID journalID + headerSize uint64 + arenaSize uint64 + arenaEnd uint64 + + dataHashTableOffset uint64 + dataHashTableSize uint64 + fieldHashTableOffset uint64 + fieldHashTableSize uint64 + tailObjectOffset uint64 + nObjects uint64 + nEntries uint64 + tailEntrySeqnum uint64 + headEntrySeqnum uint64 + entryArrayOffset uint64 + headEntryRealtime uint64 + tailEntryRealtime uint64 + tailEntryMonotonic uint64 + + tailEntryArrayOffset uint32 + tailEntryArrayNEntries uint32 + hasTailEntryArray bool + tailEntryOffset uint64 + hasTailEntryOffset bool +} + +func (h journalHeader) compact() bool { + return h.incompatibleFlags&journalHeaderIncompatibleCompact != 0 +} + +func (h journalHeader) keyedHash() bool { + return h.incompatibleFlags&journalHeaderIncompatibleKeyedHash != 0 +} + +type journalObject struct { + offset uint64 + objectType uint8 + flags uint8 + size uint64 +} + +type journalFileView struct { + name string + reader io.ReaderAt + size uint64 + header journalHeader +} + +func newJournalFileView(name string, reader io.ReaderAt, size uint64) (*journalFileView, error) { + if reader == nil { + return nil, fmt.Errorf("journal reader is nil") + } + if size > math.MaxInt64 { + return nil, journalUnsupported(name, "file is larger than the supported reader offset range") + } + + header, err := readJournalHeader(name, reader, size) + if err != nil { + return nil, err + } + return &journalFileView{name: name, reader: reader, size: size, header: header}, nil +} + +func readJournalHeader(name string, reader io.ReaderAt, fileSize uint64) (journalHeader, error) { + if fileSize < journalHeaderMinSize { + return journalHeader{}, journalCorrupt(name, 0, "file is smaller than the minimum %d-byte header", journalHeaderMinSize) + } + + var raw [journalHeaderCurrentSize]byte + if err := readJournalAt(name, reader, fileSize, 0, raw[:journalHeaderMinSize]); err != nil { + return journalHeader{}, err + } + if string(raw[:len(journalSignature)]) != string(journalSignature[:]) { + return journalHeader{}, journalCorrupt(name, 0, "invalid journal signature") + } + + headerSize := binary.LittleEndian.Uint64(raw[88:96]) + if headerSize < journalHeaderMinSize { + return journalHeader{}, journalCorrupt(name, 88, "header size %d is smaller than %d", headerSize, journalHeaderMinSize) + } + if headerSize%8 != 0 { + return journalHeader{}, journalCorrupt(name, 88, "header size %d is not 8-byte aligned", headerSize) + } + if headerSize > fileSize { + return journalHeader{}, journalCorrupt(name, 88, "header size %d exceeds the %d-byte file", headerSize, fileSize) + } + + readSize := headerSize + if readSize > journalHeaderCurrentSize { + readSize = journalHeaderCurrentSize + } + if readSize > journalHeaderMinSize { + if err := readJournalAt(name, reader, fileSize, journalHeaderMinSize, raw[journalHeaderMinSize:readSize]); err != nil { + return journalHeader{}, err + } + } + + incompatibleFlags := binary.LittleEndian.Uint32(raw[12:16]) + if unknown := incompatibleFlags &^ uint32(journalHeaderKnownIncompatible); unknown != 0 { + return journalHeader{}, journalUnsupported(name, "unknown incompatible feature flags 0x%08x", unknown) + } + state := raw[16] + if state > journalStateArchived { + return journalHeader{}, journalCorrupt(name, 16, "invalid journal state %d", state) + } + + arenaSize := binary.LittleEndian.Uint64(raw[96:104]) + if arenaSize > math.MaxUint64-headerSize { + return journalHeader{}, journalCorrupt(name, 96, "header and arena sizes overflow") + } + arenaEnd := headerSize + arenaSize + if arenaEnd > fileSize { + return journalHeader{}, journalCorrupt(name, 96, "journal arena ends at %d beyond the %d-byte file", arenaEnd, fileSize) + } + + header := journalHeader{ + compatibleFlags: binary.LittleEndian.Uint32(raw[8:12]), + incompatibleFlags: incompatibleFlags, + state: state, + headerSize: headerSize, + arenaSize: arenaSize, + arenaEnd: arenaEnd, + dataHashTableOffset: binary.LittleEndian.Uint64(raw[104:112]), + dataHashTableSize: binary.LittleEndian.Uint64(raw[112:120]), + fieldHashTableOffset: binary.LittleEndian.Uint64(raw[120:128]), + fieldHashTableSize: binary.LittleEndian.Uint64(raw[128:136]), + tailObjectOffset: binary.LittleEndian.Uint64(raw[136:144]), + nObjects: binary.LittleEndian.Uint64(raw[144:152]), + nEntries: binary.LittleEndian.Uint64(raw[152:160]), + tailEntrySeqnum: binary.LittleEndian.Uint64(raw[160:168]), + headEntrySeqnum: binary.LittleEndian.Uint64(raw[168:176]), + entryArrayOffset: binary.LittleEndian.Uint64(raw[176:184]), + headEntryRealtime: binary.LittleEndian.Uint64(raw[184:192]), + tailEntryRealtime: binary.LittleEndian.Uint64(raw[192:200]), + tailEntryMonotonic: binary.LittleEndian.Uint64(raw[200:208]), + } + copy(header.fileID[:], raw[24:40]) + copy(header.machineID[:], raw[40:56]) + copy(header.tailEntryBootID[:], raw[56:72]) + copy(header.seqnumID[:], raw[72:88]) + + if headerSize >= 264 { + header.tailEntryArrayOffset = binary.LittleEndian.Uint32(raw[256:260]) + header.tailEntryArrayNEntries = binary.LittleEndian.Uint32(raw[260:264]) + header.hasTailEntryArray = true + } + if headerSize >= 272 { + header.tailEntryOffset = binary.LittleEndian.Uint64(raw[264:272]) + header.hasTailEntryOffset = true + } + if header.hasTailEntryArray { + if (header.tailEntryArrayOffset == 0) != (header.tailEntryArrayNEntries == 0) { + return journalHeader{}, journalCorrupt(name, uint64(header.tailEntryArrayOffset), "tail entry array offset and count are inconsistent") + } + if header.tailEntryArrayOffset != 0 { + if err := validateJournalObjectOffset(name, header, uint64(header.tailEntryArrayOffset), "tail entry array"); err != nil { + return journalHeader{}, err + } + if header.entryArrayOffset == 0 || header.entryArrayOffset > uint64(header.tailEntryArrayOffset) { + return journalHeader{}, journalCorrupt(name, uint64(header.tailEntryArrayOffset), "tail entry array precedes the global entry array chain") + } + } + } + + for _, reference := range []struct { + offset uint64 + label string + }{ + {offset: header.tailObjectOffset, label: "tail object"}, + {offset: header.entryArrayOffset, label: "entry array"}, + {offset: header.tailEntryOffset, label: "tail entry"}, + } { + if reference.offset == 0 { + continue + } + if err := validateJournalObjectOffset(name, header, reference.offset, reference.label); err != nil { + return journalHeader{}, err + } + } + if err := validateJournalHashTableRange(name, header, header.dataHashTableOffset, header.dataHashTableSize, "data"); err != nil { + return journalHeader{}, err + } + if err := validateJournalHashTableRange(name, header, header.fieldHashTableOffset, header.fieldHashTableSize, "field"); err != nil { + return journalHeader{}, err + } + + return header, nil +} + +func validateJournalObjectOffset(name string, header journalHeader, offset uint64, label string) error { + if offset%8 != 0 { + return journalCorrupt(name, offset, "%s offset is not 8-byte aligned", label) + } + if offset < header.headerSize { + return journalCorrupt(name, offset, "%s offset points into the journal header", label) + } + if offset > header.arenaEnd || header.arenaEnd-offset < journalObjectHeaderSize { + return journalCorrupt(name, offset, "%s offset does not contain a complete object header", label) + } + if header.tailObjectOffset != 0 && offset > header.tailObjectOffset { + return journalCorrupt(name, offset, "%s offset is after the tail object", label) + } + return nil +} + +func validateJournalHashTableRange(name string, header journalHeader, offset, size uint64, label string) error { + if (offset == 0) != (size == 0) { + return journalCorrupt(name, offset, "%s hash table offset and size are inconsistent", label) + } + if offset == 0 { + return nil + } + if offset < journalObjectHeaderSize { + return journalCorrupt(name, offset, "%s hash table payload offset is invalid", label) + } + objectOffset := offset - journalObjectHeaderSize + if err := validateJournalObjectOffset(name, header, objectOffset, label+" hash table"); err != nil { + return err + } + if size%16 != 0 { + return journalCorrupt(name, offset, "%s hash table size %d is not a multiple of 16", label, size) + } + if size > header.arenaEnd-offset { + return journalCorrupt(name, offset, "%s hash table extends beyond the journal arena", label) + } + return nil +} + +func (f *journalFileView) objectAt(offset uint64, expectedType uint8) (journalObject, error) { + if err := validateJournalObjectOffset(f.name, f.header, offset, "object"); err != nil { + return journalObject{}, err + } + + var raw [journalObjectHeaderSize]byte + if err := readJournalAt(f.name, f.reader, f.size, offset, raw[:]); err != nil { + return journalObject{}, err + } + object := journalObject{ + offset: offset, + objectType: raw[0], + flags: raw[1], + size: binary.LittleEndian.Uint64(raw[8:16]), + } + if expectedType != 0 && object.objectType != expectedType { + return journalObject{}, journalCorrupt(f.name, offset, "expected object type %d, found %d", expectedType, object.objectType) + } + if unknown := object.flags &^ uint8(journalObjectCompressionMask); unknown != 0 { + return journalObject{}, journalUnsupported(f.name, "object at offset %d uses unknown flags 0x%02x", offset, unknown) + } + if object.flags != 0 && object.objectType != journalObjectData { + return journalObject{}, journalCorrupt(f.name, offset, "compression flags are set on non-DATA object type %d", object.objectType) + } + if object.flags != 0 && object.flags&(object.flags-1) != 0 { + return journalObject{}, journalCorrupt(f.name, offset, "DATA object has multiple compression flags") + } + for _, compression := range []struct { + objectFlag uint8 + headerFlag uint32 + name string + }{ + {objectFlag: journalObjectCompressedXZ, headerFlag: journalHeaderIncompatibleCompressedXZ, name: "XZ"}, + {objectFlag: journalObjectCompressedLZ4, headerFlag: journalHeaderIncompatibleCompressedLZ4, name: "LZ4"}, + {objectFlag: journalObjectCompressedZSTD, headerFlag: journalHeaderIncompatibleCompressedZSTD, name: "ZSTD"}, + } { + if object.flags&compression.objectFlag != 0 && f.header.incompatibleFlags&compression.headerFlag == 0 { + return journalObject{}, journalCorrupt(f.name, offset, "%s-compressed DATA object is not declared in the journal header", compression.name) + } + } + if object.size < journalObjectHeaderSize { + return journalObject{}, journalCorrupt(f.name, offset, "object size %d is smaller than its header", object.size) + } + if object.size > f.header.arenaEnd-offset { + return journalObject{}, journalCorrupt(f.name, offset, "object size %d extends beyond the journal arena", object.size) + } + return object, nil +} + +func readJournalAt(name string, reader io.ReaderAt, fileSize, offset uint64, destination []byte) error { + if offset > fileSize || uint64(len(destination)) > fileSize-offset { + return journalCorrupt(name, offset, "read of %d bytes exceeds the %d-byte file", len(destination), fileSize) + } + if offset > math.MaxInt64 { + return journalUnsupported(name, "object offset %d exceeds the supported reader range", offset) + } + n, err := reader.ReadAt(destination, int64(offset)) + if n != len(destination) { + return journalCorrupt(name, offset, "short read: got %d of %d bytes", n, len(destination)) + } + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("read systemd journal %q at offset %d: %w", name, offset, err) + } + return nil +} + +func journalCorrupt(name string, offset uint64, format string, arguments ...any) error { + return fmt.Errorf("%w %q at offset %d: %s", errJournalCorrupt, name, offset, fmt.Sprintf(format, arguments...)) +} + +func journalUnsupported(name, format string, arguments ...any) error { + return fmt.Errorf("%w %q: %s", errJournalUnsupported, name, fmt.Sprintf(format, arguments...)) +} diff --git a/internal/systemd/journal_format_test.go b/internal/systemd/journal_format_test.go new file mode 100644 index 00000000..748a8d0f --- /dev/null +++ b/internal/systemd/journal_format_test.go @@ -0,0 +1,256 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalFileViewParsesCurrentHeader(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize, 0, 0) + copy(contents[24:40], bytes.Repeat([]byte{0x11}, 16)) + copy(contents[40:56], bytes.Repeat([]byte{0x22}, 16)) + copy(contents[56:72], bytes.Repeat([]byte{0x33}, 16)) + copy(contents[72:88], bytes.Repeat([]byte{0x44}, 16)) + binary.LittleEndian.PutUint64(contents[152:160], 7) + binary.LittleEndian.PutUint64(contents[160:168], 11) + binary.LittleEndian.PutUint64(contents[168:176], 5) + binary.LittleEndian.PutUint64(contents[184:192], 1_700_000_000_000_000) + binary.LittleEndian.PutUint64(contents[192:200], 1_700_000_000_000_500) + + view, err := newJournalFileView("fixture.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + assert.Equal(t, journalStateArchived, int(view.header.state)) + assert.Equal(t, "11111111111111111111111111111111", view.header.fileID.String()) + assert.Equal(t, "22222222222222222222222222222222", view.header.machineID.String()) + assert.Equal(t, "33333333333333333333333333333333", view.header.tailEntryBootID.String()) + assert.Equal(t, "44444444444444444444444444444444", view.header.seqnumID.String()) + assert.Equal(t, uint64(7), view.header.nEntries) + assert.Equal(t, uint64(11), view.header.tailEntrySeqnum) + assert.Equal(t, uint64(5), view.header.headEntrySeqnum) + assert.Equal(t, uint64(1_700_000_000_000_000), view.header.headEntryRealtime) + assert.Equal(t, uint64(1_700_000_000_000_500), view.header.tailEntryRealtime) + assert.True(t, view.header.hasTailEntryArray) + assert.True(t, view.header.hasTailEntryOffset) + assert.False(t, view.header.compact()) + assert.False(t, view.header.keyedHash()) +} + +func TestJournalFileViewAcceptsMinimumAndFutureHeaders(t *testing.T) { + minimum := testJournalContents(journalHeaderMinSize, 0, 0) + minimumView, err := newJournalFileView("old.journal", bytes.NewReader(minimum), uint64(len(minimum))) + require.NoError(t, err) + assert.False(t, minimumView.header.hasTailEntryArray) + assert.False(t, minimumView.header.hasTailEntryOffset) + + future := testJournalContentsWithHeader(280, 280, 0, journalHeaderIncompatibleCompact|journalHeaderIncompatibleKeyedHash) + futureView, err := newJournalFileView("future.journal", bytes.NewReader(future), uint64(len(future))) + require.NoError(t, err) + assert.True(t, futureView.header.compact()) + assert.True(t, futureView.header.keyedHash()) +} + +func TestJournalFileViewRejectsInvalidHeaders(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + match string + target error + }{ + { + name: "signature", + mutate: func(contents []byte) { + contents[0] = 0 + }, + match: "invalid journal signature", + target: errJournalCorrupt, + }, + { + name: "unknown incompatible flag", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint32(contents[12:16], 1<<31) + }, + match: "unknown incompatible feature flags 0x80000000", + target: errJournalUnsupported, + }, + { + name: "invalid state", + mutate: func(contents []byte) { + contents[16] = 3 + }, + match: "invalid journal state 3", + target: errJournalCorrupt, + }, + { + name: "short header size", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint64(contents[88:96], journalHeaderMinSize-8) + }, + match: "header size 200 is smaller than 208", + target: errJournalCorrupt, + }, + { + name: "unaligned header size", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint64(contents[88:96], journalHeaderCurrentSize+1) + }, + match: "header size 273 is not 8-byte aligned", + target: errJournalCorrupt, + }, + { + name: "arena beyond file", + mutate: func(contents []byte) { + binary.LittleEndian.PutUint64(contents[96:104], 8) + }, + match: "journal arena ends at 280 beyond the 272-byte file", + target: errJournalCorrupt, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize, 0, 0) + test.mutate(contents) + _, err := newJournalFileView("bad.journal", bytes.NewReader(contents), uint64(len(contents))) + require.Error(t, err) + assert.ErrorIs(t, err, test.target) + assert.Contains(t, err.Error(), test.match) + }) + } +} + +func TestJournalFileViewValidatesHeaderOffsets(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize+64, 0, 0) + binary.LittleEndian.PutUint64(contents[136:144], journalHeaderCurrentSize) + binary.LittleEndian.PutUint64(contents[176:184], journalHeaderCurrentSize+1) + + _, err := newJournalFileView("bad-offset.journal", bytes.NewReader(contents), uint64(len(contents))) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "entry array offset is not 8-byte aligned") +} + +func TestJournalFileViewReadsBoundedObjectHeader(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize+64, 0, 0) + binary.LittleEndian.PutUint64(contents[136:144], journalHeaderCurrentSize) + contents[journalHeaderCurrentSize] = journalObjectEntry + binary.LittleEndian.PutUint64(contents[journalHeaderCurrentSize+8:journalHeaderCurrentSize+16], 64) + + view, err := newJournalFileView("entry.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + object, err := view.objectAt(journalHeaderCurrentSize, journalObjectEntry) + require.NoError(t, err) + assert.Equal(t, uint8(journalObjectEntry), object.objectType) + assert.Equal(t, uint64(64), object.size) + + _, err = view.objectAt(journalHeaderCurrentSize, journalObjectData) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected object type 1, found 3") +} + +func TestJournalFileViewRejectsInvalidObjectHeaders(t *testing.T) { + tests := []struct { + name string + mutate func([]byte) + match string + target error + }{ + { + name: "unknown flags", + mutate: func(object []byte) { + object[1] = 1 << 7 + }, + match: "unknown flags 0x80", + target: errJournalUnsupported, + }, + { + name: "compressed non-data", + mutate: func(object []byte) { + object[1] = journalObjectCompressedZSTD + }, + match: "compression flags are set on non-DATA object", + target: errJournalCorrupt, + }, + { + name: "undeclared compression", + mutate: func(object []byte) { + object[0] = journalObjectData + object[1] = journalObjectCompressedZSTD + }, + match: "ZSTD-compressed DATA object is not declared", + target: errJournalCorrupt, + }, + { + name: "multiple compression algorithms", + mutate: func(object []byte) { + object[0] = journalObjectData + object[1] = journalObjectCompressedLZ4 | journalObjectCompressedZSTD + }, + match: "multiple compression flags", + target: errJournalCorrupt, + }, + { + name: "short object size", + mutate: func(object []byte) { + binary.LittleEndian.PutUint64(object[8:16], journalObjectHeaderSize-1) + }, + match: "object size 15 is smaller than its header", + target: errJournalCorrupt, + }, + { + name: "object beyond arena", + mutate: func(object []byte) { + binary.LittleEndian.PutUint64(object[8:16], 65) + }, + match: "object size 65 extends beyond the journal arena", + target: errJournalCorrupt, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := testJournalContents(journalHeaderCurrentSize+64, 0, 0) + binary.LittleEndian.PutUint64(contents[136:144], journalHeaderCurrentSize) + object := contents[journalHeaderCurrentSize:] + object[0] = journalObjectEntry + binary.LittleEndian.PutUint64(object[8:16], 64) + test.mutate(object) + + view, err := newJournalFileView("bad-object.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + _, err = view.objectAt(journalHeaderCurrentSize, 0) + require.Error(t, err) + assert.True(t, errors.Is(err, test.target)) + assert.Contains(t, err.Error(), test.match) + }) + } +} + +func testJournalContents(size int, compatibleFlags, incompatibleFlags uint32) []byte { + headerSize := size + if headerSize > journalHeaderCurrentSize { + headerSize = journalHeaderCurrentSize + } + return testJournalContentsWithHeader(size, headerSize, compatibleFlags, incompatibleFlags) +} + +func testJournalContentsWithHeader(size, headerSize int, compatibleFlags, incompatibleFlags uint32) []byte { + contents := make([]byte, size) + copy(contents[:8], journalSignature[:]) + binary.LittleEndian.PutUint32(contents[8:12], compatibleFlags) + binary.LittleEndian.PutUint32(contents[12:16], incompatibleFlags) + contents[16] = journalStateArchived + binary.LittleEndian.PutUint64(contents[88:96], uint64(headerSize)) + binary.LittleEndian.PutUint64(contents[96:104], uint64(size-headerSize)) + return contents +} From 09e77fa9ee42fcb6116f33fe16c06fc296112d47 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 13:37:00 -0400 Subject: [PATCH 21/53] feat(systemd): decode compressed journal data --- LICENSE-3rdparty.csv | 3 + go.mod | 3 + go.sum | 6 + internal/systemd/journal_data.go | 270 ++++++++++++++++++++++++++ internal/systemd/journal_data_test.go | 209 ++++++++++++++++++++ internal/systemd/journal_format.go | 1 + 6 files changed, 492 insertions(+) create mode 100644 internal/systemd/journal_data.go create mode 100644 internal/systemd/journal_data_test.go diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 571464e6..a0e5e04e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -10,7 +10,9 @@ github.com/ebitengine/purego,https://github.com/ebitengine/purego,Apache-2.0,Cop github.com/go-ole/go-ole,https://github.com/go-ole/go-ole,MIT,"Copyright (c) 2013-2017 Yasuhiro Matsumoto" github.com/google/uuid,https://github.com/google/uuid,BSD-3-Clause,Copyright (c) 2018 Google Inc. github.com/inconshreveable/mousetrap,https://github.com/inconshreveable/mousetrap,Apache-2.0,Copyright 2014 Alan Shreve +github.com/klauspost/compress,https://github.com/klauspost/compress,BSD-3-Clause,"Copyright (c) 2012 The Go Authors; Copyright (c) 2019 Klaus Post" github.com/lufia/plan9stats,https://github.com/lufia/plan9stats,BSD-3-Clause,"Copyright (c) 2019, KADOTA, Kyohei" +github.com/pierrec/lz4/v4,https://github.com/pierrec/lz4,BSD-3-Clause,"Copyright (c) 2015, Pierre Curto" github.com/pmezard/go-difflib,https://github.com/pmezard/go-difflib,BSD-3-Clause,"Copyright (c) 2013, Patrick Mezard" github.com/power-devops/perfstat,https://github.com/power-devops/perfstat,MIT,Copyright (c) 2020 Power DevOps github.com/prometheus-community/pro-bing,https://github.com/prometheus-community/pro-bing,MIT,"Copyright 2022 The Prometheus Authors; Copyright 2016 Cameron Sparr and contributors" @@ -20,6 +22,7 @@ github.com/spf13/pflag,https://github.com/spf13/pflag,BSD-3-Clause,"Copyright (c github.com/stretchr/testify,https://github.com/stretchr/testify,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors" github.com/tklauser/go-sysconf,https://github.com/tklauser/go-sysconf,BSD-3-Clause,"Copyright (c) 2018-2022, Tobias Klauser" github.com/tklauser/numcpus,https://github.com/tklauser/numcpus,Apache-2.0,"Copyright (c) 2018-2022, Tobias Klauser" +github.com/ulikunitz/xz,https://github.com/ulikunitz/xz,BSD-3-Clause,"Copyright (c) 2014-2022, Ulrich Kunitz" github.com/yusufpapurcu/wmi,https://github.com/yusufpapurcu/wmi,MIT,Copyright (c) 2013 Stack Exchange go.uber.org/atomic,https://github.com/uber-go/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc." go.yaml.in/yaml/v3,https://github.com/yaml/go-yaml,MIT AND Apache-2.0,"Copyright (c) 2006-2011 Kirill Simonov, 2011-2019 Canonical Ltd" diff --git a/go.mod b/go.mod index 841320b5..77fd543e 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,13 @@ toolchain go1.26.2 require ( github.com/DataDog/datadog-agent/pkg/fleet/installer v0.78.0 github.com/coreos/go-systemd/v22 v22.7.0 + github.com/klauspost/compress v1.19.0 + github.com/pierrec/lz4/v4 v4.1.27 github.com/prometheus-community/pro-bing v0.8.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 + github.com/ulikunitz/xz v0.5.15 golang.org/x/sys v0.46.0 golang.org/x/term v0.44.0 golang.org/x/tools v0.44.0 diff --git a/go.sum b/go.sum index 59fa7c52..0350ab9a 100644 --- a/go.sum +++ b/go.sum @@ -26,12 +26,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= @@ -54,6 +58,8 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/internal/systemd/journal_data.go b/internal/systemd/journal_data.go new file mode 100644 index 00000000..aff5380e --- /dev/null +++ b/internal/systemd/journal_data.go @@ -0,0 +1,270 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "math" + + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" + "github.com/ulikunitz/xz" +) + +const ( + journalDataRegularHeaderSize = 64 + journalDataCompactHeaderSize = 72 + + maxJournalDecodeWindow = 8 * 1024 * 1024 + maxJournalEncodedRead = 8 * 1024 * 1024 + maxJournalLZ4DataSize = 8 * 1024 * 1024 + maxJournalPayloadRead = maxJournalFieldSize + 512 +) + +type journalDataObject struct { + journalObject + hash uint64 + nextHashOffset uint64 + nextFieldOffset uint64 + entryOffset uint64 + entryArrayOffset uint64 + nEntries uint64 + tailEntryArrayOffset uint32 + tailEntryArrayNEntries uint32 + hasTailEntryArrayReference bool + payloadOffset uint64 + payloadSize uint64 +} + +func (f *journalFileView) dataObjectAt(offset uint64) (journalDataObject, error) { + object, err := f.objectAt(offset, journalObjectData) + if err != nil { + return journalDataObject{}, err + } + + headerSize := uint64(journalDataRegularHeaderSize) + if f.header.compact() { + headerSize = journalDataCompactHeaderSize + } + if object.size < headerSize { + return journalDataObject{}, journalCorrupt(f.name, offset, "DATA object size %d is smaller than its %d-byte header", object.size, headerSize) + } + + var raw [journalDataCompactHeaderSize]byte + if err := readJournalAt(f.name, f.reader, f.size, offset, raw[:headerSize]); err != nil { + return journalDataObject{}, err + } + data := journalDataObject{ + journalObject: object, + hash: binary.LittleEndian.Uint64(raw[16:24]), + nextHashOffset: binary.LittleEndian.Uint64(raw[24:32]), + nextFieldOffset: binary.LittleEndian.Uint64(raw[32:40]), + entryOffset: binary.LittleEndian.Uint64(raw[40:48]), + entryArrayOffset: binary.LittleEndian.Uint64(raw[48:56]), + nEntries: binary.LittleEndian.Uint64(raw[56:64]), + payloadOffset: offset + headerSize, + payloadSize: object.size - headerSize, + } + if f.header.compact() { + data.tailEntryArrayOffset = binary.LittleEndian.Uint32(raw[64:68]) + data.tailEntryArrayNEntries = binary.LittleEndian.Uint32(raw[68:72]) + data.hasTailEntryArrayReference = true + } + + for _, reference := range []struct { + offset uint64 + label string + }{ + {offset: data.nextHashOffset, label: "next DATA hash"}, + {offset: data.nextFieldOffset, label: "next DATA field"}, + {offset: data.entryOffset, label: "first DATA entry"}, + {offset: data.entryArrayOffset, label: "DATA entry array"}, + } { + if reference.offset == 0 { + continue + } + if err := validateJournalObjectOffset(f.name, f.header, reference.offset, reference.label); err != nil { + return journalDataObject{}, err + } + } + if data.hasTailEntryArrayReference { + if (data.tailEntryArrayOffset == 0) != (data.tailEntryArrayNEntries == 0) { + return journalDataObject{}, journalCorrupt(f.name, uint64(data.tailEntryArrayOffset), "DATA tail entry array offset and count are inconsistent") + } + if data.tailEntryArrayOffset != 0 { + tailOffset := uint64(data.tailEntryArrayOffset) + if err := validateJournalObjectOffset(f.name, f.header, tailOffset, "DATA tail entry array"); err != nil { + return journalDataObject{}, err + } + if data.entryArrayOffset == 0 || data.entryArrayOffset > tailOffset { + return journalDataObject{}, journalCorrupt(f.name, tailOffset, "DATA tail entry array precedes its entry array chain") + } + } + } + + return data, nil +} + +func (f *journalFileView) readDataPayload(data journalDataObject, limit int) ([]byte, bool, error) { + if limit <= 0 { + return nil, false, fmt.Errorf("journal DATA payload limit must be positive") + } + if limit == math.MaxInt { + return nil, false, fmt.Errorf("journal DATA payload limit is too large") + } + if limit > maxJournalPayloadRead { + return nil, false, fmt.Errorf("journal DATA payload limit %d exceeds maximum %d", limit, maxJournalPayloadRead) + } + + switch data.flags { + case 0: + return f.readUncompressedDataPayload(data, limit) + case journalObjectCompressedXZ: + decoder, err := (xz.ReaderConfig{ + DictCap: maxJournalDecodeWindow, + SingleStream: true, + }).NewReader(f.compressedPayloadReader(data)) + if err != nil { + if errors.Is(err, errJournalLimit) { + return nil, false, err + } + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode XZ DATA payload: %v", err) + } + return readDecodedJournalPayload(f.name, data.payloadOffset, "XZ", decoder, limit) + case journalObjectCompressedLZ4: + return f.readLZ4DataPayload(data, limit) + case journalObjectCompressedZSTD: + decoder, err := zstd.NewReader( + f.compressedPayloadReader(data), + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderLowmem(true), + zstd.WithDecoderMaxMemory(maxJournalDecodeWindow), + zstd.WithDecoderMaxWindow(maxJournalDecodeWindow), + ) + if err != nil { + if errors.Is(err, errJournalLimit) { + return nil, false, err + } + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode ZSTD DATA payload: %v", err) + } + defer decoder.Close() + return readDecodedJournalPayload(f.name, data.payloadOffset, "ZSTD", decoder, limit) + default: + return nil, false, journalUnsupported(f.name, "DATA object at offset %d uses compression flags 0x%02x", data.offset, data.flags) + } +} + +func (f *journalFileView) compressedPayloadReader(data journalDataObject) io.Reader { + section := io.NewSectionReader(f.reader, int64(data.payloadOffset), int64(data.payloadSize)) + if data.payloadSize <= maxJournalEncodedRead { + return section + } + return &journalCompressedReader{ + reader: section, + remaining: maxJournalEncodedRead, + name: f.name, + offset: data.payloadOffset, + } +} + +type journalCompressedReader struct { + reader io.Reader + remaining int + name string + offset uint64 +} + +func (r *journalCompressedReader) Read(destination []byte) (int, error) { + if len(destination) == 0 { + return 0, nil + } + if r.remaining == 0 { + return 0, journalLimit(r.name, r.offset, "compressed DATA input exceeds %d bytes", maxJournalEncodedRead) + } + if len(destination) > r.remaining { + destination = destination[:r.remaining] + } + n, err := r.reader.Read(destination) + r.remaining -= n + return n, err +} + +func (f *journalFileView) readUncompressedDataPayload(data journalDataObject, limit int) ([]byte, bool, error) { + readSize := data.payloadSize + if readSize > uint64(limit)+1 { + readSize = uint64(limit) + 1 + } + payload := make([]byte, int(readSize)) + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset, payload); err != nil { + return nil, false, err + } + if len(payload) > limit { + return payload[:limit], true, nil + } + return payload, false, nil +} + +func (f *journalFileView) readLZ4DataPayload(data journalDataObject, limit int) ([]byte, bool, error) { + if data.payloadSize <= 8 { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "LZ4 DATA payload is too small") + } + + var sizeBytes [8]byte + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset, sizeBytes[:]); err != nil { + return nil, false, err + } + decodedSize := binary.LittleEndian.Uint64(sizeBytes[:]) + if decodedSize == 0 { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "LZ4 DATA payload has an empty decoded size") + } + if decodedSize > maxJournalLZ4DataSize { + return nil, false, journalLimit(f.name, data.payloadOffset, "LZ4 DATA payload expands to %d bytes; maximum is %d", decodedSize, maxJournalLZ4DataSize) + } + if data.payloadSize-8 > maxJournalEncodedRead { + return nil, false, journalLimit(f.name, data.payloadOffset, "compressed LZ4 DATA input exceeds %d bytes", maxJournalEncodedRead) + } + if decodedSize > uint64(math.MaxInt) || data.payloadSize-8 > uint64(math.MaxInt) { + return nil, false, journalLimit(f.name, data.payloadOffset, "LZ4 DATA payload exceeds the platform allocation range") + } + + compressed := make([]byte, int(data.payloadSize-8)) + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset+8, compressed); err != nil { + return nil, false, err + } + decoded := make([]byte, int(decodedSize)) + n, err := lz4.UncompressBlock(compressed, decoded) + if err != nil { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: %v", err) + } + if n != len(decoded) { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: got %d of %d bytes", n, len(decoded)) + } + if len(decoded) > limit { + return decoded[:limit], true, nil + } + return decoded, false, nil +} + +func readDecodedJournalPayload(name string, offset uint64, algorithm string, reader io.Reader, limit int) ([]byte, bool, error) { + payload, err := io.ReadAll(io.LimitReader(reader, int64(limit)+1)) + if err != nil { + if errors.Is(err, errJournalLimit) { + return nil, false, err + } + return nil, false, journalCorrupt(name, offset, "decode %s DATA payload: %v", algorithm, err) + } + if len(payload) > limit { + return payload[:limit], true, nil + } + return payload, false, nil +} + +func journalLimit(name string, offset uint64, format string, arguments ...any) error { + return fmt.Errorf("%w %q at offset %d: %s", errJournalLimit, name, offset, fmt.Sprintf(format, arguments...)) +} diff --git a/internal/systemd/journal_data_test.go b/internal/systemd/journal_data_test.go new file mode 100644 index 00000000..e09dabca --- /dev/null +++ b/internal/systemd/journal_data_test.go @@ -0,0 +1,209 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ulikunitz/xz" +) + +func TestJournalDataObjectParsesRegularAndCompactLayouts(t *testing.T) { + for _, compact := range []bool{false, true} { + name := "regular" + if compact { + name = "compact" + } + t.Run(name, func(t *testing.T) { + contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, compact) + binary.LittleEndian.PutUint64(contents[offset+16:offset+24], 0x0102030405060708) + binary.LittleEndian.PutUint64(contents[offset+56:offset+64], 3) + + view, err := newJournalFileView("data.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + assert.Equal(t, uint64(0x0102030405060708), data.hash) + assert.Equal(t, uint64(3), data.nEntries) + assert.Equal(t, uint64(len("MESSAGE=hello")), data.payloadSize) + headerSize := uint64(journalDataRegularHeaderSize) + if compact { + headerSize = journalDataCompactHeaderSize + } + assert.Equal(t, uint64(offset)+headerSize, data.payloadOffset) + assert.Equal(t, compact, data.hasTailEntryArrayReference) + }) + } +} + +func TestJournalDataPayloadDecodesSupportedCompression(t *testing.T) { + payload := []byte("MESSAGE=" + strings.Repeat("journal payload ", 32)) + tests := []struct { + name string + objectFlag uint8 + encode func(*testing.T, []byte) []byte + }{ + {name: "uncompressed", encode: func(_ *testing.T, data []byte) []byte { return data }}, + {name: "xz", objectFlag: journalObjectCompressedXZ, encode: encodeJournalXZ}, + {name: "lz4", objectFlag: journalObjectCompressedLZ4, encode: encodeJournalLZ4}, + {name: "zstd", objectFlag: journalObjectCompressedZSTD, encode: encodeJournalZSTD}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + encoded := test.encode(t, payload) + contents, offset := testJournalDataContents(t, encoded, test.objectFlag, false) + view, err := newJournalFileView(test.name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + decoded, truncated, err := view.readDataPayload(data, len(payload)+1) + require.NoError(t, err) + assert.False(t, truncated) + assert.Equal(t, payload, decoded) + + decoded, truncated, err = view.readDataPayload(data, 17) + require.NoError(t, err) + assert.True(t, truncated) + assert.Equal(t, payload[:17], decoded) + }) + } +} + +func TestJournalDataPayloadRejectsMalformedCompression(t *testing.T) { + for _, objectFlag := range []uint8{ + journalObjectCompressedXZ, + journalObjectCompressedLZ4, + journalObjectCompressedZSTD, + } { + malformed := bytes.Repeat([]byte{0xff}, 32) + if objectFlag == journalObjectCompressedLZ4 { + binary.LittleEndian.PutUint64(malformed[:8], 64) + } + contents, offset := testJournalDataContents(t, malformed, objectFlag, false) + view, err := newJournalFileView("malformed.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + _, _, err = view.readDataPayload(data, 64) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "decode") + } +} + +func TestJournalDataPayloadBoundsLZ4Expansion(t *testing.T) { + encoded := make([]byte, 9) + binary.LittleEndian.PutUint64(encoded[:8], maxJournalLZ4DataSize+1) + contents, offset := testJournalDataContents(t, encoded, journalObjectCompressedLZ4, false) + view, err := newJournalFileView("large-lz4.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + _, _, err = view.readDataPayload(data, 64) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalLimit) + assert.Contains(t, err.Error(), "expands to 8388609 bytes") +} + +func TestJournalDataPayloadRejectsUnboundedReadLimit(t *testing.T) { + contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, false) + view, err := newJournalFileView("limit.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + _, _, err = view.readDataPayload(data, maxJournalPayloadRead+1) + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeds maximum") +} + +func TestJournalDataObjectRejectsInvalidReferences(t *testing.T) { + contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, false) + binary.LittleEndian.PutUint64(contents[offset+24:offset+32], uint64(offset+1)) + view, err := newJournalFileView("reference.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + _, err = view.dataObjectAt(uint64(offset)) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "next DATA hash offset is not 8-byte aligned") +} + +func testJournalDataContents(t *testing.T, encoded []byte, objectFlag uint8, compact bool) ([]byte, int) { + t.Helper() + headerFlags := uint32(0) + switch objectFlag { + case journalObjectCompressedXZ: + headerFlags |= journalHeaderIncompatibleCompressedXZ + case journalObjectCompressedLZ4: + headerFlags |= journalHeaderIncompatibleCompressedLZ4 + case journalObjectCompressedZSTD: + headerFlags |= journalHeaderIncompatibleCompressedZSTD + } + headerSize := journalHeaderCurrentSize + dataHeaderSize := journalDataRegularHeaderSize + if compact { + headerFlags |= journalHeaderIncompatibleCompact + dataHeaderSize = journalDataCompactHeaderSize + } + objectSize := dataHeaderSize + len(encoded) + fileSize := alignJournalTestSize(headerSize + objectSize) + contents := testJournalContents(fileSize, 0, headerFlags) + binary.LittleEndian.PutUint64(contents[136:144], uint64(headerSize)) + object := contents[headerSize:] + object[0] = journalObjectData + object[1] = objectFlag + binary.LittleEndian.PutUint64(object[8:16], uint64(objectSize)) + copy(object[dataHeaderSize:], encoded) + return contents, headerSize +} + +func alignJournalTestSize(size int) int { + return (size + 7) &^ 7 +} + +func encodeJournalXZ(t *testing.T, payload []byte) []byte { + t.Helper() + var encoded bytes.Buffer + writer, err := xz.NewWriter(&encoded) + require.NoError(t, err) + _, err = writer.Write(payload) + require.NoError(t, err) + require.NoError(t, writer.Close()) + return encoded.Bytes() +} + +func encodeJournalLZ4(t *testing.T, payload []byte) []byte { + t.Helper() + compressed := make([]byte, lz4.CompressBlockBound(len(payload))) + n, err := lz4.CompressBlock(payload, compressed, nil) + require.NoError(t, err) + require.Positive(t, n) + encoded := make([]byte, 8+n) + binary.LittleEndian.PutUint64(encoded[:8], uint64(len(payload))) + copy(encoded[8:], compressed[:n]) + return encoded +} + +func encodeJournalZSTD(t *testing.T, payload []byte) []byte { + t.Helper() + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + require.NoError(t, err) + defer encoder.Close() + return encoder.EncodeAll(payload, nil) +} diff --git a/internal/systemd/journal_format.go b/internal/systemd/journal_format.go index dca5a886..19ced3b1 100644 --- a/internal/systemd/journal_format.go +++ b/internal/systemd/journal_format.go @@ -53,6 +53,7 @@ const ( var ( errJournalCorrupt = errors.New("corrupt systemd journal") errJournalUnsupported = errors.New("unsupported systemd journal format") + errJournalLimit = errors.New("systemd journal resource limit exceeded") journalSignature = [8]byte{'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H'} ) From 4b3e861b5b437548a8a2d59057b8fc49eec8bae3 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 13:39:07 -0400 Subject: [PATCH 22/53] feat(systemd): add journal hash compatibility --- internal/systemd/journal_hash.go | 136 ++++++++++++++++++++++++++ internal/systemd/journal_hash_test.go | 43 ++++++++ 2 files changed, 179 insertions(+) create mode 100644 internal/systemd/journal_hash.go create mode 100644 internal/systemd/journal_hash_test.go diff --git a/internal/systemd/journal_hash.go b/internal/systemd/journal_hash.go new file mode 100644 index 00000000..e2ab2430 --- /dev/null +++ b/internal/systemd/journal_hash.go @@ -0,0 +1,136 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "math/bits" +) + +// journalJenkinsHash64 implements lookup3 hashlittle2 with zero seeds, which +// older journal files use for DATA and FIELD objects. +func journalJenkinsHash64(data []byte) uint64 { + a := uint32(0xdeadbeef) + uint32(len(data)) + b := a + c := a + + for len(data) > 12 { + a += binary.LittleEndian.Uint32(data[0:4]) + b += binary.LittleEndian.Uint32(data[4:8]) + c += binary.LittleEndian.Uint32(data[8:12]) + a, b, c = journalJenkinsMix(a, b, c) + data = data[12:] + } + if len(data) == 0 { + return uint64(c)<<32 | uint64(b) + } + for index, value := range data { + shift := uint(index%4) * 8 + switch index / 4 { + case 0: + a += uint32(value) << shift + case 1: + b += uint32(value) << shift + case 2: + c += uint32(value) << shift + } + } + a, b, c = journalJenkinsFinal(a, b, c) + return uint64(c)<<32 | uint64(b) +} + +func journalJenkinsMix(a, b, c uint32) (uint32, uint32, uint32) { + a -= c + a ^= bits.RotateLeft32(c, 4) + c += b + b -= a + b ^= bits.RotateLeft32(a, 6) + a += c + c -= b + c ^= bits.RotateLeft32(b, 8) + b += a + a -= c + a ^= bits.RotateLeft32(c, 16) + c += b + b -= a + b ^= bits.RotateLeft32(a, 19) + a += c + c -= b + c ^= bits.RotateLeft32(b, 4) + b += a + return a, b, c +} + +func journalJenkinsFinal(a, b, c uint32) (uint32, uint32, uint32) { + c ^= b + c -= bits.RotateLeft32(b, 14) + a ^= c + a -= bits.RotateLeft32(c, 11) + b ^= a + b -= bits.RotateLeft32(a, 25) + c ^= b + c -= bits.RotateLeft32(b, 16) + a ^= c + a -= bits.RotateLeft32(c, 4) + b ^= a + b -= bits.RotateLeft32(a, 14) + c ^= b + c -= bits.RotateLeft32(b, 24) + return a, b, c +} + +// journalSipHash24 implements SipHash-2-4 with the journal file ID as its +// 128-bit key, which modern journal files use for DATA and FIELD objects. +func journalSipHash24(data []byte, key journalID) uint64 { + k0 := binary.LittleEndian.Uint64(key[0:8]) + k1 := binary.LittleEndian.Uint64(key[8:16]) + v0 := uint64(0x736f6d6570736575) ^ k0 + v1 := uint64(0x646f72616e646f6d) ^ k1 + v2 := uint64(0x6c7967656e657261) ^ k0 + v3 := uint64(0x7465646279746573) ^ k1 + length := len(data) + + for len(data) >= 8 { + message := binary.LittleEndian.Uint64(data[:8]) + v3 ^= message + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + v0 ^= message + data = data[8:] + } + + last := uint64(length) << 56 + for index, value := range data { + last |= uint64(value) << (uint(index) * 8) + } + v3 ^= last + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + v0 ^= last + v2 ^= 0xff + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + journalSipRound(&v0, &v1, &v2, &v3) + return v0 ^ v1 ^ v2 ^ v3 +} + +func journalSipRound(v0, v1, v2, v3 *uint64) { + *v0 += *v1 + *v1 = bits.RotateLeft64(*v1, 13) + *v1 ^= *v0 + *v0 = bits.RotateLeft64(*v0, 32) + *v2 += *v3 + *v3 = bits.RotateLeft64(*v3, 16) + *v3 ^= *v2 + *v0 += *v3 + *v3 = bits.RotateLeft64(*v3, 21) + *v3 ^= *v0 + *v2 += *v1 + *v1 = bits.RotateLeft64(*v1, 17) + *v1 ^= *v2 + *v2 = bits.RotateLeft64(*v2, 32) +} diff --git a/internal/systemd/journal_hash_test.go b/internal/systemd/journal_hash_test.go new file mode 100644 index 00000000..d852d2db --- /dev/null +++ b/internal/systemd/journal_hash_test.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJournalJenkinsHash64MatchesLookup3Vectors(t *testing.T) { + assert.Equal(t, uint64(0xdeadbeefdeadbeef), journalJenkinsHash64(nil)) + assert.Equal(t, uint64(0x17770551ce7226e6), journalJenkinsHash64([]byte("Four score and seven years ago"))) +} + +func TestJournalSipHash24MatchesReferenceVectors(t *testing.T) { + var key journalID + var message [64]byte + for index := range key { + key[index] = byte(index) + } + for index := range message { + message[index] = byte(index) + } + + vectors := []struct { + length int + hash uint64 + }{ + {length: 0, hash: 0x726fdb47dd0e0e31}, + {length: 1, hash: 0x74f839c593dc67fd}, + {length: 7, hash: 0xab0200f58b01d137}, + {length: 8, hash: 0x93f5f5799a932462}, + {length: 15, hash: 0xa129ca6149be45e5}, + {length: 63, hash: 0x958a324ceb064572}, + } + for _, vector := range vectors { + assert.Equal(t, vector.hash, journalSipHash24(message[:vector.length], key), "length %d", vector.length) + } +} From c1974fdcfc4f5a26b9822fbb56490c146b5279a8 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 13:41:29 -0400 Subject: [PATCH 23/53] feat(systemd): add indexed journal data lookup --- internal/systemd/journal_index.go | 144 +++++++++++++++++++++ internal/systemd/journal_index_test.go | 172 +++++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 internal/systemd/journal_index.go create mode 100644 internal/systemd/journal_index_test.go diff --git a/internal/systemd/journal_index.go b/internal/systemd/journal_index.go new file mode 100644 index 00000000..30f0767c --- /dev/null +++ b/internal/systemd/journal_index.go @@ -0,0 +1,144 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +const ( + journalHashItemSize = 16 + maxJournalHashChainDepth = 128 +) + +func (f *journalFileView) dataHash(payload []byte) uint64 { + if f.header.keyedHash() { + return journalSipHash24(payload, f.header.fileID) + } + return journalJenkinsHash64(payload) +} + +func (f *journalFileView) findDataObject(payload []byte) (journalDataObject, bool, error) { + if len(payload) == 0 { + return journalDataObject{}, false, fmt.Errorf("journal DATA lookup payload is empty") + } + if len(payload) > maxJournalPayloadRead { + return journalDataObject{}, false, fmt.Errorf("journal DATA lookup payload exceeds %d bytes", maxJournalPayloadRead) + } + if f.header.dataHashTableSize == 0 { + return journalDataObject{}, false, nil + } + + tableObjectOffset := f.header.dataHashTableOffset - journalObjectHeaderSize + table, err := f.objectAt(tableObjectOffset, journalObjectDataHashTable) + if err != nil { + return journalDataObject{}, false, err + } + if table.size-journalObjectHeaderSize != f.header.dataHashTableSize { + return journalDataObject{}, false, journalCorrupt( + f.name, + tableObjectOffset, + "DATA hash table object payload is %d bytes; header declares %d", + table.size-journalObjectHeaderSize, + f.header.dataHashTableSize, + ) + } + bucketCount := f.header.dataHashTableSize / journalHashItemSize + if bucketCount == 0 { + return journalDataObject{}, false, journalCorrupt(f.name, f.header.dataHashTableOffset, "DATA hash table has no buckets") + } + + hash := f.dataHash(payload) + bucket := hash % bucketCount + itemOffset := f.header.dataHashTableOffset + bucket*journalHashItemSize + var item [journalHashItemSize]byte + if err := readJournalAt(f.name, f.reader, f.size, itemOffset, item[:]); err != nil { + return journalDataObject{}, false, err + } + headOffset := binary.LittleEndian.Uint64(item[0:8]) + tailOffset := binary.LittleEndian.Uint64(item[8:16]) + if (headOffset == 0) != (tailOffset == 0) { + return journalDataObject{}, false, journalCorrupt(f.name, itemOffset, "DATA hash bucket head and tail are inconsistent") + } + if headOffset == 0 { + return journalDataObject{}, false, nil + } + if err := validateJournalObjectOffset(f.name, f.header, headOffset, "DATA hash bucket head"); err != nil { + return journalDataObject{}, false, err + } + if err := validateJournalObjectOffset(f.name, f.header, tailOffset, "DATA hash bucket tail"); err != nil { + return journalDataObject{}, false, err + } + + seen := make(map[uint64]struct{}, maxJournalHashChainDepth) + for offset, depth := headOffset, 0; offset != 0; depth++ { + if depth >= maxJournalHashChainDepth { + return journalDataObject{}, false, journalLimit(f.name, offset, "DATA hash chain exceeds %d objects", maxJournalHashChainDepth) + } + if _, exists := seen[offset]; exists { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA hash chain contains a cycle") + } + seen[offset] = struct{}{} + + data, err := f.dataObjectAt(offset) + if err != nil { + return journalDataObject{}, false, err + } + if data.hash%bucketCount != bucket { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA object hash belongs to bucket %d instead of %d", data.hash%bucketCount, bucket) + } + if data.hash == hash { + equal, err := f.dataPayloadEqual(data, payload) + if err != nil { + return journalDataObject{}, false, err + } + if equal { + return data, true, nil + } + } + + if offset == tailOffset { + if data.nextHashOffset != 0 { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA hash bucket continues after its declared tail") + } + return journalDataObject{}, false, nil + } + if data.nextHashOffset == 0 { + return journalDataObject{}, false, journalCorrupt(f.name, offset, "DATA hash bucket ends before its declared tail") + } + offset = data.nextHashOffset + } + + return journalDataObject{}, false, journalCorrupt(f.name, itemOffset, "DATA hash bucket did not reach its declared tail") +} + +func (f *journalFileView) dataPayloadEqual(data journalDataObject, expected []byte) (bool, error) { + switch data.flags { + case 0: + if data.payloadSize != uint64(len(expected)) { + return false, nil + } + case journalObjectCompressedLZ4: + if data.payloadSize <= 8 { + return false, journalCorrupt(f.name, data.payloadOffset, "LZ4 DATA payload is too small") + } + var sizeBytes [8]byte + if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset, sizeBytes[:]); err != nil { + return false, err + } + if binary.LittleEndian.Uint64(sizeBytes[:]) != uint64(len(expected)) { + return false, nil + } + } + + payload, truncated, err := f.readDataPayload(data, len(expected)) + if err != nil { + return false, err + } + return !truncated && bytes.Equal(payload, expected), nil +} diff --git a/internal/systemd/journal_index_test.go b/internal/systemd/journal_index_test.go new file mode 100644 index 00000000..80c26fc7 --- /dev/null +++ b/internal/systemd/journal_index_test.go @@ -0,0 +1,172 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalDataIndexFindsExactPayload(t *testing.T) { + for _, keyed := range []bool{false, true} { + name := "jenkins" + if keyed { + name = "siphash" + } + t.Run(name, func(t *testing.T) { + contents, offsets := testIndexedJournal(t, [][]byte{ + []byte("_SYSTEMD_UNIT=other.service"), + []byte("_SYSTEMD_UNIT=api.service"), + }, keyed) + view, err := newJournalFileView(name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + data, found, err := view.findDataObject([]byte("_SYSTEMD_UNIT=api.service")) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(offsets[1]), data.offset) + + _, found, err = view.findDataObject([]byte("_SYSTEMD_UNIT=missing.service")) + require.NoError(t, err) + assert.False(t, found) + }) + } +} + +func TestJournalDataIndexVerifiesPayloadAfterHashMatch(t *testing.T) { + target := []byte("_SYSTEMD_UNIT=api.service") + contents, offsets := testIndexedJournal(t, [][]byte{ + []byte("_SYSTEMD_UNIT=not-api.service"), + target, + }, false) + targetHash := journalJenkinsHash64(target) + binary.LittleEndian.PutUint64(contents[offsets[0]+16:offsets[0]+24], targetHash) + + view, err := newJournalFileView("collision.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, found, err := view.findDataObject(target) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(offsets[1]), data.offset) +} + +func TestJournalDataIndexRejectsCycle(t *testing.T) { + contents, offsets := testIndexedJournal(t, [][]byte{ + []byte("_SYSTEMD_UNIT=one.service"), + []byte("_SYSTEMD_UNIT=two.service"), + []byte("_SYSTEMD_UNIT=three.service"), + }, false) + binary.LittleEndian.PutUint64(contents[offsets[1]+24:offsets[1]+32], uint64(offsets[0])) + + view, err := newJournalFileView("cycle.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + _, _, err = view.findDataObject([]byte("_SYSTEMD_UNIT=missing.service")) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "contains a cycle") +} + +func TestJournalDataIndexRejectsWrongTableObject(t *testing.T) { + contents, _ := testIndexedJournal(t, [][]byte{[]byte("MESSAGE=hello")}, false) + contents[journalHeaderCurrentSize] = journalObjectFieldHashTable + view, err := newJournalFileView("table.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + _, _, err = view.findDataObject([]byte("MESSAGE=hello")) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "expected object type 4, found 5") +} + +func TestJournalDataIndexSkipsOversizedLZ4Collision(t *testing.T) { + target := []byte("MESSAGE=hello") + contents, offsets := testIndexedJournal(t, [][]byte{[]byte("MESSAGE=other"), target}, false) + targetHash := journalJenkinsHash64(target) + first := offsets[0] + binary.LittleEndian.PutUint64(contents[first+16:first+24], targetHash) + contents[first+1] = journalObjectCompressedLZ4 + binary.LittleEndian.PutUint32(contents[12:16], journalHeaderIncompatibleCompressedLZ4) + binary.LittleEndian.PutUint64(contents[first+64:first+72], maxJournalLZ4DataSize+1) + + view, err := newJournalFileView("large-collision.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, found, err := view.findDataObject(target) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(offsets[1]), data.offset) +} + +func TestJournalDataPayloadEqualDecodesCompressedValue(t *testing.T) { + payload := []byte("_SYSTEMD_UNIT=api.service") + encoded := encodeJournalZSTD(t, payload) + contents, offset := testJournalDataContents(t, encoded, journalObjectCompressedZSTD, false) + view, err := newJournalFileView("compressed-match.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + equal, err := view.dataPayloadEqual(data, payload) + require.NoError(t, err) + assert.True(t, equal) + equal, err = view.dataPayloadEqual(data, []byte("_SYSTEMD_UNIT=other.service")) + require.NoError(t, err) + assert.False(t, equal) +} + +func testIndexedJournal(t *testing.T, payloads [][]byte, keyed bool) ([]byte, []int) { + t.Helper() + require.NotEmpty(t, payloads) + + const bucketCount = 1 + tableOffset := journalHeaderCurrentSize + tableSize := journalObjectHeaderSize + bucketCount*journalHashItemSize + nextOffset := alignJournalTestSize(tableOffset + tableSize) + offsets := make([]int, len(payloads)) + for index, payload := range payloads { + offsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalDataRegularHeaderSize + len(payload)) + } + contents := testJournalContents(nextOffset, 0, 0) + if keyed { + binary.LittleEndian.PutUint32(contents[12:16], journalHeaderIncompatibleKeyedHash) + for index := 0; index < 16; index++ { + contents[24+index] = byte(index) + } + } + binary.LittleEndian.PutUint64(contents[104:112], uint64(tableOffset+journalObjectHeaderSize)) + binary.LittleEndian.PutUint64(contents[112:120], bucketCount*journalHashItemSize) + binary.LittleEndian.PutUint64(contents[136:144], uint64(offsets[len(offsets)-1])) + binary.LittleEndian.PutUint64(contents[144:152], uint64(len(payloads)+1)) + + table := contents[tableOffset:] + table[0] = journalObjectDataHashTable + binary.LittleEndian.PutUint64(table[8:16], uint64(tableSize)) + binary.LittleEndian.PutUint64(table[16:24], uint64(offsets[0])) + binary.LittleEndian.PutUint64(table[24:32], uint64(offsets[len(offsets)-1])) + + var fileID journalID + copy(fileID[:], contents[24:40]) + for index, payload := range payloads { + offset := offsets[index] + object := contents[offset:] + object[0] = journalObjectData + binary.LittleEndian.PutUint64(object[8:16], uint64(journalDataRegularHeaderSize+len(payload))) + hash := journalJenkinsHash64(payload) + if keyed { + hash = journalSipHash24(payload, fileID) + } + binary.LittleEndian.PutUint64(object[16:24], hash) + if index+1 < len(offsets) { + binary.LittleEndian.PutUint64(object[24:32], uint64(offsets[index+1])) + } + copy(object[journalDataRegularHeaderSize:], payload) + } + return contents, offsets +} From e4505a5f686dcde8d711d863603d7d4fe762ec2a Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 13:44:38 -0400 Subject: [PATCH 24/53] feat(systemd): traverse journal entry indexes --- internal/systemd/journal_entry.go | 291 +++++++++++++++++++++++++ internal/systemd/journal_entry_test.go | 218 ++++++++++++++++++ 2 files changed, 509 insertions(+) create mode 100644 internal/systemd/journal_entry.go create mode 100644 internal/systemd/journal_entry_test.go diff --git a/internal/systemd/journal_entry.go b/internal/systemd/journal_entry.go new file mode 100644 index 00000000..389d2a7f --- /dev/null +++ b/internal/systemd/journal_entry.go @@ -0,0 +1,291 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/binary" + "fmt" +) + +const ( + journalEntryHeaderSize = 64 + journalEntryArrayHeaderSize = 24 + journalEntryRegularItemSize = 16 + journalEntryCompactItemSize = 4 + journalEntryArrayRegularItemSize = 8 + + maxJournalEntryFields = 1024 + maxJournalEntryArrays = 128 +) + +type journalEntryItem struct { + dataOffset uint64 + hash uint64 +} + +type journalEntryObject struct { + journalObject + seqnum uint64 + realtime uint64 + monotonic uint64 + bootID journalID + xorHash uint64 + items []journalEntryItem +} + +func (f *journalFileView) entryObjectAt(offset uint64) (journalEntryObject, error) { + object, err := f.objectAt(offset, journalObjectEntry) + if err != nil { + return journalEntryObject{}, err + } + if object.size < journalEntryHeaderSize { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY object size %d is smaller than its header", object.size) + } + + itemSize := uint64(journalEntryRegularItemSize) + if f.header.compact() { + itemSize = journalEntryCompactItemSize + } + itemsSize := object.size - journalEntryHeaderSize + if itemsSize%itemSize != 0 { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY item payload size %d is not a multiple of %d", itemsSize, itemSize) + } + itemCount := itemsSize / itemSize + if itemCount == 0 { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY object has no DATA items") + } + if itemCount > maxJournalEntryFields { + return journalEntryObject{}, journalLimit(f.name, offset, "ENTRY object has %d DATA items; maximum is %d", itemCount, maxJournalEntryFields) + } + + contents := make([]byte, int(object.size)) + if err := readJournalAt(f.name, f.reader, f.size, offset, contents); err != nil { + return journalEntryObject{}, err + } + entry := journalEntryObject{ + journalObject: object, + seqnum: binary.LittleEndian.Uint64(contents[16:24]), + realtime: binary.LittleEndian.Uint64(contents[24:32]), + monotonic: binary.LittleEndian.Uint64(contents[32:40]), + xorHash: binary.LittleEndian.Uint64(contents[56:64]), + items: make([]journalEntryItem, 0, int(itemCount)), + } + copy(entry.bootID[:], contents[40:56]) + for index := uint64(0); index < itemCount; index++ { + position := uint64(journalEntryHeaderSize) + index*itemSize + item := journalEntryItem{} + if f.header.compact() { + item.dataOffset = uint64(binary.LittleEndian.Uint32(contents[position : position+4])) + } else { + item.dataOffset = binary.LittleEndian.Uint64(contents[position : position+8]) + item.hash = binary.LittleEndian.Uint64(contents[position+8 : position+16]) + } + if item.dataOffset == 0 { + return journalEntryObject{}, journalCorrupt(f.name, offset, "ENTRY item %d has a zero DATA offset", index) + } + if err := validateJournalObjectOffset(f.name, f.header, item.dataOffset, fmt.Sprintf("ENTRY item %d DATA", index)); err != nil { + return journalEntryObject{}, err + } + entry.items = append(entry.items, item) + } + return entry, nil +} + +type journalEntryArray struct { + journalObject + nextOffset uint64 + itemsOffset uint64 + capacity uint64 +} + +func (f *journalFileView) entryArrayAt(offset uint64) (journalEntryArray, error) { + object, err := f.objectAt(offset, journalObjectEntryArray) + if err != nil { + return journalEntryArray{}, err + } + if object.size < journalEntryArrayHeaderSize { + return journalEntryArray{}, journalCorrupt(f.name, offset, "ENTRY_ARRAY object size %d is smaller than its header", object.size) + } + + itemSize := uint64(journalEntryArrayRegularItemSize) + if f.header.compact() { + itemSize = journalEntryCompactItemSize + } + itemsSize := object.size - journalEntryArrayHeaderSize + if itemsSize%itemSize != 0 { + return journalEntryArray{}, journalCorrupt(f.name, offset, "ENTRY_ARRAY item payload size %d is not a multiple of %d", itemsSize, itemSize) + } + capacity := itemsSize / itemSize + if capacity == 0 { + return journalEntryArray{}, journalCorrupt(f.name, offset, "ENTRY_ARRAY object has no item capacity") + } + + var header [journalEntryArrayHeaderSize]byte + if err := readJournalAt(f.name, f.reader, f.size, offset, header[:]); err != nil { + return journalEntryArray{}, err + } + array := journalEntryArray{ + journalObject: object, + nextOffset: binary.LittleEndian.Uint64(header[16:24]), + itemsOffset: offset + journalEntryArrayHeaderSize, + capacity: capacity, + } + if array.nextOffset != 0 { + if err := validateJournalObjectOffset(f.name, f.header, array.nextOffset, "next ENTRY_ARRAY"); err != nil { + return journalEntryArray{}, err + } + } + return array, nil +} + +func (f *journalFileView) entryArrayItem(array journalEntryArray, index uint64) (uint64, error) { + if index >= array.capacity { + return 0, journalCorrupt(f.name, array.offset, "ENTRY_ARRAY item index %d exceeds capacity %d", index, array.capacity) + } + itemSize := uint64(journalEntryArrayRegularItemSize) + if f.header.compact() { + itemSize = journalEntryCompactItemSize + } + position := array.itemsOffset + index*itemSize + var raw [8]byte + if err := readJournalAt(f.name, f.reader, f.size, position, raw[:itemSize]); err != nil { + return 0, err + } + offset := binary.LittleEndian.Uint64(raw[:]) + if f.header.compact() { + offset = uint64(binary.LittleEndian.Uint32(raw[:4])) + } + if offset == 0 { + return 0, journalCorrupt(f.name, position, "ENTRY_ARRAY item %d has a zero ENTRY offset", index) + } + if err := validateJournalObjectOffset(f.name, f.header, offset, "ENTRY_ARRAY item"); err != nil { + return 0, err + } + return offset, nil +} + +type journalEntryArraySegment struct { + array journalEntryArray + count uint64 +} + +type journalEntryOffsetIterator struct { + file *journalFileView + inlineOffset uint64 + inlinePending bool + segments []journalEntryArraySegment + segmentIndex int + itemIndex uint64 + lastOffset uint64 +} + +func (f *journalFileView) entryOffsetsForData(data journalDataObject) (*journalEntryOffsetIterator, error) { + iterator := &journalEntryOffsetIterator{file: f, segmentIndex: -1} + if data.nEntries == 0 { + if data.entryOffset != 0 || data.entryArrayOffset != 0 || data.tailEntryArrayOffset != 0 || data.tailEntryArrayNEntries != 0 { + return nil, journalCorrupt(f.name, data.offset, "unreferenced DATA object has entry offsets") + } + return iterator, nil + } + if data.entryOffset == 0 { + return nil, journalCorrupt(f.name, data.offset, "referenced DATA object has no inline ENTRY offset") + } + if data.nEntries > f.header.nEntries { + return nil, journalCorrupt(f.name, data.offset, "DATA object references %d entries but the journal has %d", data.nEntries, f.header.nEntries) + } + iterator.inlineOffset = data.entryOffset + iterator.inlinePending = true + + remaining := data.nEntries - 1 + if remaining == 0 { + if data.entryArrayOffset != 0 { + return nil, journalCorrupt(f.name, data.offset, "single-entry DATA object has an ENTRY_ARRAY") + } + return iterator, nil + } + if data.entryArrayOffset == 0 { + return nil, journalCorrupt(f.name, data.offset, "multi-entry DATA object has no ENTRY_ARRAY") + } + + seen := make(map[uint64]struct{}, maxJournalEntryArrays) + offset := data.entryArrayOffset + for remaining > 0 { + if len(iterator.segments) >= maxJournalEntryArrays { + return nil, journalLimit(f.name, offset, "ENTRY_ARRAY chain exceeds %d objects", maxJournalEntryArrays) + } + if _, exists := seen[offset]; exists { + return nil, journalCorrupt(f.name, offset, "ENTRY_ARRAY chain contains a cycle") + } + seen[offset] = struct{}{} + + array, err := f.entryArrayAt(offset) + if err != nil { + return nil, err + } + count := array.capacity + if count > remaining { + count = remaining + } + iterator.segments = append(iterator.segments, journalEntryArraySegment{array: array, count: count}) + remaining -= count + if remaining == 0 { + if array.nextOffset != 0 { + return nil, journalCorrupt(f.name, array.offset, "ENTRY_ARRAY chain continues after all referenced entries") + } + break + } + if array.nextOffset == 0 { + return nil, journalCorrupt(f.name, array.offset, "ENTRY_ARRAY chain ends before all referenced entries") + } + offset = array.nextOffset + } + + if data.hasTailEntryArrayReference { + last := iterator.segments[len(iterator.segments)-1] + if uint64(data.tailEntryArrayOffset) != last.array.offset || uint64(data.tailEntryArrayNEntries) != last.count { + return nil, journalCorrupt(f.name, data.offset, "DATA tail ENTRY_ARRAY metadata does not match its array chain") + } + } + iterator.segmentIndex = len(iterator.segments) - 1 + iterator.itemIndex = iterator.segments[iterator.segmentIndex].count + return iterator, nil +} + +func (i *journalEntryOffsetIterator) previous() (uint64, bool, error) { + for i.segmentIndex >= 0 { + if i.itemIndex > 0 { + i.itemIndex-- + offset, err := i.file.entryArrayItem(i.segments[i.segmentIndex].array, i.itemIndex) + if err != nil { + return 0, false, err + } + if err := i.validateDescending(offset); err != nil { + return 0, false, err + } + return offset, true, nil + } + i.segmentIndex-- + if i.segmentIndex >= 0 { + i.itemIndex = i.segments[i.segmentIndex].count + } + } + if i.inlinePending { + i.inlinePending = false + if err := i.validateDescending(i.inlineOffset); err != nil { + return 0, false, err + } + return i.inlineOffset, true, nil + } + return 0, false, nil +} + +func (i *journalEntryOffsetIterator) validateDescending(offset uint64) error { + if i.lastOffset != 0 && offset >= i.lastOffset { + return journalCorrupt(i.file.name, offset, "ENTRY offsets are not strictly increasing in the forward index") + } + i.lastOffset = offset + return nil +} diff --git a/internal/systemd/journal_entry_test.go b/internal/systemd/journal_entry_test.go new file mode 100644 index 00000000..d101cf30 --- /dev/null +++ b/internal/systemd/journal_entry_test.go @@ -0,0 +1,218 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalEntryObjectParsesRegularAndCompactItems(t *testing.T) { + for _, compact := range []bool{false, true} { + name := "regular" + if compact { + name = "compact" + } + t.Run(name, func(t *testing.T) { + contents, entryOffset, dataOffset := testJournalEntryContents(compact) + view, err := newJournalFileView(name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + + entry, err := view.entryObjectAt(uint64(entryOffset)) + require.NoError(t, err) + assert.Equal(t, uint64(42), entry.seqnum) + assert.Equal(t, uint64(1_700_000_000_000_000), entry.realtime) + assert.Equal(t, uint64(1234), entry.monotonic) + assert.Equal(t, "11111111111111111111111111111111", entry.bootID.String()) + require.Len(t, entry.items, 1) + assert.Equal(t, uint64(dataOffset), entry.items[0].dataOffset) + if compact { + assert.Zero(t, entry.items[0].hash) + } else { + assert.Equal(t, uint64(0x0102030405060708), entry.items[0].hash) + } + }) + } +} + +func TestJournalEntryOffsetIteratorReadsNewestFirst(t *testing.T) { + for _, compact := range []bool{false, true} { + name := "regular" + if compact { + name = "compact" + } + t.Run(name, func(t *testing.T) { + contents, dataOffset, entryOffsets := testJournalEntryArrayContents(compact) + view, err := newJournalFileView(name+"-array.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(dataOffset)) + require.NoError(t, err) + iterator, err := view.entryOffsetsForData(data) + require.NoError(t, err) + + var actual []uint64 + for { + offset, found, err := iterator.previous() + require.NoError(t, err) + if !found { + break + } + actual = append(actual, offset) + } + expected := make([]uint64, len(entryOffsets)) + for index := range entryOffsets { + expected[index] = uint64(entryOffsets[len(entryOffsets)-1-index]) + } + assert.Equal(t, expected, actual) + }) + } +} + +func TestJournalEntryOffsetIteratorHandlesUnreferencedData(t *testing.T) { + contents, dataOffset := testJournalDataContents(t, []byte("MESSAGE=unused"), 0, false) + view, err := newJournalFileView("unused.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(dataOffset)) + require.NoError(t, err) + iterator, err := view.entryOffsetsForData(data) + require.NoError(t, err) + + _, found, err := iterator.previous() + require.NoError(t, err) + assert.False(t, found) +} + +func TestJournalEntryOffsetIteratorRejectsUnsortedItems(t *testing.T) { + contents, dataOffset, _ := testJournalEntryArrayContents(false) + firstArrayOffset := binary.LittleEndian.Uint64(contents[dataOffset+48 : dataOffset+56]) + firstItem := firstArrayOffset + journalEntryArrayHeaderSize + first := binary.LittleEndian.Uint64(contents[firstItem : firstItem+8]) + binary.LittleEndian.PutUint64(contents[firstItem+8:firstItem+16], first-8) + + view, err := newJournalFileView("unsorted.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(dataOffset)) + require.NoError(t, err) + iterator, err := view.entryOffsetsForData(data) + require.NoError(t, err) + + for { + _, found, err := iterator.previous() + if err != nil { + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "not strictly increasing") + break + } + require.True(t, found) + } +} + +func testJournalEntryContents(compact bool) ([]byte, int, int) { + headerFlags := uint32(0) + dataHeaderSize := journalDataRegularHeaderSize + itemSize := journalEntryRegularItemSize + if compact { + headerFlags = journalHeaderIncompatibleCompact + dataHeaderSize = journalDataCompactHeaderSize + itemSize = journalEntryCompactItemSize + } + dataOffset := journalHeaderCurrentSize + entryOffset := alignJournalTestSize(dataOffset + dataHeaderSize) + fileSize := alignJournalTestSize(entryOffset + journalEntryHeaderSize + itemSize) + contents := testJournalContents(fileSize, 0, headerFlags) + binary.LittleEndian.PutUint64(contents[136:144], uint64(entryOffset)) + binary.LittleEndian.PutUint64(contents[152:160], 1) + + data := contents[dataOffset:] + data[0] = journalObjectData + binary.LittleEndian.PutUint64(data[8:16], uint64(dataHeaderSize)) + + entry := contents[entryOffset:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], uint64(journalEntryHeaderSize+itemSize)) + binary.LittleEndian.PutUint64(entry[16:24], 42) + binary.LittleEndian.PutUint64(entry[24:32], 1_700_000_000_000_000) + binary.LittleEndian.PutUint64(entry[32:40], 1234) + copy(entry[40:56], bytes.Repeat([]byte{0x11}, 16)) + if compact { + binary.LittleEndian.PutUint32(entry[64:68], uint32(dataOffset)) + } else { + binary.LittleEndian.PutUint64(entry[64:72], uint64(dataOffset)) + binary.LittleEndian.PutUint64(entry[72:80], 0x0102030405060708) + } + return contents, entryOffset, dataOffset +} + +func testJournalEntryArrayContents(compact bool) ([]byte, int, []int) { + headerFlags := uint32(0) + dataHeaderSize := journalDataRegularHeaderSize + arrayItemSize := journalEntryArrayRegularItemSize + if compact { + headerFlags = journalHeaderIncompatibleCompact + dataHeaderSize = journalDataCompactHeaderSize + arrayItemSize = journalEntryCompactItemSize + } + dataOffset := journalHeaderCurrentSize + nextOffset := alignJournalTestSize(dataOffset + dataHeaderSize) + entryOffsets := make([]int, 6) + for index := range entryOffsets { + entryOffsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalEntryHeaderSize) + } + firstArrayOffset := nextOffset + firstArrayCapacity := 4 + nextOffset = alignJournalTestSize(nextOffset + journalEntryArrayHeaderSize + firstArrayCapacity*arrayItemSize) + secondArrayOffset := nextOffset + secondArrayCapacity := 4 + nextOffset = alignJournalTestSize(nextOffset + journalEntryArrayHeaderSize + secondArrayCapacity*arrayItemSize) + + contents := testJournalContents(nextOffset, 0, headerFlags) + binary.LittleEndian.PutUint64(contents[136:144], uint64(secondArrayOffset)) + binary.LittleEndian.PutUint64(contents[144:152], 9) + binary.LittleEndian.PutUint64(contents[152:160], uint64(len(entryOffsets))) + data := contents[dataOffset:] + data[0] = journalObjectData + binary.LittleEndian.PutUint64(data[8:16], uint64(dataHeaderSize)) + binary.LittleEndian.PutUint64(data[40:48], uint64(entryOffsets[0])) + binary.LittleEndian.PutUint64(data[48:56], uint64(firstArrayOffset)) + binary.LittleEndian.PutUint64(data[56:64], uint64(len(entryOffsets))) + if compact { + binary.LittleEndian.PutUint32(data[64:68], uint32(secondArrayOffset)) + binary.LittleEndian.PutUint32(data[68:72], 1) + } + for _, offset := range entryOffsets { + entry := contents[offset:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], journalEntryHeaderSize) + } + + firstArray := contents[firstArrayOffset:] + firstArray[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(firstArray[8:16], uint64(journalEntryArrayHeaderSize+firstArrayCapacity*arrayItemSize)) + binary.LittleEndian.PutUint64(firstArray[16:24], uint64(secondArrayOffset)) + secondArray := contents[secondArrayOffset:] + secondArray[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(secondArray[8:16], uint64(journalEntryArrayHeaderSize+secondArrayCapacity*arrayItemSize)) + for index, offset := range entryOffsets[1:] { + array := firstArray + itemIndex := index + if index >= firstArrayCapacity { + array = secondArray + itemIndex -= firstArrayCapacity + } + position := journalEntryArrayHeaderSize + itemIndex*arrayItemSize + if compact { + binary.LittleEndian.PutUint32(array[position:position+4], uint32(offset)) + } else { + binary.LittleEndian.PutUint64(array[position:position+8], uint64(offset)) + } + } + return contents, dataOffset, entryOffsets +} From 14eb49bd5522050f0eb58de209e3e9584d11de67 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 13:51:09 -0400 Subject: [PATCH 25/53] feat(systemd): evaluate restricted journal queries --- internal/systemd/journal_format.go | 4 + internal/systemd/journal_query_file.go | 367 +++++++++++++++++++ internal/systemd/journal_query_file_test.go | 381 ++++++++++++++++++++ 3 files changed, 752 insertions(+) create mode 100644 internal/systemd/journal_query_file.go create mode 100644 internal/systemd/journal_query_file_test.go diff --git a/internal/systemd/journal_format.go b/internal/systemd/journal_format.go index 19ced3b1..77f63e0d 100644 --- a/internal/systemd/journal_format.go +++ b/internal/systemd/journal_format.go @@ -63,6 +63,10 @@ func (id journalID) String() string { return hex.EncodeToString(id[:]) } +func (id journalID) zero() bool { + return id == journalID{} +} + type journalHeader struct { compatibleFlags uint32 incompatibleFlags uint32 diff --git a/internal/systemd/journal_query_file.go b/internal/systemd/journal_query_file.go new file mode 100644 index 00000000..4cbd9f49 --- /dev/null +++ b/internal/systemd/journal_query_file.go @@ -0,0 +1,367 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "fmt" + "math" + "strings" + "time" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalCandidatesScanned = 100_000 + maxJournalFieldNameSize = 64 +) + +type journalQueryEntry struct { + fileID journalID + seqnumID journalID + bootID journalID + seqnum uint64 + realtime uint64 + monotonic uint64 + offset uint64 + selected builtins.JournalEntry +} + +type journalDataCandidateSource struct { + iterator *journalEntryOffsetIterator + index journalDataObject + required []journalDataObject + current *journalEntryObject + done bool +} + +func (s *journalDataCandidateSource) load(ctx context.Context, scanned *int) error { + if s.current != nil || s.done { + return nil + } + for { + if err := ctx.Err(); err != nil { + return err + } + offset, found, err := s.iterator.previous() + if err != nil { + return err + } + if !found { + s.done = true + return nil + } + if *scanned >= maxJournalCandidatesScanned { + return journalLimit(s.iterator.file.name, offset, "query examines more than %d indexed candidates", maxJournalCandidatesScanned) + } + (*scanned)++ + + entry, err := s.iterator.file.entryObjectAt(offset) + if err != nil { + return err + } + indexed, err := journalEntryContainsData(s.iterator.file, entry, s.index) + if err != nil { + return err + } + if !indexed { + return journalCorrupt(s.iterator.file.name, offset, "DATA entry index points to an ENTRY that does not reference the DATA object") + } + matches := true + for _, required := range s.required { + contains, err := journalEntryContainsData(s.iterator.file, entry, required) + if err != nil { + return err + } + if !contains { + matches = false + break + } + } + if matches { + s.current = &entry + return nil + } + } +} + +func journalEntryContainsData(file *journalFileView, entry journalEntryObject, data journalDataObject) (bool, error) { + for index, item := range entry.items { + if item.dataOffset != data.offset { + continue + } + if !file.header.compact() && item.hash != data.hash { + return false, journalCorrupt(file.name, entry.offset, "ENTRY item %d hash does not match its DATA object", index) + } + return true, nil + } + return false, nil +} + +type journalFileQueryIterator struct { + file *journalFileView + sources []*journalDataCandidateSource + bootID journalID + filterBoot bool + sinceUsec uint64 + scanned int +} + +func newJournalFileQueryIterator(file *journalFileView, query builtins.JournalQuery, currentBoot *journalID) (*journalFileQueryIterator, error) { + if err := validateJournalQuery(query); err != nil { + return nil, err + } + iterator := &journalFileQueryIterator{file: file} + if query.CurrentBoot { + if currentBoot == nil { + return nil, fmt.Errorf("current journal boot is not available") + } + iterator.bootID = *currentBoot + iterator.filterBoot = true + } + if !query.Since.IsZero() && query.Since.UnixMicro() > 0 { + iterator.sinceUsec = uint64(query.Since.UnixMicro()) + } + + if query.Kernel { + data, found, err := file.findDataObject([]byte("_TRANSPORT=kernel")) + if err != nil { + return nil, err + } + if found { + if err := iterator.addSource(data); err != nil { + return nil, err + } + } + return iterator, nil + } + + pidOne, pidOneFound, err := file.findDataObject([]byte("_PID=1")) + if err != nil { + return nil, err + } + seenUnits := make(map[string]struct{}, len(query.Units)) + for _, unit := range query.Units { + if _, exists := seenUnits[unit]; exists { + continue + } + seenUnits[unit] = struct{}{} + direct, found, err := file.findDataObject([]byte("_SYSTEMD_UNIT=" + unit)) + if err != nil { + return nil, err + } + if found { + if err := iterator.addSource(direct); err != nil { + return nil, err + } + } + + manager, found, err := file.findDataObject([]byte("UNIT=" + unit)) + if err != nil { + return nil, err + } + if found && pidOneFound { + if err := iterator.addSource(manager, pidOne); err != nil { + return nil, err + } + } + } + return iterator, nil +} + +func (i *journalFileQueryIterator) addSource(index journalDataObject, required ...journalDataObject) error { + offsets, err := i.file.entryOffsetsForData(index) + if err != nil { + return err + } + i.sources = append(i.sources, &journalDataCandidateSource{iterator: offsets, index: index, required: required}) + return nil +} + +func (i *journalFileQueryIterator) previous(ctx context.Context) (journalQueryEntry, bool, error) { + for { + entry, found, err := i.previousCandidate(ctx) + if err != nil || !found { + return journalQueryEntry{}, found, err + } + if i.filterBoot && entry.bootID != i.bootID { + continue + } + if i.sinceUsec != 0 && entry.realtime < i.sinceUsec { + continue + } + if entry.seqnum == 0 { + return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY sequence number is zero") + } + if entry.bootID.zero() { + return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY boot ID is zero") + } + if entry.realtime == 0 || entry.realtime > math.MaxInt64 { + return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY realtime timestamp %d is invalid", entry.realtime) + } + + selected, err := i.file.materializeJournalEntry(ctx, entry) + if err != nil { + return journalQueryEntry{}, false, err + } + return journalQueryEntry{ + fileID: i.file.header.fileID, + seqnumID: i.file.header.seqnumID, + bootID: entry.bootID, + seqnum: entry.seqnum, + realtime: entry.realtime, + monotonic: entry.monotonic, + offset: entry.offset, + selected: selected, + }, true, nil + } +} + +func (i *journalFileQueryIterator) previousCandidate(ctx context.Context) (journalEntryObject, bool, error) { + var newest *journalEntryObject + for _, source := range i.sources { + if err := source.load(ctx, &i.scanned); err != nil { + return journalEntryObject{}, false, err + } + if source.current != nil && (newest == nil || source.current.offset > newest.offset) { + newest = source.current + } + } + if newest == nil { + return journalEntryObject{}, false, nil + } + + selected := *newest + for _, source := range i.sources { + if source.current != nil && source.current.offset == selected.offset { + source.current = nil + } + } + return selected, true, nil +} + +func (f *journalFileView) materializeJournalEntry(ctx context.Context, entry journalEntryObject) (builtins.JournalEntry, error) { + selected := builtins.JournalEntry{ + Timestamp: time.UnixMicro(int64(entry.realtime)), + } + var syslogIdentifier, command string + var haveHostname, haveSyslogIdentifier, haveCommand, havePID, haveMessage bool + seen := make(map[uint64]struct{}, len(entry.items)) + + for index, item := range entry.items { + if err := ctx.Err(); err != nil { + return builtins.JournalEntry{}, err + } + if _, exists := seen[item.dataOffset]; exists { + return builtins.JournalEntry{}, journalCorrupt(f.name, entry.offset, "ENTRY contains duplicate DATA offset %d", item.dataOffset) + } + seen[item.dataOffset] = struct{}{} + + data, err := f.dataObjectAt(item.dataOffset) + if err != nil { + return builtins.JournalEntry{}, err + } + if !f.header.compact() && item.hash != data.hash { + return builtins.JournalEntry{}, journalCorrupt(f.name, entry.offset, "ENTRY item %d hash does not match its DATA object", index) + } + field, value, selectedField, err := f.selectedDataField(data) + if err != nil { + return builtins.JournalEntry{}, err + } + if !selectedField { + continue + } + switch field { + case "_HOSTNAME": + if !haveHostname { + selected.Hostname = value + haveHostname = true + } + case "SYSLOG_IDENTIFIER": + if !haveSyslogIdentifier { + syslogIdentifier = value + haveSyslogIdentifier = true + } + case "_COMM": + if !haveCommand { + command = value + haveCommand = true + } + case "_PID": + if !havePID { + selected.PID = value + havePID = true + } + case "MESSAGE": + if !haveMessage { + selected.Message = value + haveMessage = true + } + } + } + selected.Identifier = syslogIdentifier + if selected.Identifier == "" { + selected.Identifier = command + } + return selected, nil +} + +func (f *journalFileView) selectedDataField(data journalDataObject) (string, string, bool, error) { + prefix, _, err := f.readDataPayload(data, maxJournalFieldNameSize+1) + if err != nil { + return "", "", false, err + } + separator := bytes.IndexByte(prefix, '=') + if separator <= 0 || separator > maxJournalFieldNameSize { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload has an invalid field name") + } + fieldBytes := prefix[:separator] + if !validJournalFieldName(fieldBytes) { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload has an invalid field name") + } + field := string(fieldBytes) + if !selectedJournalField(field) { + return field, "", false, nil + } + + limit := separator + 1 + maxJournalFieldSize + payload, truncated, err := f.readDataPayload(data, limit) + if err != nil { + return "", "", false, err + } + fieldPrefix := field + "=" + if !strings.HasPrefix(string(payload), fieldPrefix) { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload changed while being read") + } + if !truncated && f.dataHash(payload) != data.hash { + return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload hash does not match its object") + } + return field, string(payload[len(fieldPrefix):]), true, nil +} + +func validJournalFieldName(field []byte) bool { + if len(field) == 0 || len(field) > maxJournalFieldNameSize || field[0] >= '0' && field[0] <= '9' { + return false + } + for _, character := range field { + if character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '_' { + continue + } + return false + } + return true +} + +func selectedJournalField(field string) bool { + switch field { + case "_HOSTNAME", "SYSLOG_IDENTIFIER", "_COMM", "_PID", "MESSAGE": + return true + default: + return false + } +} diff --git a/internal/systemd/journal_query_file_test.go b/internal/systemd/journal_query_file_test.go new file mode 100644 index 00000000..3dfd66c5 --- /dev/null +++ b/internal/systemd/journal_query_file_test.go @@ -0,0 +1,381 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +func TestJournalFileQuerySelectsDirectAndManagerUnitEntries(t *testing.T) { + oldBoot := repeatedJournalID(0x11) + currentBoot := repeatedJournalID(0x22) + start := time.Unix(1_700_000_000, 0) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + { + bootID: oldBoot, + realtime: uint64(start.UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=api.service", "_HOSTNAME=node", "_COMM=api", "_PID=10", "MESSAGE=old boot"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(time.Second).UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=worker.service", "MESSAGE=worker"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(2 * time.Second).UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=api.service", "_HOSTNAME=node", "SYSLOG_IDENTIFIER=api", "_PID=20", "MESSAGE=ready"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(3 * time.Second).UnixMicro()), + fields: []string{"UNIT=api.service", "_PID=1", "SYSLOG_IDENTIFIER=systemd", "MESSAGE=stopped"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(4 * time.Second).UnixMicro()), + fields: []string{"UNIT=api.service", "_PID=22", "MESSAGE=untrusted unit field"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(5 * time.Second).UnixMicro()), + fields: []string{"_SYSTEMD_UNIT=api.service", "UNIT=api.service", "_PID=1", "_COMM=systemd", "MESSAGE=deduplicated"}, + }, + { + bootID: currentBoot, + realtime: uint64(start.Add(6 * time.Second).UnixMicro()), + fields: []string{"_TRANSPORT=kernel", "_HOSTNAME=node", "_COMM=kernel", "MESSAGE=kernel message"}, + }, + }) + view, err := newJournalFileView("query.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"api.service"}, + CurrentBoot: true, + Since: start.Add(2 * time.Second), + MaxEntries: 100, + }, ¤tBoot) + require.NoError(t, err) + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 3) + assert.Equal(t, []string{"deduplicated", "stopped", "ready"}, []string{ + entries[0].selected.Message, + entries[1].selected.Message, + entries[2].selected.Message, + }) + assert.Equal(t, "systemd", entries[0].selected.Identifier) + assert.Equal(t, "systemd", entries[1].selected.Identifier) + assert.Equal(t, "api", entries[2].selected.Identifier) + assert.Equal(t, "node", entries[2].selected.Hostname) + assert.Equal(t, "20", entries[2].selected.PID) + assert.Equal(t, start.Add(2*time.Second), entries[2].selected.Timestamp) +} + +func TestJournalFileQuerySelectsKernelEntries(t *testing.T) { + bootID := repeatedJournalID(0x33) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=service"}}, + {bootID: bootID, realtime: 1_700_000_000_000_001, fields: []string{"_TRANSPORT=kernel", "_COMM=kernel", "MESSAGE=kernel"}}, + }) + view, err := newJournalFileView("kernel.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Kernel: true, + CurrentBoot: true, + MaxEntries: 10, + }, &bootID) + require.NoError(t, err) + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 1) + assert.Equal(t, "kernel", entries[0].selected.Message) + assert.Equal(t, "kernel", entries[0].selected.Identifier) +} + +func TestJournalFileQuerySupportsCompactKeyedLayout(t *testing.T) { + bootID := repeatedJournalID(0x77) + contents := buildJournalQueryFixtureWithLayout(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=first"}}, + {bootID: bootID, realtime: 1_700_000_000_000_001, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=second"}}, + }, true) + view, err := newJournalFileView("compact.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 10}, nil) + require.NoError(t, err) + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 2) + assert.Equal(t, "second", entries[0].selected.Message) + assert.Equal(t, "first", entries[1].selected.Message) +} + +func TestJournalFileQueryHonorsCancellation(t *testing.T) { + bootID := repeatedJournalID(0x44) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=service"}}, + }) + view, err := newJournalFileView("cancel.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 1}, nil) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, _, err = iterator.previous(ctx) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestJournalFileQueryRejectsCandidateMissingIndexedData(t *testing.T) { + bootID := repeatedJournalID(0x66) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=service"}}, + }) + initial, err := newJournalFileView("broken-index.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + selector, found, err := initial.findDataObject([]byte("_SYSTEMD_UNIT=api.service")) + require.NoError(t, err) + require.True(t, found) + message, found, err := initial.findDataObject([]byte("MESSAGE=service")) + require.NoError(t, err) + require.True(t, found) + entry, err := initial.entryObjectAt(selector.entryOffset) + require.NoError(t, err) + for index, item := range entry.items { + if item.dataOffset != selector.offset { + continue + } + position := int(entry.offset) + journalEntryHeaderSize + index*journalEntryRegularItemSize + binary.LittleEndian.PutUint64(contents[position:position+8], message.offset) + binary.LittleEndian.PutUint64(contents[position+8:position+16], message.hash) + } + + view, err := newJournalFileView("broken-index.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 1}, nil) + require.NoError(t, err) + _, _, err = iterator.previous(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) + assert.Contains(t, err.Error(), "does not reference the DATA object") +} + +func collectJournalQueryEntries(t *testing.T, iterator *journalFileQueryIterator) []journalQueryEntry { + t.Helper() + var entries []journalQueryEntry + for { + entry, found, err := iterator.previous(context.Background()) + require.NoError(t, err) + if !found { + return entries + } + entries = append(entries, entry) + } +} + +type journalQueryFixtureEntry struct { + bootID journalID + realtime uint64 + fields []string +} + +type journalQueryFixtureData struct { + payload []byte + hash uint64 + offset int + references []int + arrayOffset int +} + +func buildJournalQueryFixture(t *testing.T, entries []journalQueryFixtureEntry) []byte { + t.Helper() + return buildJournalQueryFixtureWithLayout(t, entries, false) +} + +func buildJournalQueryFixtureWithLayout(t *testing.T, entries []journalQueryFixtureEntry, compact bool) []byte { + t.Helper() + require.NotEmpty(t, entries) + headerFlags := uint32(0) + dataHeaderSize := journalDataRegularHeaderSize + entryItemSize := journalEntryRegularItemSize + arrayItemSize := journalEntryArrayRegularItemSize + var fileID journalID + for index := range fileID { + fileID[index] = byte(index) + } + if compact { + headerFlags = journalHeaderIncompatibleCompact | journalHeaderIncompatibleKeyedHash + dataHeaderSize = journalDataCompactHeaderSize + entryItemSize = journalEntryCompactItemSize + arrayItemSize = journalEntryCompactItemSize + } + + dataByValue := make(map[string]*journalQueryFixtureData) + var dataObjects []*journalQueryFixtureData + for entryIndex, entry := range entries { + require.NotEmpty(t, entry.fields) + for _, field := range entry.fields { + data := dataByValue[field] + if data == nil { + payload := []byte(field) + hash := journalJenkinsHash64(payload) + if compact { + hash = journalSipHash24(payload, fileID) + } + data = &journalQueryFixtureData{payload: payload, hash: hash} + dataByValue[field] = data + dataObjects = append(dataObjects, data) + } + data.references = append(data.references, entryIndex) + } + } + + const bucketCount = 64 + tableOffset := journalHeaderCurrentSize + tableObjectSize := journalObjectHeaderSize + bucketCount*journalHashItemSize + nextOffset := alignJournalTestSize(tableOffset + tableObjectSize) + for _, data := range dataObjects { + data.offset = nextOffset + nextOffset = alignJournalTestSize(nextOffset + dataHeaderSize + len(data.payload)) + } + entryOffsets := make([]int, len(entries)) + for index, entry := range entries { + entryOffsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalEntryHeaderSize + len(entry.fields)*entryItemSize) + } + arrayCount := 0 + for _, data := range dataObjects { + if len(data.references) <= 1 { + continue + } + data.arrayOffset = nextOffset + capacity := len(data.references) - 1 + if capacity < 4 { + capacity = 4 + } + nextOffset = alignJournalTestSize(nextOffset + journalEntryArrayHeaderSize + capacity*arrayItemSize) + arrayCount++ + } + + contents := testJournalContents(nextOffset, 0, headerFlags) + copy(contents[24:40], fileID[:]) + copy(contents[72:88], bytes.Repeat([]byte{0x55}, 16)) + binary.LittleEndian.PutUint64(contents[104:112], uint64(tableOffset+journalObjectHeaderSize)) + binary.LittleEndian.PutUint64(contents[112:120], bucketCount*journalHashItemSize) + binary.LittleEndian.PutUint64(contents[136:144], uint64(lastJournalFixtureObjectOffset(dataObjects, entryOffsets))) + binary.LittleEndian.PutUint64(contents[144:152], uint64(1+len(dataObjects)+len(entries)+arrayCount)) + binary.LittleEndian.PutUint64(contents[152:160], uint64(len(entries))) + binary.LittleEndian.PutUint64(contents[160:168], uint64(len(entries))) + binary.LittleEndian.PutUint64(contents[168:176], 1) + binary.LittleEndian.PutUint64(contents[184:192], entries[0].realtime) + binary.LittleEndian.PutUint64(contents[192:200], entries[len(entries)-1].realtime) + binary.LittleEndian.PutUint64(contents[264:272], uint64(entryOffsets[len(entryOffsets)-1])) + + table := contents[tableOffset:] + table[0] = journalObjectDataHashTable + binary.LittleEndian.PutUint64(table[8:16], uint64(tableObjectSize)) + buckets := make([][]*journalQueryFixtureData, bucketCount) + for _, data := range dataObjects { + bucket := data.hash % bucketCount + buckets[bucket] = append(buckets[bucket], data) + } + for bucket, chain := range buckets { + if len(chain) == 0 { + continue + } + position := journalObjectHeaderSize + bucket*journalHashItemSize + binary.LittleEndian.PutUint64(table[position:position+8], uint64(chain[0].offset)) + binary.LittleEndian.PutUint64(table[position+8:position+16], uint64(chain[len(chain)-1].offset)) + for index := 0; index+1 < len(chain); index++ { + next := chain[index+1].offset + binary.LittleEndian.PutUint64(contents[chain[index].offset+24:chain[index].offset+32], uint64(next)) + } + } + + for _, data := range dataObjects { + object := contents[data.offset:] + object[0] = journalObjectData + binary.LittleEndian.PutUint64(object[8:16], uint64(dataHeaderSize+len(data.payload))) + binary.LittleEndian.PutUint64(object[16:24], data.hash) + binary.LittleEndian.PutUint64(object[40:48], uint64(entryOffsets[data.references[0]])) + binary.LittleEndian.PutUint64(object[48:56], uint64(data.arrayOffset)) + binary.LittleEndian.PutUint64(object[56:64], uint64(len(data.references))) + copy(object[dataHeaderSize:], data.payload) + + if data.arrayOffset != 0 { + capacity := len(data.references) - 1 + if capacity < 4 { + capacity = 4 + } + array := contents[data.arrayOffset:] + array[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(array[8:16], uint64(journalEntryArrayHeaderSize+capacity*arrayItemSize)) + if compact { + binary.LittleEndian.PutUint32(object[64:68], uint32(data.arrayOffset)) + binary.LittleEndian.PutUint32(object[68:72], uint32(len(data.references)-1)) + } + for index, entryIndex := range data.references[1:] { + position := journalEntryArrayHeaderSize + index*arrayItemSize + if compact { + binary.LittleEndian.PutUint32(array[position:position+4], uint32(entryOffsets[entryIndex])) + } else { + binary.LittleEndian.PutUint64(array[position:position+8], uint64(entryOffsets[entryIndex])) + } + } + } + } + + for index, entrySpec := range entries { + entry := contents[entryOffsets[index]:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], uint64(journalEntryHeaderSize+len(entrySpec.fields)*entryItemSize)) + binary.LittleEndian.PutUint64(entry[16:24], uint64(index+1)) + binary.LittleEndian.PutUint64(entry[24:32], entrySpec.realtime) + binary.LittleEndian.PutUint64(entry[32:40], uint64(index+1)*100) + copy(entry[40:56], entrySpec.bootID[:]) + var xorHash uint64 + for itemIndex, field := range entrySpec.fields { + data := dataByValue[field] + position := journalEntryHeaderSize + itemIndex*entryItemSize + if compact { + binary.LittleEndian.PutUint32(entry[position:position+4], uint32(data.offset)) + } else { + binary.LittleEndian.PutUint64(entry[position:position+8], uint64(data.offset)) + binary.LittleEndian.PutUint64(entry[position+8:position+16], data.hash) + } + xorHash ^= journalJenkinsHash64(data.payload) + } + binary.LittleEndian.PutUint64(entry[56:64], xorHash) + } + return contents +} + +func lastJournalFixtureObjectOffset(data []*journalQueryFixtureData, entryOffsets []int) int { + last := entryOffsets[len(entryOffsets)-1] + for _, object := range data { + if object.arrayOffset > last { + last = object.arrayOffset + } + } + return last +} + +func repeatedJournalID(value byte) journalID { + var id journalID + for index := range id { + id[index] = value + } + return id +} From 825deb25e88ee53cbec3fd0c70da323302b9b8fa Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:02:59 -0400 Subject: [PATCH 26/53] feat(systemd): snapshot journal file sets --- internal/systemd/journal_snapshot.go | 207 ++++++++++++++++++++++ internal/systemd/journal_snapshot_test.go | 78 ++++++++ 2 files changed, 285 insertions(+) create mode 100644 internal/systemd/journal_snapshot.go create mode 100644 internal/systemd/journal_snapshot_test.go diff --git a/internal/systemd/journal_snapshot.go b/internal/systemd/journal_snapshot.go new file mode 100644 index 00000000..13eedf55 --- /dev/null +++ b/internal/systemd/journal_snapshot.go @@ -0,0 +1,207 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" +) + +const maxJournalQueryFiles = 128 + +var errJournalChanged = errors.New("systemd journal changed while being read") + +type journalSnapshotFile struct { + path string + file *os.File + info fs.FileInfo + view *journalFileView +} + +type journalSnapshot struct { + machineID journalID + machineIDText string + paths []string + files []*journalSnapshotFile +} + +func (c *Client) openJournalSnapshot() (*journalSnapshot, error) { + machineIDText, paths, err := c.journalFiles() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal files changed during discovery") + } + return nil, err + } + if len(paths) == 0 { + return nil, fmt.Errorf("no journal files found for machine %s", machineIDText) + } + if len(paths) > maxJournalQueryFiles { + return nil, fmt.Errorf("%w: journal query opens %d files; maximum is %d", errJournalLimit, len(paths), maxJournalQueryFiles) + } + machineID, err := parseJournalID(machineIDText) + if err != nil { + return nil, err + } + + snapshot := &journalSnapshot{ + machineID: machineID, + machineIDText: machineIDText, + paths: append([]string(nil), paths...), + files: make([]*journalSnapshotFile, 0, len(paths)), + } + fileIDs := make(map[journalID]string, len(paths)) + for _, path := range paths { + opened, err := openJournalSnapshotFile(path) + if err != nil { + snapshot.close() + return nil, err + } + if opened.view.header.machineID != machineID { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 40, "journal machine ID %s does not match configured machine %s", opened.view.header.machineID, machineID) + } + if opened.view.header.fileID.zero() { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 24, "journal file ID is zero") + } + if opened.view.header.seqnumID.zero() { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 72, "journal sequence ID is zero") + } + if previous, exists := fileIDs[opened.view.header.fileID]; exists { + opened.file.Close() + snapshot.close() + return nil, journalCorrupt(path, 24, "journal file ID duplicates %q", previous) + } + fileIDs[opened.view.header.fileID] = path + snapshot.files = append(snapshot.files, opened) + } + if err := snapshot.stable(c); err != nil { + snapshot.close() + return nil, err + } + return snapshot, nil +} + +func openJournalSnapshotFile(path string) (*journalSnapshotFile, error) { + before, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal file %q disappeared before open", path) + } + return nil, fmt.Errorf("inspect journal file %q before open: %w", path, err) + } + if before.Mode()&fs.ModeSymlink != 0 || !before.Mode().IsRegular() { + return nil, journalChanged("journal file %q is no longer a regular non-symlink file", path) + } + + file, err := os.Open(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal file %q disappeared during open", path) + } + return nil, fmt.Errorf("open journal file %q: %w", path, err) + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, fmt.Errorf("inspect open journal file %q: %w", path, err) + } + after, err := os.Lstat(path) + if err != nil { + file.Close() + if errors.Is(err, fs.ErrNotExist) { + return nil, journalChanged("journal file %q disappeared during open", path) + } + return nil, fmt.Errorf("inspect journal file %q after open: %w", path, err) + } + if !info.Mode().IsRegular() || after.Mode()&fs.ModeSymlink != 0 || !after.Mode().IsRegular() || !os.SameFile(before, info) || !os.SameFile(after, info) { + file.Close() + return nil, journalChanged("journal file %q was replaced during open", path) + } + if info.Size() < 0 { + file.Close() + return nil, journalCorrupt(path, 0, "journal file has a negative size") + } + view, err := newJournalFileView(path, file, uint64(info.Size())) + if err != nil { + file.Close() + return nil, err + } + return &journalSnapshotFile{path: path, file: file, info: info, view: view}, nil +} + +func (s *journalSnapshot) stable(client *Client) error { + machineID, paths, err := client.journalFiles() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return journalChanged("journal files changed during snapshot verification") + } + return err + } + if machineID != s.machineIDText || !sameJournalPaths(paths, s.paths) { + return journalChanged("journal file set changed during snapshot") + } + for _, opened := range s.files { + current, err := os.Lstat(opened.path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return journalChanged("journal file %q disappeared during snapshot", opened.path) + } + return fmt.Errorf("verify journal file %q: %w", opened.path, err) + } + if current.Mode()&fs.ModeSymlink != 0 || !current.Mode().IsRegular() || !os.SameFile(current, opened.info) { + return journalChanged("journal file %q was replaced during snapshot", opened.path) + } + if current.Size() < opened.info.Size() { + return journalChanged("journal file %q shrank during snapshot", opened.path) + } + } + return nil +} + +func (s *journalSnapshot) close() error { + var result error + for _, opened := range s.files { + if err := opened.file.Close(); err != nil { + result = errors.Join(result, fmt.Errorf("close journal file %q: %w", opened.path, err)) + } + } + return result +} + +func sameJournalPaths(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +func parseJournalID(value string) (journalID, error) { + var id journalID + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != len(id) { + return journalID{}, fmt.Errorf("invalid 128-bit journal ID %q", value) + } + copy(id[:], decoded) + return id, nil +} + +func journalChanged(format string, arguments ...any) error { + return fmt.Errorf("%w: %s", errJournalChanged, fmt.Sprintf(format, arguments...)) +} diff --git a/internal/systemd/journal_snapshot_test.go b/internal/systemd/journal_snapshot_test.go new file mode 100644 index 00000000..b0b31d66 --- /dev/null +++ b/internal/systemd/journal_snapshot_test.go @@ -0,0 +1,78 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJournalSnapshotDetectsFileReplacement(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not permit replacing this open fixture consistently") + } + + client, journalDir, machineID := newJournalSnapshotTestClient(t) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: repeatedJournalID(0x22), realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=ready"}}, + }) + path := writeJournalSnapshotFixture(t, journalDir, "system.journal", contents, machineID, repeatedJournalID(0x31), repeatedJournalID(0x44)) + replacement, err := os.ReadFile(path) + require.NoError(t, err) + + snapshot, err := client.openJournalSnapshot() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, snapshot.close()) }) + require.NoError(t, os.Rename(path, path+".replaced")) + require.NoError(t, os.WriteFile(path, replacement, 0o600)) + + err = snapshot.stable(client) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalChanged) + assert.Contains(t, err.Error(), "replaced during snapshot") +} + +func TestOpenJournalSnapshotRejectsTooManyFilesBeforeParsing(t *testing.T) { + client, journalDir, _ := newJournalSnapshotTestClient(t) + for index := 0; index <= maxJournalQueryFiles; index++ { + path := filepath.Join(journalDir, fmt.Sprintf("%03d.journal", index)) + require.NoError(t, os.WriteFile(path, []byte("not a journal"), 0o600)) + } + + _, err := client.openJournalSnapshot() + require.Error(t, err) + assert.ErrorIs(t, err, errJournalLimit) + assert.Contains(t, err.Error(), "maximum is 128") +} + +func newJournalSnapshotTestClient(t *testing.T) (*Client, string, journalID) { + t.Helper() + root := t.TempDir() + machineID := repeatedJournalID(0xa1) + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID.String()+"\n"), 0o600)) + baseDir := filepath.Join(root, "journal") + journalDir := filepath.Join(baseDir, machineID.String()) + require.NoError(t, os.MkdirAll(journalDir, 0o755)) + return NewClient(Target{JournalDirs: []string{baseDir}, MachineIDPath: machineIDPath}), journalDir, machineID +} + +func writeJournalSnapshotFixture(t *testing.T, journalDir, name string, contents []byte, machineID, fileID, sequenceID journalID) string { + t.Helper() + contents = append([]byte(nil), contents...) + copy(contents[24:40], fileID[:]) + copy(contents[40:56], machineID[:]) + copy(contents[72:88], sequenceID[:]) + path := filepath.Join(journalDir, name) + require.NoError(t, os.WriteFile(path, contents, 0o600)) + return path +} From 78c1b92d579e55f7ae7ed4341c827a79de02582f Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:03:10 -0400 Subject: [PATCH 27/53] feat(systemd): merge rotated journal entries --- internal/systemd/journal_merge.go | 298 ++++++++++++++++++++ internal/systemd/journal_merge_test.go | 159 +++++++++++ internal/systemd/journal_query_file.go | 2 + internal/systemd/journal_query_file_test.go | 29 +- 4 files changed, 481 insertions(+), 7 deletions(-) create mode 100644 internal/systemd/journal_merge.go create mode 100644 internal/systemd/journal_merge_test.go diff --git a/internal/systemd/journal_merge.go b/internal/systemd/journal_merge.go new file mode 100644 index 00000000..bbf6f671 --- /dev/null +++ b/internal/systemd/journal_merge.go @@ -0,0 +1,298 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "errors" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalResultBytes = 8 * 1024 * 1024 +) + +type journalMergeSource struct { + iterator *journalFileQueryIterator + current *journalQueryEntry + done bool +} + +func (s *journalMergeSource) load(ctx context.Context) error { + if s.current != nil || s.done { + return nil + } + entry, found, err := s.iterator.previous(ctx) + if err != nil { + return err + } + if !found { + s.done = true + return nil + } + s.current = &entry + return nil +} + +func (c *Client) queryJournalEntries(ctx context.Context, query builtins.JournalQuery) ([]journalQueryEntry, error) { + if err := validateJournalQuery(query); err != nil { + return nil, err + } + if query.MaxEntries == 0 { + return nil, nil + } + + for attempt := 0; attempt < 2; attempt++ { + entries, err := c.queryJournalEntriesOnce(ctx, query) + if err == nil { + return entries, nil + } + if attempt == 0 && (errors.Is(err, errJournalChanged) || errors.Is(err, errJournalCorrupt)) { + continue + } + return nil, err + } + return nil, journalChanged("journal did not stabilize after retry") +} + +func (c *Client) queryJournalEntriesOnce(ctx context.Context, query builtins.JournalQuery) ([]journalQueryEntry, error) { + snapshot, err := c.openJournalSnapshot() + if err != nil { + return nil, err + } + entries, queryErr := queryJournalSnapshot(ctx, snapshot, query) + if queryErr == nil { + queryErr = snapshot.stable(c) + } + closeErr := snapshot.close() + if queryErr != nil { + return nil, queryErr + } + if closeErr != nil { + return nil, closeErr + } + return entries, nil +} + +func queryJournalSnapshot(ctx context.Context, snapshot *journalSnapshot, query builtins.JournalQuery) ([]journalQueryEntry, error) { + var currentBoot *journalID + if query.CurrentBoot { + bootID, err := newestJournalBoot(snapshot) + if err != nil { + return nil, err + } + currentBoot = &bootID + } + + sources := make([]*journalMergeSource, 0, len(snapshot.files)) + for _, opened := range snapshot.files { + iterator, err := newJournalFileQueryIterator(opened.view, query, currentBoot) + if err != nil { + return nil, err + } + sources = append(sources, &journalMergeSource{iterator: iterator}) + } + + entries := make([]journalQueryEntry, 0, query.MaxEntries) + resultBytes := 0 + for len(entries) < query.MaxEntries { + if err := ctx.Err(); err != nil { + return nil, err + } + var newest *journalQueryEntry + for _, source := range sources { + if err := source.load(ctx); err != nil { + return nil, err + } + if source.current != nil && (newest == nil || compareJournalQueryEntries(*source.current, *newest) > 0) { + newest = source.current + } + } + if newest == nil { + break + } + + selected := *newest + for _, source := range sources { + if source.current == nil || !sameJournalEntryLocation(*source.current, selected) { + continue + } + if source.current.selected != selected.selected { + return nil, journalCorrupt(source.iterator.file.name, source.current.offset, "duplicate journal location has conflicting selected fields") + } + source.current = nil + } + + resultBytes += len(selected.selected.Hostname) + len(selected.selected.Identifier) + len(selected.selected.PID) + len(selected.selected.Message) + if resultBytes > maxJournalResultBytes { + return nil, fmt.Errorf("%w: selected journal fields exceed %d bytes", errJournalLimit, maxJournalResultBytes) + } + entries = append(entries, selected) + } + return entries, nil +} + +func newestJournalBoot(snapshot *journalSnapshot) (journalID, error) { + var newest *journalQueryEntry + for _, opened := range snapshot.files { + entry, found, err := newestJournalEntry(opened.view) + if err != nil { + return journalID{}, err + } + if found && (newest == nil || compareJournalQueryEntries(entry, *newest) > 0) { + copy := entry + newest = © + } + } + if newest == nil { + return journalID{}, fmt.Errorf("could not determine the current boot from the selected journal") + } + return newest.bootID, nil +} + +func newestJournalEntry(file *journalFileView) (journalQueryEntry, bool, error) { + if file.header.nEntries == 0 { + return journalQueryEntry{}, false, nil + } + + var offset uint64 + if file.header.hasTailEntryOffset { + if file.header.tailEntryOffset == 0 { + return journalQueryEntry{}, false, journalCorrupt(file.name, 264, "journal has entries but no tail ENTRY offset") + } + offset = file.header.tailEntryOffset + } else { + iterator, err := file.globalEntryOffsets() + if err != nil { + return journalQueryEntry{}, false, err + } + var found bool + offset, found, err = iterator.previous() + if err != nil { + return journalQueryEntry{}, false, err + } + if !found { + return journalQueryEntry{}, false, journalCorrupt(file.name, file.header.entryArrayOffset, "global ENTRY_ARRAY is empty") + } + } + + entry, err := file.entryObjectAt(offset) + if err != nil { + return journalQueryEntry{}, false, err + } + if entry.seqnum == 0 || entry.realtime == 0 || entry.bootID.zero() { + return journalQueryEntry{}, false, journalCorrupt(file.name, entry.offset, "tail ENTRY has invalid sequence, timestamp, or boot metadata") + } + if file.header.tailEntrySeqnum != 0 && file.header.tailEntrySeqnum != entry.seqnum { + return journalQueryEntry{}, false, journalCorrupt(file.name, entry.offset, "tail ENTRY sequence %d does not match header sequence %d", entry.seqnum, file.header.tailEntrySeqnum) + } + return journalQueryEntry{ + fileID: file.header.fileID, + seqnumID: file.header.seqnumID, + bootID: entry.bootID, + seqnum: entry.seqnum, + realtime: entry.realtime, + monotonic: entry.monotonic, + xorHash: entry.xorHash, + offset: entry.offset, + }, true, nil +} + +func (f *journalFileView) globalEntryOffsets() (*journalEntryOffsetIterator, error) { + iterator := &journalEntryOffsetIterator{file: f, segmentIndex: -1} + if f.header.nEntries == 0 { + return iterator, nil + } + if f.header.entryArrayOffset == 0 { + return nil, journalCorrupt(f.name, 176, "journal has entries but no global ENTRY_ARRAY") + } + + remaining := f.header.nEntries + offset := f.header.entryArrayOffset + seen := make(map[uint64]struct{}, maxJournalEntryArrays) + for remaining > 0 { + if len(iterator.segments) >= maxJournalEntryArrays { + return nil, journalLimit(f.name, offset, "global ENTRY_ARRAY chain exceeds %d objects", maxJournalEntryArrays) + } + if _, exists := seen[offset]; exists { + return nil, journalCorrupt(f.name, offset, "global ENTRY_ARRAY chain contains a cycle") + } + seen[offset] = struct{}{} + + array, err := f.entryArrayAt(offset) + if err != nil { + return nil, err + } + count := array.capacity + if count > remaining { + count = remaining + } + iterator.segments = append(iterator.segments, journalEntryArraySegment{array: array, count: count}) + remaining -= count + if remaining == 0 { + if array.nextOffset != 0 { + return nil, journalCorrupt(f.name, array.offset, "global ENTRY_ARRAY continues after all entries") + } + break + } + if array.nextOffset == 0 { + return nil, journalCorrupt(f.name, array.offset, "global ENTRY_ARRAY ends before all entries") + } + offset = array.nextOffset + } + + if f.header.hasTailEntryArray { + last := iterator.segments[len(iterator.segments)-1] + if uint64(f.header.tailEntryArrayOffset) != last.array.offset || uint64(f.header.tailEntryArrayNEntries) != last.count { + return nil, journalCorrupt(f.name, 256, "global tail ENTRY_ARRAY metadata does not match its array chain") + } + } + iterator.segmentIndex = len(iterator.segments) - 1 + iterator.itemIndex = iterator.segments[iterator.segmentIndex].count + return iterator, nil +} + +func compareJournalQueryEntries(left, right journalQueryEntry) int { + if !left.seqnumID.zero() && left.seqnumID == right.seqnumID && left.seqnum != right.seqnum { + return compareUint64(left.seqnum, right.seqnum) + } + if !left.bootID.zero() && left.bootID == right.bootID && left.monotonic != right.monotonic { + return compareUint64(left.monotonic, right.monotonic) + } + if left.realtime != right.realtime { + return compareUint64(left.realtime, right.realtime) + } + if left.xorHash != right.xorHash { + return compareUint64(left.xorHash, right.xorHash) + } + if compared := bytes.Compare(left.fileID[:], right.fileID[:]); compared != 0 { + return compared + } + return compareUint64(left.offset, right.offset) +} + +func compareUint64(left, right uint64) int { + if left < right { + return -1 + } + if left > right { + return 1 + } + return 0 +} + +func sameJournalEntryLocation(left, right journalQueryEntry) bool { + return !left.seqnumID.zero() && + left.seqnumID == right.seqnumID && + left.seqnum == right.seqnum && + left.bootID == right.bootID && + left.realtime == right.realtime && + left.monotonic == right.monotonic && + left.xorHash == right.xorHash +} diff --git a/internal/systemd/journal_merge_test.go b/internal/systemd/journal_merge_test.go new file mode 100644 index 00000000..162d9f1b --- /dev/null +++ b/internal/systemd/journal_merge_test.go @@ -0,0 +1,159 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +func TestQueryJournalEntriesMergesRotatedFilesAndFindsCurrentBoot(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + sequenceID := repeatedJournalID(0x44) + oldBoot := repeatedJournalID(0x11) + currentBoot := repeatedJournalID(0x22) + start := time.Unix(1_700_000_000, 0) + + writeJournalSnapshotFixture(t, journalDir, "system@old.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 1, bootID: oldBoot, realtime: uint64(start.UnixMicro()), monotonic: 900, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=old boot"}}, + {seqnum: 2, bootID: currentBoot, realtime: uint64(start.Add(time.Second).UnixMicro()), monotonic: 100, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=started"}}, + }), machineID, repeatedJournalID(0x31), sequenceID) + writeJournalSnapshotFixture(t, journalDir, "system.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 3, bootID: currentBoot, realtime: uint64(start.Add(2 * time.Second).UnixMicro()), monotonic: 200, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=ready"}}, + {seqnum: 4, bootID: currentBoot, realtime: uint64(start.Add(3 * time.Second).UnixMicro()), monotonic: 300, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=healthy"}}, + }), machineID, repeatedJournalID(0x32), sequenceID) + + entries, err := client.queryJournalEntries(context.Background(), builtins.JournalQuery{ + Units: []string{"api.service"}, + CurrentBoot: true, + MaxEntries: 10, + }) + require.NoError(t, err) + require.Len(t, entries, 3) + assert.Equal(t, []string{"healthy", "ready", "started"}, journalMessages(entries)) +} + +func TestQueryJournalEntriesHandlesDuplicateSequenceNumbers(t *testing.T) { + bootID := repeatedJournalID(0x22) + sequenceID := repeatedJournalID(0x44) + start := time.Unix(1_700_000_000, 0) + + t.Run("different locations remain visible", func(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + writeJournalSnapshotFixture(t, journalDir, "one.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 5, bootID: bootID, realtime: uint64(start.UnixMicro()), monotonic: 100, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=first"}}, + }), machineID, repeatedJournalID(0x31), sequenceID) + writeJournalSnapshotFixture(t, journalDir, "two.journal", buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {seqnum: 5, bootID: bootID, realtime: uint64(start.Add(time.Second).UnixMicro()), monotonic: 200, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=second"}}, + }), machineID, repeatedJournalID(0x32), sequenceID) + + entries, err := client.queryJournalEntries(context.Background(), builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 10}) + require.NoError(t, err) + assert.Equal(t, []string{"second", "first"}, journalMessages(entries)) + }) + + t.Run("same location is deduplicated", func(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + spec := []journalQueryFixtureEntry{ + {seqnum: 5, bootID: bootID, realtime: uint64(start.UnixMicro()), monotonic: 100, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=duplicate"}}, + } + writeJournalSnapshotFixture(t, journalDir, "one.journal", buildJournalQueryFixture(t, spec), machineID, repeatedJournalID(0x31), sequenceID) + writeJournalSnapshotFixture(t, journalDir, "two.journal", buildJournalQueryFixture(t, spec), machineID, repeatedJournalID(0x32), sequenceID) + + entries, err := client.queryJournalEntries(context.Background(), builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 10}) + require.NoError(t, err) + assert.Equal(t, []string{"duplicate"}, journalMessages(entries)) + }) +} + +func TestNewestJournalEntrySupportsLegacyHeader(t *testing.T) { + bootID := repeatedJournalID(0x22) + contents := buildLegacyJournalTailFixture(bootID) + view, err := newJournalFileView("legacy.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + require.False(t, view.header.hasTailEntryOffset) + + entry, found, err := newestJournalEntry(view) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, uint64(3), entry.seqnum) + assert.Equal(t, bootID, entry.bootID) +} + +func journalMessages(entries []journalQueryEntry) []string { + messages := make([]string, len(entries)) + for index, entry := range entries { + messages[index] = entry.selected.Message + } + return messages +} + +func buildLegacyJournalTailFixture(bootID journalID) []byte { + const entryCount = 3 + payload := []byte("MESSAGE=x") + hash := journalJenkinsHash64(payload) + dataOffset := journalHeaderMinSize + dataSize := journalDataRegularHeaderSize + len(payload) + arrayOffset := alignJournalTestSize(dataOffset + dataSize) + arraySize := journalEntryArrayHeaderSize + entryCount*journalEntryArrayRegularItemSize + nextOffset := alignJournalTestSize(arrayOffset + arraySize) + entryOffsets := make([]int, entryCount) + for index := range entryOffsets { + entryOffsets[index] = nextOffset + nextOffset = alignJournalTestSize(nextOffset + journalEntryHeaderSize + journalEntryRegularItemSize) + } + + contents := testJournalContentsWithHeader(nextOffset, journalHeaderMinSize, 0, 0) + fileID := repeatedJournalID(0x31) + sequenceID := repeatedJournalID(0x44) + copy(contents[24:40], fileID[:]) + copy(contents[56:72], bootID[:]) + copy(contents[72:88], sequenceID[:]) + binary.LittleEndian.PutUint64(contents[136:144], uint64(entryOffsets[len(entryOffsets)-1])) + binary.LittleEndian.PutUint64(contents[144:152], 2+entryCount) + binary.LittleEndian.PutUint64(contents[152:160], entryCount) + binary.LittleEndian.PutUint64(contents[160:168], entryCount) + binary.LittleEndian.PutUint64(contents[168:176], 1) + binary.LittleEndian.PutUint64(contents[176:184], uint64(arrayOffset)) + binary.LittleEndian.PutUint64(contents[184:192], 1_700_000_000_000_001) + binary.LittleEndian.PutUint64(contents[192:200], 1_700_000_000_000_003) + + data := contents[dataOffset:] + data[0] = journalObjectData + binary.LittleEndian.PutUint64(data[8:16], uint64(dataSize)) + binary.LittleEndian.PutUint64(data[16:24], hash) + copy(data[journalDataRegularHeaderSize:], payload) + + array := contents[arrayOffset:] + array[0] = journalObjectEntryArray + binary.LittleEndian.PutUint64(array[8:16], uint64(arraySize)) + for index, offset := range entryOffsets { + position := journalEntryArrayHeaderSize + index*journalEntryArrayRegularItemSize + binary.LittleEndian.PutUint64(array[position:position+8], uint64(offset)) + } + + for index, offset := range entryOffsets { + entry := contents[offset:] + entry[0] = journalObjectEntry + binary.LittleEndian.PutUint64(entry[8:16], journalEntryHeaderSize+journalEntryRegularItemSize) + binary.LittleEndian.PutUint64(entry[16:24], uint64(index+1)) + binary.LittleEndian.PutUint64(entry[24:32], uint64(1_700_000_000_000_001+index)) + binary.LittleEndian.PutUint64(entry[32:40], uint64(100+index)) + copy(entry[40:56], bootID[:]) + binary.LittleEndian.PutUint64(entry[56:64], hash) + binary.LittleEndian.PutUint64(entry[64:72], uint64(dataOffset)) + binary.LittleEndian.PutUint64(entry[72:80], hash) + } + return contents +} diff --git a/internal/systemd/journal_query_file.go b/internal/systemd/journal_query_file.go index 4cbd9f49..e1ede5c3 100644 --- a/internal/systemd/journal_query_file.go +++ b/internal/systemd/journal_query_file.go @@ -28,6 +28,7 @@ type journalQueryEntry struct { seqnum uint64 realtime uint64 monotonic uint64 + xorHash uint64 offset uint64 selected builtins.JournalEntry } @@ -216,6 +217,7 @@ func (i *journalFileQueryIterator) previous(ctx context.Context) (journalQueryEn seqnum: entry.seqnum, realtime: entry.realtime, monotonic: entry.monotonic, + xorHash: entry.xorHash, offset: entry.offset, selected: selected, }, true, nil diff --git a/internal/systemd/journal_query_file_test.go b/internal/systemd/journal_query_file_test.go index 3dfd66c5..b04abf34 100644 --- a/internal/systemd/journal_query_file_test.go +++ b/internal/systemd/journal_query_file_test.go @@ -186,9 +186,11 @@ func collectJournalQueryEntries(t *testing.T, iterator *journalFileQueryIterator } type journalQueryFixtureEntry struct { - bootID journalID - realtime uint64 - fields []string + seqnum uint64 + bootID journalID + realtime uint64 + monotonic uint64 + fields []string } type journalQueryFixtureData struct { @@ -271,14 +273,15 @@ func buildJournalQueryFixtureWithLayout(t *testing.T, entries []journalQueryFixt contents := testJournalContents(nextOffset, 0, headerFlags) copy(contents[24:40], fileID[:]) + copy(contents[56:72], entries[len(entries)-1].bootID[:]) copy(contents[72:88], bytes.Repeat([]byte{0x55}, 16)) binary.LittleEndian.PutUint64(contents[104:112], uint64(tableOffset+journalObjectHeaderSize)) binary.LittleEndian.PutUint64(contents[112:120], bucketCount*journalHashItemSize) binary.LittleEndian.PutUint64(contents[136:144], uint64(lastJournalFixtureObjectOffset(dataObjects, entryOffsets))) binary.LittleEndian.PutUint64(contents[144:152], uint64(1+len(dataObjects)+len(entries)+arrayCount)) binary.LittleEndian.PutUint64(contents[152:160], uint64(len(entries))) - binary.LittleEndian.PutUint64(contents[160:168], uint64(len(entries))) - binary.LittleEndian.PutUint64(contents[168:176], 1) + binary.LittleEndian.PutUint64(contents[160:168], journalQueryFixtureSeqnum(entries[len(entries)-1], len(entries)-1)) + binary.LittleEndian.PutUint64(contents[168:176], journalQueryFixtureSeqnum(entries[0], 0)) binary.LittleEndian.PutUint64(contents[184:192], entries[0].realtime) binary.LittleEndian.PutUint64(contents[192:200], entries[len(entries)-1].realtime) binary.LittleEndian.PutUint64(contents[264:272], uint64(entryOffsets[len(entryOffsets)-1])) @@ -341,9 +344,14 @@ func buildJournalQueryFixtureWithLayout(t *testing.T, entries []journalQueryFixt entry := contents[entryOffsets[index]:] entry[0] = journalObjectEntry binary.LittleEndian.PutUint64(entry[8:16], uint64(journalEntryHeaderSize+len(entrySpec.fields)*entryItemSize)) - binary.LittleEndian.PutUint64(entry[16:24], uint64(index+1)) + seqnum := journalQueryFixtureSeqnum(entrySpec, index) + monotonic := entrySpec.monotonic + if monotonic == 0 { + monotonic = seqnum * 100 + } + binary.LittleEndian.PutUint64(entry[16:24], seqnum) binary.LittleEndian.PutUint64(entry[24:32], entrySpec.realtime) - binary.LittleEndian.PutUint64(entry[32:40], uint64(index+1)*100) + binary.LittleEndian.PutUint64(entry[32:40], monotonic) copy(entry[40:56], entrySpec.bootID[:]) var xorHash uint64 for itemIndex, field := range entrySpec.fields { @@ -362,6 +370,13 @@ func buildJournalQueryFixtureWithLayout(t *testing.T, entries []journalQueryFixt return contents } +func journalQueryFixtureSeqnum(entry journalQueryFixtureEntry, index int) uint64 { + if entry.seqnum != 0 { + return entry.seqnum + } + return uint64(index + 1) +} + func lastJournalFixtureObjectOffset(data []*journalQueryFixtureData, entryOffsets []int) int { last := entryOffsets[len(entryOffsets)-1] for _, object := range data { From 50598e1df8ca29fde834b88b0391d693ade95fe4 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:13:57 -0400 Subject: [PATCH 28/53] test(systemd): add real journal fixtures --- internal/systemd/journal_fixture_test.go | 146 ++++++++++++++++++ internal/systemd/testdata/journal/README.md | 22 +++ internal/systemd/testdata/journal/SHA256SUMS | 2 + .../journal/compact-keyed-zstd.journal.gz | Bin 0 -> 9682 bytes .../systemd/testdata/journal/fixture.export | 50 ++++++ internal/systemd/testdata/journal/generate.sh | 58 +++++++ .../journal/regular-uncompressed.journal.gz | Bin 0 -> 9665 bytes 7 files changed, 278 insertions(+) create mode 100644 internal/systemd/journal_fixture_test.go create mode 100644 internal/systemd/testdata/journal/README.md create mode 100644 internal/systemd/testdata/journal/SHA256SUMS create mode 100644 internal/systemd/testdata/journal/compact-keyed-zstd.journal.gz create mode 100644 internal/systemd/testdata/journal/fixture.export create mode 100755 internal/systemd/testdata/journal/generate.sh create mode 100644 internal/systemd/testdata/journal/regular-uncompressed.journal.gz diff --git a/internal/systemd/journal_fixture_test.go b/internal/systemd/journal_fixture_test.go new file mode 100644 index 00000000..e1c9d6d0 --- /dev/null +++ b/internal/systemd/journal_fixture_test.go @@ -0,0 +1,146 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +const journalFixtureSize = 8 * 1024 * 1024 + +func TestRealJournalFixtures(t *testing.T) { + longMessage := journalFixtureLongMessage() + bootID := repeatedJournalID(0xbb) + tests := []struct { + name string + compact bool + keyedHash bool + compression uint8 + }{ + {name: "regular-uncompressed.journal.gz"}, + { + name: "compact-keyed-zstd.journal.gz", + compact: true, + keyedHash: true, + compression: journalObjectCompressedZSTD, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := readJournalFixture(t, test.name) + view, err := newJournalFileView(test.name, bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", view.header.machineID.String()) + assert.Equal(t, uint64(4), view.header.nEntries) + assert.Equal(t, test.compact, view.header.compact()) + assert.Equal(t, test.keyedHash, view.header.keyedHash()) + + messageData, found, err := view.findDataObject([]byte("MESSAGE=" + longMessage)) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, test.compression, messageData.flags) + payload, truncated, err := view.readDataPayload(messageData, len(longMessage)+len("MESSAGE=")) + require.NoError(t, err) + assert.False(t, truncated) + assert.Equal(t, "MESSAGE="+longMessage, string(payload)) + + unitIterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + CurrentBoot: true, + MaxEntries: 10, + }, &bootID) + require.NoError(t, err) + unitEntries := collectJournalQueryEntries(t, unitIterator) + require.Len(t, unitEntries, 3) + assert.Equal(t, []string{longMessage, "manager noticed service", "service started"}, journalMessages(unitEntries)) + assert.Equal(t, "systemd", unitEntries[1].selected.Identifier) + assert.Equal(t, "1", unitEntries[1].selected.PID) + + kernelIterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Kernel: true, + CurrentBoot: true, + MaxEntries: 10, + }, &bootID) + require.NoError(t, err) + kernelEntries := collectJournalQueryEntries(t, kernelIterator) + require.Len(t, kernelEntries, 1) + assert.Equal(t, "synthetic kernel event", kernelEntries[0].selected.Message) + assert.Equal(t, "kernel", kernelEntries[0].selected.Identifier) + }) + } +} + +func TestJournalFixtureMatchesJournalctl(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("journalctl differential check requires Linux") + } + journalctl, err := exec.LookPath("journalctl") + if err != nil { + t.Skip("journalctl is not installed") + } + + path := filepath.Join(t.TempDir(), "fixture.journal") + require.NoError(t, os.WriteFile(path, readJournalFixture(t, "regular-uncompressed.journal.gz"), 0o600)) + + unitOutput := runFixtureJournalctl(t, journalctl, path, "--unit=rshell-fixture.service") + assert.Equal(t, strings.Join([]string{ + "service started", + "manager noticed service", + journalFixtureLongMessage(), + "", + }, "\n"), unitOutput) + + kernelOutput := runFixtureJournalctl(t, journalctl, path, "--dmesg") + assert.Equal(t, "synthetic kernel event\n", kernelOutput) +} + +func readJournalFixture(t *testing.T, name string) []byte { + t.Helper() + file, err := os.Open(filepath.Join("testdata", "journal", name)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, file.Close()) }) + reader, err := gzip.NewReader(file) + require.NoError(t, err) + contents, err := io.ReadAll(io.LimitReader(reader, journalFixtureSize+1)) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Len(t, contents, journalFixtureSize) + return contents +} + +func journalFixtureLongMessage() string { + return "compressed payload " + strings.TrimSpace(strings.Repeat(strings.Repeat("a", 64)+" ", 10)) +} + +func runFixtureJournalctl(t *testing.T, executable, journalPath string, selection ...string) string { + t.Helper() + arguments := []string{"--file=" + journalPath, "--no-pager", "--quiet", "--output=cat"} + arguments = append(arguments, selection...) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, executable, arguments...) + var stderr bytes.Buffer + command.Stderr = &stderr + output, err := command.Output() + require.NoError(t, err, stderr.String()) + return string(output) +} diff --git a/internal/systemd/testdata/journal/README.md b/internal/systemd/testdata/journal/README.md new file mode 100644 index 00000000..4ad45d7c --- /dev/null +++ b/internal/systemd/testdata/journal/README.md @@ -0,0 +1,22 @@ +# Journal fixtures + +These fixtures contain four synthetic records with fixed machine, boot, and +timestamps. They contain no logs or identifiers copied from a real host. The +generation script also replaces the journal header's generator machine ID with +the fixed synthetic machine ID before verification. + +The committed files were generated and verified with systemd 255.4 on Ubuntu +24.04. `regular-uncompressed.journal.gz` exercises the original regular layout. +`compact-keyed-zstd.journal.gz` exercises compact offsets, keyed hashes, and a +Zstandard-compressed DATA object. Gzip only keeps the sparse 8 MiB journal files +small in Git; tests decompress them before parsing. + +On a Linux system with `systemd-journal-remote`, regenerate them with: + +```sh +./generate.sh +``` + +Journal file and sequence IDs are randomized by systemd, so regeneration is +semantically deterministic but does not reproduce the same checksums. Review +the semantic test output and commit the updated `SHA256SUMS` with the fixtures. diff --git a/internal/systemd/testdata/journal/SHA256SUMS b/internal/systemd/testdata/journal/SHA256SUMS new file mode 100644 index 00000000..034f2a82 --- /dev/null +++ b/internal/systemd/testdata/journal/SHA256SUMS @@ -0,0 +1,2 @@ +9024a795f8dcb533277056d911ec7bf0f01e0e05e94ffba044e254fa6adc20eb compact-keyed-zstd.journal.gz +ace04706878f35355d7d94f246576f1a9d12083b583ae8bab4605bd4ac9a6b76 regular-uncompressed.journal.gz diff --git a/internal/systemd/testdata/journal/compact-keyed-zstd.journal.gz b/internal/systemd/testdata/journal/compact-keyed-zstd.journal.gz new file mode 100644 index 0000000000000000000000000000000000000000..de80616117e6ad035f465d38835410d4d459886f GIT binary patch literal 9682 zcmeI%dr(tX8US#r?PBr4w5UK4C}J0_5-ckqAZ7|w8kOSm2u4U_7a_(FLV*C0gskl% zTb}ZkQj(>V4X*?>l>m9UQXvt6sz6A<#86Cl#6$=okmq)~GaYCDVmj2d-FyGIb06n? z-*RjmHXAbU#0WgR01DD_C9<-#EmwC<{lG)n5h1hu` zxi9A;<)I|~AD@Hy%EubWmtML_#Sujo*^o1Y`S6T1UKkn0=t+tP)#M;kzN_p6YbinZ zSBj~^>$&UThOGz8`-AuV5y1f7WqFiTQR3$UJPRAH(!dlp4Pn*Ap6lkA>!hd{(9j5<1vi9b|U5LYX4OXe1t?k8;f^n#w z*|HbnFt9COaaY#1M1YvVINdIj?SB-11LxR*Zt>o#^)EDG@N7us@%4|p_J|GxA9s;o`60UNK?*}cfzAV+e z;5xDy-KXrE-h9DXfpq^r0%zMr+}+3DS>u4Q+o(>zfmPzx&Bv$0D~~%2WIlPW+Gl@g zDSYrhS$T^b2aEsw?kj6V{yjJA;@%Xlc{)E9?X%{#zh|_UV{CMU9s29%c8tHYs(5s3 zb=ZHc=#RzFHx?`0)Xhh6(uLRNE%naKgv42OMAo6oG(sACMelhhIqt#8h#1Oh*W1;+ zG2~>QmcoZ{w4)p(l(?r`LMO}y2-uK%qFb!5dB!?S)Bnd}sKR`*-cS+Xw9|{$g1Oh< zlo@{vU+=mJpDWV2z*)#=&H`;YZ@{z8?6#L*nS5J0q>^yUddhh==9oL*3~@H==qKet^&?eZxt4g~Xj@nFI~(UC2Epd< zn{hrh-rXM?1IUduoOY_OTK>;b+9-!u*JG#{e#gH5P;jhJc`ZY&_FS4|;1e2* z?AP3MlO$;m%?kqoQR+_H%^je0v=0rsQ8}zYKOFoKHmogO-h7<)Iu8QOA#+0jVtxQ6N?`@8X0H6FeM<@K$$dlXxJKX{_BeM&o>vm8*E z^by4eADnW(d|!xH>A53IOW(;^wmO~zm@>t2RWqLM|GFX9KiY`uC{9xpfbDK)#ME^0 zX72=DW1Wsb&WOhQLvpjjm!?XGr5BOjN6CS{9GOi*$Dm$_j$;;R2rzy!q>WcndXc+bI-iYLhn(5@XM zWBd6IuVs{J#srhpU2h8l8BChF8tBn%jou@Sys;r7(+~bBPYRzAL1*q3op1?tRT5PE za69YVT7T1`#tU=d{NB0%m__7fD={y+V?0QUC%w7;c*z~b6Of8!A+%xyMX-Ecbx|oe ze}AjHiq-AjUt!&QT}n|~23L5KWkrD*d77Y|AP!&2 + exit 1 +fi + +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT HUP INT TERM +export_file="$tmpdir/fixture.export" +cp "$fixture_dir/fixture.export" "$export_file" +# Journal Export Format terminates every record, including the last, with an +# empty line. Keep that delimiter out of the tracked source's trailing space. +printf '\n' >>"$export_file" + +normalize_machine_id() { + # journal-remote uses the generator host's ID in the file header. Keep the + # fixture synthetic and consistent with fixture.export instead. + printf '\252\252\252\252\252\252\252\252\252\252\252\252\252\252\252\252' | + dd of="$1" bs=1 seek=40 conv=notrunc status=none +} + +SYSTEMD_JOURNAL_COMPACT=0 \ + SYSTEMD_JOURNAL_KEYED_HASH=0 \ + SYSTEMD_JOURNAL_COMPRESS=none \ + "$remote" \ + --split-mode=none \ + --compress=no \ + --output="$tmpdir/regular-uncompressed.journal" \ + "$export_file" + +SYSTEMD_JOURNAL_COMPACT=1 \ + SYSTEMD_JOURNAL_KEYED_HASH=1 \ + SYSTEMD_JOURNAL_COMPRESS=zstd \ + "$remote" \ + --split-mode=none \ + --compress=yes \ + --output="$tmpdir/compact-keyed-zstd.journal" \ + "$export_file" + +normalize_machine_id "$tmpdir/regular-uncompressed.journal" +normalize_machine_id "$tmpdir/compact-keyed-zstd.journal" + +journalctl --verify --file="$tmpdir/regular-uncompressed.journal" +journalctl --verify --file="$tmpdir/compact-keyed-zstd.journal" + +gzip -n "$tmpdir/regular-uncompressed.journal" +gzip -n "$tmpdir/compact-keyed-zstd.journal" +mv "$tmpdir/regular-uncompressed.journal.gz" "$fixture_dir/regular-uncompressed.journal.gz" +mv "$tmpdir/compact-keyed-zstd.journal.gz" "$fixture_dir/compact-keyed-zstd.journal.gz" + +cd "$fixture_dir" +sha256sum compact-keyed-zstd.journal.gz regular-uncompressed.journal.gz >SHA256SUMS diff --git a/internal/systemd/testdata/journal/regular-uncompressed.journal.gz b/internal/systemd/testdata/journal/regular-uncompressed.journal.gz new file mode 100644 index 0000000000000000000000000000000000000000..d4c06a9429026e8697aee93673b57f60e802fdaa GIT binary patch literal 9665 zcmeI2dr(tn8pdhqTDz;oHVW7Tyi`S@26t=-cUqkmQ$b;|1cVR;F>VvV$Ofe3qRSGu zl_*6mA_R5|LaAJ8APEpi7$F1%5-LK3Bt{63NJs*NB!ob+GqpQYXJp*9lxb(q`Qywv z&wIY-eZSv1^L^*G>C$Su9~b+{ha?;-(dG3iKj(wd@h3aBW*tcT(3+30#vF1!keHwb zt0e=6;D4^X{;2tTZZ<0uda9Cf>JOmc?DjRi8_}D;IhnJr+__@aTOjNCJ^!TQyy}A- zOTI$vSy04FEdH_p3@5T1A~Tv7^fDh)dXe5wlwOpQEPlXZHUIlKsr93VI<0_n87i#e z66FWPrgjf1+8o~Aw{g&@qenoLG*GdpRYBrAN8b!d3?y1d?#4&rb{5%=)mx<@)bR#( z@@`gFRpob0sn#ax!=|!|Zq(7egn3495_z2LezqL$^Jo-iWIG9(w-s(eN0r&zDQ_iU zuO4}|u*Uta?#Yd*B;T;&Qbr?;-ZBZhf1~v5jtq6~kvP@NaI;eiE2Cr!Vhq>kIPsa^ zenWG3LFr5^$QDG<-A}0{KgG8^>l-~EgL99(`qZqYlx9GR$o=GbAwUBNEFXc#vXsg|#P**XPVsKX z6_mdHbLppTZ;LB7^e=BbC$f~(l6AUn`?K~-9SZ0xMxdoZ`$57BMwiWb;AF&-=X$|L zd$6P8wU@8h8=YPK|6NKio2wkgj~A|0!sYje zhVC@rLwhhM*A1c9GkiyzgPPQf@J|<6L2;N;F9Z#&R=;7M8Fq4iQ)BtapY#do`tJPs zo5-r9Q@9HHdv~y_ZW&11MuX%-41-&+_5NEmS8EP36lLSnpe;3xNiwk$gv;Z^(oYZi zf{#l4jz+@71GvFJ|2piyzCRITd)(Xkd2UVuN7sn4^1Hwz=R|_f<85aG2Wp`w##-H~ zxzqWi+xPnW2E)70HTSrvk<26SJ=vA3JplYhWQU56GBfPiPC$$qKww2YUzHbbA`1LiT2nCTOwu}Dz^m{MHF~lX1j<y4S3|TF}bI1AV?t7EX zQ$8XaonrfE;lrmdGj#puEGLTNCf6W_rjDbTzOLtoG$9!EJw#gC9xnhEvLQRfbm z!67sxIxSF$N~H_ZW6k{S5o+Rz!~GBNsdP%B;h@9m`m zyTwfn(-H8%8@p;2>Tkys6da^xiztYWaxWB9rJ}19>dkOh9>vWvb_k`sqNP0Ug?Xgv zE@>2ckBr4mLL6yk&X}`Fm9ELpv%Wt7mR#xCBb?2kq{(|(ciY|(Iq7zx>p8vfX~ouR zz5N!&bVn^bv`T+R`DI!mO35@ytwZ0C%ooxcg$muZxk5z~UkT!wMu-89mlY&|#rGZ8 z4hJgP*z+y{sr|y_BFT%v-~+WPoy;f98B+Lg#*dR~noT1pG4uJpAbaP>*=xcY9R&N@ zdC2y0Sg58;dJ9%A%k$ti&iVzTqo$;TR{92tTl->^*0P!7Su4E0tZz7;Ge`jvKmZ^B z5C8}O1ONg60e}EN03ZMm00;mC00IC3fB-;X*$A}oB$@U Date: Wed, 15 Jul 2026 14:19:26 -0400 Subject: [PATCH 29/53] test(systemd): fuzz journal file reader --- internal/systemd/journal_fixture_test.go | 22 +++ internal/systemd/journal_fuzz_test.go | 142 ++++++++++++++++++++ internal/systemd/journal_query_file_test.go | 4 +- 3 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 internal/systemd/journal_fuzz_test.go diff --git a/internal/systemd/journal_fixture_test.go b/internal/systemd/journal_fixture_test.go index e1c9d6d0..48925a5e 100644 --- a/internal/systemd/journal_fixture_test.go +++ b/internal/systemd/journal_fixture_test.go @@ -113,6 +113,28 @@ func TestJournalFixtureMatchesJournalctl(t *testing.T) { assert.Equal(t, "synthetic kernel event\n", kernelOutput) } +func TestRealJournalFixtureRejectsCorruptedCompressedData(t *testing.T) { + contents := readJournalFixture(t, "compact-keyed-zstd.journal.gz") + view, err := newJournalFileView("corrupt-compressed.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, found, err := view.findDataObject([]byte("MESSAGE=" + journalFixtureLongMessage())) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint8(journalObjectCompressedZSTD), data.flags) + contents[data.payloadOffset+data.payloadSize/2] ^= 0xff + + view, err = newJournalFileView("corrupt-compressed.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + MaxEntries: 10, + }, nil) + require.NoError(t, err) + _, _, err = iterator.previous(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalCorrupt) +} + func readJournalFixture(t *testing.T, name string) []byte { t.Helper() file, err := os.Open(filepath.Join("testdata", "journal", name)) diff --git a/internal/systemd/journal_fuzz_test.go b/internal/systemd/journal_fuzz_test.go new file mode 100644 index 00000000..3b52313e --- /dev/null +++ b/internal/systemd/journal_fuzz_test.go @@ -0,0 +1,142 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/DataDog/rshell/builtins" +) + +const ( + maxJournalFuzzInput = 256 * 1024 + maxJournalFuzzEncodedData = 64 * 1024 + maxJournalFuzzPayloadRead = 4 * 1024 + journalFuzzTimeout = 10 * time.Millisecond +) + +// FuzzJournalObjects drives every bounded object decoder reachable from a +// parsed header. Corrupt and unsupported inputs are expected; the contract is +// that they return without panicking or allocating from untrusted sizes. +func FuzzJournalObjects(f *testing.F) { + f.Add([]byte{}, []byte("MESSAGE=x"), uint64(0)) + f.Add(testJournalContents(journalHeaderCurrentSize, 0, 0), []byte("MESSAGE=x"), uint64(journalHeaderCurrentSize)) + f.Add(buildJournalFuzzSeed(f, false), []byte("_SYSTEMD_UNIT=api.service"), uint64(journalHeaderCurrentSize)) + f.Add(buildJournalFuzzSeed(f, true), []byte("MESSAGE=ready"), uint64(journalHeaderCurrentSize)) + + f.Fuzz(func(t *testing.T, contents, selector []byte, rawOffset uint64) { + if len(contents) > maxJournalFuzzInput || len(selector) == 0 || len(selector) > 512 { + return + } + view, err := newJournalFileView("fuzz.journal", bytes.NewReader(contents), uint64(len(contents))) + if err != nil { + return + } + + offsets := []uint64{ + rawOffset &^ 7, + view.header.tailObjectOffset, + view.header.tailEntryOffset, + } + if view.header.dataHashTableOffset >= journalObjectHeaderSize { + offsets = append(offsets, view.header.dataHashTableOffset-journalObjectHeaderSize) + } + for _, offset := range offsets { + fuzzJournalObject(view, offset) + } + + data, found, err := view.findDataObject(selector) + if err == nil && found && data.payloadSize <= maxJournalFuzzEncodedData { + _, _, _ = view.readDataPayload(data, maxJournalFuzzPayloadRead) + } + _, _, _ = newestJournalEntry(view) + }) +} + +// FuzzJournalQuery mutates structurally valid regular and compact journals and +// runs the same indexed unit/kernel query path used by the builtin. +func FuzzJournalQuery(f *testing.F) { + f.Add(buildJournalFuzzSeed(f, false), "api.service", false, byte(10)) + f.Add(buildJournalFuzzSeed(f, true), "api.service", false, byte(10)) + f.Add(buildJournalFuzzSeed(f, false), "", true, byte(10)) + + f.Fuzz(func(t *testing.T, contents []byte, unit string, kernel bool, rawLimit byte) { + if len(contents) > maxJournalFuzzInput || len(unit) > 256 { + return + } + view, err := newJournalFileView("query-fuzz.journal", bytes.NewReader(contents), uint64(len(contents))) + if err != nil { + return + } + if view.header.incompatibleFlags&(journalHeaderIncompatibleCompressedXZ|journalHeaderIncompatibleCompressedLZ4|journalHeaderIncompatibleCompressedZSTD) != 0 { + return + } + + query := builtins.JournalQuery{Kernel: kernel, MaxEntries: int(rawLimit%10) + 1} + if !kernel { + query.Units = []string{unit} + } + if err := validateJournalQuery(query); err != nil { + return + } + + iterator, err := newJournalFileQueryIterator(view, query, nil) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), journalFuzzTimeout) + defer cancel() + for count := 0; count < query.MaxEntries; count++ { + entry, found, err := iterator.previous(ctx) + if err != nil || !found { + return + } + if len(entry.selected.Hostname) > maxJournalFieldSize || + len(entry.selected.Identifier) > maxJournalFieldSize || + len(entry.selected.PID) > maxJournalFieldSize || + len(entry.selected.Message) > maxJournalFieldSize { + t.Fatal("journal query returned a selected field over its size limit") + } + } + }) +} + +func fuzzJournalObject(view *journalFileView, offset uint64) { + if offset == 0 { + return + } + object, err := view.objectAt(offset, 0) + if err != nil { + return + } + switch object.objectType { + case journalObjectData: + data, err := view.dataObjectAt(offset) + if err == nil && data.payloadSize <= maxJournalFuzzEncodedData { + _, _, _ = view.readDataPayload(data, maxJournalFuzzPayloadRead) + } + case journalObjectEntry: + _, _ = view.entryObjectAt(offset) + case journalObjectEntryArray: + array, err := view.entryArrayAt(offset) + if err == nil && array.capacity > 0 { + _, _ = view.entryArrayItem(array, 0) + } + } +} + +func buildJournalFuzzSeed(t testing.TB, compact bool) []byte { + t.Helper() + bootID := repeatedJournalID(0x22) + return buildJournalQueryFixtureWithLayout(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "_HOSTNAME=node", "SYSLOG_IDENTIFIER=api", "_PID=42", "MESSAGE=ready"}}, + {bootID: bootID, realtime: 1_700_000_000_000_001, fields: []string{"UNIT=api.service", "_PID=1", "SYSLOG_IDENTIFIER=systemd", "MESSAGE=manager"}}, + {bootID: bootID, realtime: 1_700_000_000_000_002, fields: []string{"_TRANSPORT=kernel", "_COMM=kernel", "MESSAGE=kernel"}}, + }, compact) +} diff --git a/internal/systemd/journal_query_file_test.go b/internal/systemd/journal_query_file_test.go index b04abf34..e2259fd1 100644 --- a/internal/systemd/journal_query_file_test.go +++ b/internal/systemd/journal_query_file_test.go @@ -201,12 +201,12 @@ type journalQueryFixtureData struct { arrayOffset int } -func buildJournalQueryFixture(t *testing.T, entries []journalQueryFixtureEntry) []byte { +func buildJournalQueryFixture(t testing.TB, entries []journalQueryFixtureEntry) []byte { t.Helper() return buildJournalQueryFixtureWithLayout(t, entries, false) } -func buildJournalQueryFixtureWithLayout(t *testing.T, entries []journalQueryFixtureEntry, compact bool) []byte { +func buildJournalQueryFixtureWithLayout(t testing.TB, entries []journalQueryFixtureEntry, compact bool) []byte { t.Helper() require.NotEmpty(t, entries) headerFlags := uint32(0) From 3247fa91f0efa3ace86894b413566177174d582d Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:22:43 -0400 Subject: [PATCH 30/53] refactor(systemd): use pure-go journal backend --- LICENSE-3rdparty.csv | 1 - go.mod | 1 - go.sum | 2 - internal/systemd/journal_fixture_test.go | 39 ++++++++++++++++++ internal/systemd/journal_read.go | 38 +++++++++++++++++ internal/systemd/journal_reader_linux_cgo.go | 41 ------------------- .../systemd/journal_reader_unsupported.go | 27 ------------ 7 files changed, 77 insertions(+), 72 deletions(-) create mode 100644 internal/systemd/journal_read.go delete mode 100644 internal/systemd/journal_reader_linux_cgo.go delete mode 100644 internal/systemd/journal_reader_unsupported.go diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index a0e5e04e..8e5bcbd7 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -4,7 +4,6 @@ github.com/DataDog/datadog-agent/pkg/template,https://github.com/DataDog/datadog github.com/DataDog/datadog-agent/pkg/util/log,https://github.com/DataDog/datadog-agent,Apache-2.0,"Copyright 2016-present Datadog, Inc." github.com/DataDog/datadog-agent/pkg/util/scrubber,https://github.com/DataDog/datadog-agent,Apache-2.0,"Copyright 2016-present Datadog, Inc." github.com/DataDog/datadog-agent/pkg/version,https://github.com/DataDog/datadog-agent,Apache-2.0,"Copyright 2016-present Datadog, Inc." -github.com/coreos/go-systemd/v22,https://github.com/coreos/go-systemd,Apache-2.0,"Copyright 2018 CoreOS, Inc" github.com/davecgh/go-spew,https://github.com/davecgh/go-spew,ISC,Copyright (c) 2012-2016 Dave Collins github.com/ebitengine/purego,https://github.com/ebitengine/purego,Apache-2.0,Copyright 2022 The Ebitengine Authors github.com/go-ole/go-ole,https://github.com/go-ole/go-ole,MIT,"Copyright (c) 2013-2017 Yasuhiro Matsumoto" diff --git a/go.mod b/go.mod index 77fd543e..40b21224 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ toolchain go1.26.2 require ( github.com/DataDog/datadog-agent/pkg/fleet/installer v0.78.0 - github.com/coreos/go-systemd/v22 v22.7.0 github.com/klauspost/compress v1.19.0 github.com/pierrec/lz4/v4 v4.1.27 github.com/prometheus-community/pro-bing v0.8.0 diff --git a/go.sum b/go.sum index 0350ab9a..c7812b78 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,6 @@ github.com/DataDog/datadog-agent/pkg/util/scrubber v0.78.0 h1:MydZgoGPYjGt8jCwo2 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.78.0/go.mod h1:zM5uAINxu8o9OZGCCvqeH1E/vIsX6baiO2iPf6bs1xs= github.com/DataDog/datadog-agent/pkg/version v0.78.0 h1:piLsyG7dP/ie6d4MQ17pTh5XPt/BV55dy8910eXiNj0= github.com/DataDog/datadog-agent/pkg/version v0.78.0/go.mod h1:cOWF+29ZahL6qcC3KAntJEupdMdteiOGaLmIMz8zYBg= -github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= -github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/systemd/journal_fixture_test.go b/internal/systemd/journal_fixture_test.go index 48925a5e..e11110af 100644 --- a/internal/systemd/journal_fixture_test.go +++ b/internal/systemd/journal_fixture_test.go @@ -113,6 +113,45 @@ func TestJournalFixtureMatchesJournalctl(t *testing.T) { assert.Equal(t, "synthetic kernel event\n", kernelOutput) } +func TestReadJournalUsesPureGoFixtureBackend(t *testing.T) { + root := t.TempDir() + machineID := repeatedJournalID(0xaa) + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID.String()+"\n"), 0o600)) + baseDir := filepath.Join(root, "journal") + journalDir := filepath.Join(baseDir, machineID.String()) + require.NoError(t, os.MkdirAll(journalDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(journalDir, "system.journal"), + readJournalFixture(t, "compact-keyed-zstd.journal.gz"), + 0o600, + )) + + client := NewClient(Target{JournalDirs: []string{baseDir}, MachineIDPath: machineIDPath}) + var entries []builtins.JournalEntry + err := client.ReadJournal(context.Background(), builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + CurrentBoot: true, + MaxEntries: 2, + }, func(entry builtins.JournalEntry) error { + entries = append(entries, entry) + return nil + }) + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, []string{"manager noticed service", journalFixtureLongMessage()}, []string{entries[0].Message, entries[1].Message}) +} + +func TestReadJournalReturnsCanceledContextBeforeFilesystemAccess(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := NewClient(Target{}).ReadJournal(ctx, builtins.JournalQuery{ + Units: []string{"api.service"}, + MaxEntries: 1, + }, func(builtins.JournalEntry) error { return nil }) + assert.ErrorIs(t, err, context.Canceled) +} + func TestRealJournalFixtureRejectsCorruptedCompressedData(t *testing.T) { contents := readJournalFixture(t, "compact-keyed-zstd.journal.gz") view, err := newJournalFileView("corrupt-compressed.journal", bytes.NewReader(contents), uint64(len(contents))) diff --git a/internal/systemd/journal_read.go b/internal/systemd/journal_read.go new file mode 100644 index 00000000..498e6a97 --- /dev/null +++ b/internal/systemd/journal_read.go @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "context" + "fmt" + + "github.com/DataDog/rshell/builtins" +) + +// ReadJournal reads a bounded structured query from the configured journal +// files. Results are buffered until the file snapshot is stable, then yielded +// oldest first as required by builtins.JournalReader. +func (c *Client) ReadJournal(ctx context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { + if err := ctx.Err(); err != nil { + return err + } + if yield == nil { + return fmt.Errorf("journal entry yield function is nil") + } + entries, err := c.queryJournalEntries(ctx, query) + if err != nil { + return err + } + for index := len(entries) - 1; index >= 0; index-- { + if err := ctx.Err(); err != nil { + return err + } + if err := yield(entries[index].selected); err != nil { + return err + } + } + return nil +} diff --git a/internal/systemd/journal_reader_linux_cgo.go b/internal/systemd/journal_reader_linux_cgo.go deleted file mode 100644 index 4fcef4eb..00000000 --- a/internal/systemd/journal_reader_linux_cgo.go +++ /dev/null @@ -1,41 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -//go:build linux && cgo - -package systemd - -import ( - "context" - "fmt" - - "github.com/coreos/go-systemd/v22/sdjournal" - - "github.com/DataDog/rshell/builtins" -) - -// ReadJournal opens only regular journal files belonging to the configured -// target machine and executes a bounded structured query against them. -func (c *Client) ReadJournal(ctx context.Context, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { - if err := validateJournalQuery(query); err != nil { - return err - } - if query.MaxEntries == 0 { - return nil - } - machineID, files, err := c.journalFiles() - if err != nil { - return err - } - if len(files) == 0 { - return fmt.Errorf("no journal files found for machine %s", machineID) - } - journal, err := sdjournal.NewJournalFromFiles(files...) - if err != nil { - return fmt.Errorf("open systemd journal: %w", err) - } - defer journal.Close() - return readJournal(ctx, journal, machineID, query, yield) -} diff --git a/internal/systemd/journal_reader_unsupported.go b/internal/systemd/journal_reader_unsupported.go deleted file mode 100644 index fcd2e14a..00000000 --- a/internal/systemd/journal_reader_unsupported.go +++ /dev/null @@ -1,27 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -//go:build !linux || !cgo - -package systemd - -import ( - "context" - "fmt" - - "github.com/DataDog/rshell/builtins" -) - -// ReadJournal reports the platform requirement without attempting filesystem -// access. Journal reading uses libsystemd's sd-journal API on Linux. -func (c *Client) ReadJournal(_ context.Context, query builtins.JournalQuery, _ func(builtins.JournalEntry) error) error { - if err := validateJournalQuery(query); err != nil { - return err - } - if query.MaxEntries == 0 { - return nil - } - return fmt.Errorf("%w: journal reading requires Linux with cgo", builtins.ErrSystemdUnsupported) -} From 64a49eebd17b47e1a88f83fde70b69ee808adc48 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:24:06 -0400 Subject: [PATCH 31/53] refactor(systemd): remove sd-journal adapter --- internal/systemd/journal_query.go | 36 ++++ internal/systemd/journal_query_test.go | 50 +++++ internal/systemd/journal_reader.go | 248 ------------------------ internal/systemd/journal_reader_test.go | 248 ------------------------ 4 files changed, 86 insertions(+), 496 deletions(-) create mode 100644 internal/systemd/journal_query.go create mode 100644 internal/systemd/journal_query_test.go delete mode 100644 internal/systemd/journal_reader.go delete mode 100644 internal/systemd/journal_reader_test.go diff --git a/internal/systemd/journal_query.go b/internal/systemd/journal_query.go new file mode 100644 index 00000000..0e00b789 --- /dev/null +++ b/internal/systemd/journal_query.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "fmt" + "strings" + + "github.com/DataDog/rshell/builtins" +) + +const maxJournalFieldSize = 64 * 1024 + +func validateJournalQuery(query builtins.JournalQuery) error { + if query.MaxEntries < 0 || query.MaxEntries > builtins.MaxJournalQueryEntries { + return fmt.Errorf("journal query entry limit must be between 0 and %d", builtins.MaxJournalQueryEntries) + } + if query.Kernel && len(query.Units) > 0 { + return fmt.Errorf("journal query cannot combine kernel and unit scopes") + } + if !query.Kernel && len(query.Units) == 0 { + return fmt.Errorf("journal query requires a kernel or unit scope") + } + if len(query.Units) > builtins.MaxJournalQueryUnits { + return fmt.Errorf("journal query has too many units (maximum %d)", builtins.MaxJournalQueryUnits) + } + for _, unit := range query.Units { + if unit == "" || len(unit) > 256 || strings.IndexByte(unit, 0) >= 0 { + return fmt.Errorf("journal query contains an invalid unit name") + } + } + return nil +} diff --git a/internal/systemd/journal_query_test.go b/internal/systemd/journal_query_test.go new file mode 100644 index 00000000..dbcb80d6 --- /dev/null +++ b/internal/systemd/journal_query_test.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/rshell/builtins" +) + +func TestValidateJournalQueryAcceptsBoundedScopes(t *testing.T) { + require.NoError(t, validateJournalQuery(builtins.JournalQuery{Kernel: true, MaxEntries: builtins.MaxJournalQueryEntries})) + require.NoError(t, validateJournalQuery(builtins.JournalQuery{Units: []string{"api.service"}, MaxEntries: 1})) +} + +func TestValidateJournalQueryRejectsInvalidScopes(t *testing.T) { + tooManyUnits := make([]string, builtins.MaxJournalQueryUnits+1) + for index := range tooManyUnits { + tooManyUnits[index] = "api.service" + } + tests := []struct { + name string + query builtins.JournalQuery + match string + }{ + {name: "negative limit", query: builtins.JournalQuery{Kernel: true, MaxEntries: -1}, match: "entry limit"}, + {name: "large limit", query: builtins.JournalQuery{Kernel: true, MaxEntries: builtins.MaxJournalQueryEntries + 1}, match: "entry limit"}, + {name: "missing scope", query: builtins.JournalQuery{MaxEntries: 1}, match: "requires a kernel or unit scope"}, + {name: "mixed scopes", query: builtins.JournalQuery{Kernel: true, Units: []string{"api.service"}, MaxEntries: 1}, match: "cannot combine"}, + {name: "too many units", query: builtins.JournalQuery{Units: tooManyUnits, MaxEntries: 1}, match: "too many units"}, + {name: "empty unit", query: builtins.JournalQuery{Units: []string{""}, MaxEntries: 1}, match: "invalid unit name"}, + {name: "long unit", query: builtins.JournalQuery{Units: []string{strings.Repeat("a", 257)}, MaxEntries: 1}, match: "invalid unit name"}, + {name: "nul unit", query: builtins.JournalQuery{Units: []string{"api\x00.service"}, MaxEntries: 1}, match: "invalid unit name"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateJournalQuery(test.query) + require.Error(t, err) + assert.Contains(t, err.Error(), test.match) + }) + } +} diff --git a/internal/systemd/journal_reader.go b/internal/systemd/journal_reader.go deleted file mode 100644 index f903c6c4..00000000 --- a/internal/systemd/journal_reader.go +++ /dev/null @@ -1,248 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -package systemd - -import ( - "context" - "errors" - "fmt" - "strings" - "syscall" - "time" - - "github.com/DataDog/rshell/builtins" -) - -const ( - maxJournalFieldSize = 64 * 1024 - maxBootSearch = 1024 -) - -type journalHandle interface { - AddMatch(match string) error - AddDisjunction() error - AddConjunction() error - FlushMatches() - Next() (uint64, error) - Previous() (uint64, error) - PreviousSkip(skip uint64) (uint64, error) - GetData(field string) (string, error) - GetRealtimeUsec() (uint64, error) - SetDataThreshold(threshold uint64) error - SeekHead() error - SeekTail() error -} - -func validateJournalQuery(query builtins.JournalQuery) error { - if query.MaxEntries < 0 || query.MaxEntries > builtins.MaxJournalQueryEntries { - return fmt.Errorf("journal query entry limit must be between 0 and %d", builtins.MaxJournalQueryEntries) - } - if query.Kernel && len(query.Units) > 0 { - return fmt.Errorf("journal query cannot combine kernel and unit scopes") - } - if !query.Kernel && len(query.Units) == 0 { - return fmt.Errorf("journal query requires a kernel or unit scope") - } - if len(query.Units) > builtins.MaxJournalQueryUnits { - return fmt.Errorf("journal query has too many units (maximum %d)", builtins.MaxJournalQueryUnits) - } - for _, unit := range query.Units { - if unit == "" || len(unit) > 256 || strings.IndexByte(unit, 0) >= 0 { - return fmt.Errorf("journal query contains an invalid unit name") - } - } - return nil -} - -func readJournal(ctx context.Context, journal journalHandle, machineID string, query builtins.JournalQuery, yield func(builtins.JournalEntry) error) error { - if err := validateJournalQuery(query); err != nil { - return err - } - if query.MaxEntries == 0 { - return nil - } - if err := journal.SetDataThreshold(maxJournalFieldSize + 64); err != nil { - return fmt.Errorf("set journal field limit: %w", err) - } - - bootID := "" - if query.CurrentBoot { - var err error - bootID, err = newestBootID(ctx, journal, machineID) - if err != nil { - return err - } - journal.FlushMatches() - } - if err := addJournalMatches(journal, machineID, bootID, query); err != nil { - return err - } - if err := seekJournalTail(journal, query.MaxEntries); err != nil { - return err - } - - for scanned := 0; scanned < query.MaxEntries; scanned++ { - if err := ctx.Err(); err != nil { - return err - } - advanced, err := journal.Next() - if err != nil { - return fmt.Errorf("iterate journal: %w", err) - } - if advanced == 0 { - return nil - } - - entry, err := selectedJournalEntry(journal, machineID) - if err != nil { - return err - } - if !query.Since.IsZero() && entry.Timestamp.Before(query.Since) { - continue - } - if err := yield(entry); err != nil { - return err - } - } - return nil -} - -func newestBootID(ctx context.Context, journal journalHandle, machineID string) (string, error) { - if err := journal.AddMatch("_MACHINE_ID=" + machineID); err != nil { - return "", fmt.Errorf("match journal machine ID: %w", err) - } - if err := journal.SeekTail(); err != nil { - return "", fmt.Errorf("seek journal tail: %w", err) - } - for i := 0; i < maxBootSearch; i++ { - if err := ctx.Err(); err != nil { - return "", err - } - advanced, err := journal.Previous() - if err != nil { - return "", fmt.Errorf("search current journal boot: %w", err) - } - if advanced == 0 { - break - } - bootID, found, err := journalData(journal, "_BOOT_ID") - if err != nil { - return "", err - } - if found && validID128(bootID) { - return strings.ToLower(bootID), nil - } - } - return "", fmt.Errorf("could not determine the current boot from the selected journal") -} - -func addJournalMatches(journal journalHandle, machineID, bootID string, query builtins.JournalQuery) error { - if query.Kernel { - if err := journal.AddMatch("_TRANSPORT=kernel"); err != nil { - return fmt.Errorf("match kernel journal entries: %w", err) - } - } else { - for _, unit := range query.Units { - if err := journal.AddMatch("_SYSTEMD_UNIT=" + unit); err != nil { - return fmt.Errorf("match journal unit: %w", err) - } - } - if err := journal.AddDisjunction(); err != nil { - return fmt.Errorf("combine journal unit matches: %w", err) - } - if err := journal.AddMatch("_PID=1"); err != nil { - return fmt.Errorf("match system manager journal entries: %w", err) - } - for _, unit := range query.Units { - if err := journal.AddMatch("UNIT=" + unit); err != nil { - return fmt.Errorf("match manager messages about journal unit: %w", err) - } - } - if err := journal.AddConjunction(); err != nil { - return fmt.Errorf("constrain journal unit matches: %w", err) - } - } - if err := journal.AddMatch("_MACHINE_ID=" + machineID); err != nil { - return fmt.Errorf("match journal machine ID: %w", err) - } - if bootID != "" { - if err := journal.AddMatch("_BOOT_ID=" + bootID); err != nil { - return fmt.Errorf("match current journal boot: %w", err) - } - } - return nil -} - -func seekJournalTail(journal journalHandle, entries int) error { - if err := journal.SeekTail(); err != nil { - return fmt.Errorf("seek journal tail: %w", err) - } - skipped, err := journal.PreviousSkip(uint64(entries) + 1) - if err != nil { - return fmt.Errorf("seek to last %d journal entries: %w", entries, err) - } - if skipped != uint64(entries)+1 { - if err := journal.SeekHead(); err != nil { - return fmt.Errorf("seek journal head: %w", err) - } - } - return nil -} - -func selectedJournalEntry(journal journalHandle, machineID string) (builtins.JournalEntry, error) { - entryMachineID, found, err := journalData(journal, "_MACHINE_ID") - if err != nil { - return builtins.JournalEntry{}, err - } - if !found || entryMachineID != machineID { - return builtins.JournalEntry{}, fmt.Errorf("journal entry machine ID does not match the configured target") - } - - realtimeUsec, err := journal.GetRealtimeUsec() - if err != nil { - return builtins.JournalEntry{}, fmt.Errorf("read journal timestamp: %w", err) - } - entry := builtins.JournalEntry{ - Timestamp: time.Unix(int64(realtimeUsec/1_000_000), int64(realtimeUsec%1_000_000)*1000), - } - if entry.Hostname, _, err = journalData(journal, "_HOSTNAME"); err != nil { - return builtins.JournalEntry{}, err - } - if entry.Identifier, _, err = journalData(journal, "SYSLOG_IDENTIFIER"); err != nil { - return builtins.JournalEntry{}, err - } - if entry.Identifier == "" { - if entry.Identifier, _, err = journalData(journal, "_COMM"); err != nil { - return builtins.JournalEntry{}, err - } - } - if entry.PID, _, err = journalData(journal, "_PID"); err != nil { - return builtins.JournalEntry{}, err - } - if entry.Message, _, err = journalData(journal, "MESSAGE"); err != nil { - return builtins.JournalEntry{}, err - } - return entry, nil -} - -func journalData(journal journalHandle, field string) (string, bool, error) { - data, err := journal.GetData(field) - if err != nil { - if errors.Is(err, syscall.ENOENT) { - return "", false, nil - } - return "", false, fmt.Errorf("read journal field %s: %w", field, err) - } - prefix := field + "=" - if !strings.HasPrefix(data, prefix) { - return "", false, fmt.Errorf("journal returned malformed field %s", field) - } - value := strings.TrimPrefix(data, prefix) - if len(value) > maxJournalFieldSize { - value = value[:maxJournalFieldSize] - } - return value, true, nil -} diff --git a/internal/systemd/journal_reader_test.go b/internal/systemd/journal_reader_test.go deleted file mode 100644 index d69964d2..00000000 --- a/internal/systemd/journal_reader_test.go +++ /dev/null @@ -1,248 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-present Datadog, Inc. - -package systemd - -import ( - "context" - "fmt" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/DataDog/rshell/builtins" -) - -type fakeJournalEntry struct { - realtimeUsec uint64 - fields map[string]string -} - -type fakeJournal struct { - calls []string - entries []fakeJournalEntry - position int - threshold uint64 - dataFields []string -} - -func (j *fakeJournal) AddMatch(match string) error { - j.calls = append(j.calls, "match "+match) - return nil -} - -func (j *fakeJournal) AddDisjunction() error { - j.calls = append(j.calls, "or") - return nil -} - -func (j *fakeJournal) AddConjunction() error { - j.calls = append(j.calls, "and") - return nil -} - -func (j *fakeJournal) FlushMatches() { - j.calls = append(j.calls, "flush") -} - -func (j *fakeJournal) Next() (uint64, error) { - if j.position+1 >= len(j.entries) { - j.position = len(j.entries) - return 0, nil - } - j.position++ - return 1, nil -} - -func (j *fakeJournal) Previous() (uint64, error) { - if j.position <= 0 { - j.position = -1 - return 0, nil - } - j.position-- - return 1, nil -} - -func (j *fakeJournal) PreviousSkip(skip uint64) (uint64, error) { - available := j.position - if available < 0 { - available = 0 - } - actual := int(skip) - if actual > available { - actual = available - } - j.position -= actual - return uint64(actual), nil -} - -func (j *fakeJournal) GetData(field string) (string, error) { - j.dataFields = append(j.dataFields, field) - if j.position < 0 || j.position >= len(j.entries) { - return "", fmt.Errorf("no current entry") - } - value, ok := j.entries[j.position].fields[field] - if !ok { - return "", fmt.Errorf("missing field: %w", syscall.ENOENT) - } - return field + "=" + value, nil -} - -func (j *fakeJournal) GetRealtimeUsec() (uint64, error) { - if j.position < 0 || j.position >= len(j.entries) { - return 0, fmt.Errorf("no current entry") - } - return j.entries[j.position].realtimeUsec, nil -} - -func (j *fakeJournal) SetDataThreshold(threshold uint64) error { - j.threshold = threshold - return nil -} - -func (j *fakeJournal) SeekHead() error { - j.calls = append(j.calls, "head") - j.position = -1 - return nil -} - -func (j *fakeJournal) SeekTail() error { - j.calls = append(j.calls, "tail") - j.position = len(j.entries) - return nil -} - -func TestAddJournalMatchesUsesRestrictedUnitExpansion(t *testing.T) { - journal := &fakeJournal{} - err := addJournalMatches(journal, "machine", "boot", builtins.JournalQuery{ - Units: []string{"api.service", "worker.service"}, - }) - require.NoError(t, err) - assert.Equal(t, []string{ - "match _SYSTEMD_UNIT=api.service", - "match _SYSTEMD_UNIT=worker.service", - "or", - "match _PID=1", - "match UNIT=api.service", - "match UNIT=worker.service", - "and", - "match _MACHINE_ID=machine", - "match _BOOT_ID=boot", - }, journal.calls) -} - -func TestAddJournalMatchesConstrainsKernelToTarget(t *testing.T) { - journal := &fakeJournal{} - err := addJournalMatches(journal, "machine", "boot", builtins.JournalQuery{Kernel: true}) - require.NoError(t, err) - assert.Equal(t, []string{ - "match _TRANSPORT=kernel", - "match _MACHINE_ID=machine", - "match _BOOT_ID=boot", - }, journal.calls) -} - -func TestReadJournalYieldsOnlySelectedFields(t *testing.T) { - machineID := "0123456789abcdef0123456789abcdef" - start := time.Unix(1_700_000_000, 0) - journal := &fakeJournal{ - position: -1, - entries: []fakeJournalEntry{ - { - realtimeUsec: uint64(start.Add(-time.Minute).UnixMicro()), - fields: map[string]string{ - "_MACHINE_ID": machineID, - "_HOSTNAME": "host", - "SYSLOG_IDENTIFIER": "api", - "_PID": "12", - "MESSAGE": "old", - "_CMDLINE": "secret-token", - }, - }, - { - realtimeUsec: uint64(start.UnixMicro()), - fields: map[string]string{ - "_MACHINE_ID": machineID, - "_HOSTNAME": "host", - "SYSLOG_IDENTIFIER": "api", - "_PID": "13", - "MESSAGE": "ready", - }, - }, - }, - } - - var entries []builtins.JournalEntry - err := readJournal(context.Background(), journal, machineID, builtins.JournalQuery{ - Units: []string{"api.service"}, - Since: start, - MaxEntries: 2, - }, func(entry builtins.JournalEntry) error { - entries = append(entries, entry) - return nil - }) - require.NoError(t, err) - require.Len(t, entries, 1) - assert.Equal(t, "ready", entries[0].Message) - assert.Equal(t, uint64(maxJournalFieldSize+64), journal.threshold) - assert.NotContains(t, journal.dataFields, "_CMDLINE") - assert.Contains(t, journal.calls, "head") -} - -func TestReadJournalFindsCurrentBootInTargetJournal(t *testing.T) { - machineID := "0123456789abcdef0123456789abcdef" - bootID := "abcdef0123456789abcdef0123456789" - journal := &fakeJournal{ - position: -1, - entries: []fakeJournalEntry{{ - realtimeUsec: 1, - fields: map[string]string{ - "_MACHINE_ID": machineID, - "_BOOT_ID": bootID, - "MESSAGE": "booted", - }, - }}, - } - - err := readJournal(context.Background(), journal, machineID, builtins.JournalQuery{ - Kernel: true, - CurrentBoot: true, - MaxEntries: 1, - }, func(builtins.JournalEntry) error { return nil }) - require.NoError(t, err) - assert.Contains(t, journal.calls, "flush") - assert.Contains(t, journal.calls, "match _BOOT_ID="+bootID) -} - -func TestReadJournalRejectsEntryFromAnotherMachine(t *testing.T) { - journal := &fakeJournal{ - position: -1, - entries: []fakeJournalEntry{{ - realtimeUsec: 1, - fields: map[string]string{ - "_MACHINE_ID": "ffffffffffffffffffffffffffffffff", - }, - }}, - } - err := readJournal(context.Background(), journal, "0123456789abcdef0123456789abcdef", builtins.JournalQuery{ - Kernel: true, - MaxEntries: 1, - }, func(builtins.JournalEntry) error { return nil }) - require.Error(t, err) - assert.Contains(t, err.Error(), "does not match") -} - -func TestValidateJournalQueryRejectsUnboundedOrMixedScopes(t *testing.T) { - for _, query := range []builtins.JournalQuery{ - {MaxEntries: builtins.MaxJournalQueryEntries + 1, Kernel: true}, - {MaxEntries: 1}, - {MaxEntries: 1, Kernel: true, Units: []string{"api.service"}}, - } { - require.Error(t, validateJournalQuery(query)) - } -} From 5882f3ed0b5adee2b936d355be8a2777b2309bd4 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:25:39 -0400 Subject: [PATCH 32/53] docs(journalctl): document pure-go reader --- README.md | 6 ++++-- SHELL_FEATURES.md | 2 +- docs/RULES.md | 15 ++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3ebad34a..7de40707 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ The development CLI accepts equivalent grants through `--allowed-services mysql. **SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. -The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Journal entries are checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. +The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Every selected journal file header is checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. | Operation | Required target access | |-----------|------------------------| @@ -116,7 +116,9 @@ Root targets derive those paths below `Root`; explicit targets may map them to a Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. -Log reading requires Linux, cgo, and `libsystemd`. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--rotate` may be combined with vacuum flags; rotation completes first so the newly archived files are eligible for cleanup. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. +Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. + +The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--rotate` may be combined with vacuum flags; rotation completes first so the newly archived files are eligible for cleanup. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 86b51d92..f0f3bf4c 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads require Linux+cgo+libsystemd, rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected diff --git a/docs/RULES.md b/docs/RULES.md index 0f007503..64449122 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -80,11 +80,12 @@ bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding application and cannot be supplied by shell scripts. Journal reads and storage metadata are limited to regular, non-symlink `.journal` -files directly under the configured machine-ID directories. The reader adds the -configured machine ID to every native query, verifies it again on every returned -entry, and applies fixed file, field-size, entry-count, and cancellation bounds. -Builtins receive selected fields only and never receive a raw journal handle, -target path, or arbitrary field-match capability. +files directly under the configured machine-ID directories. The pure-Go reader +verifies each file header's machine ID, snapshots the file set and identities, +retries once after a concurrent rotation or structurally inconsistent read, and +applies fixed file, index, field-size, entry-count, decompression, and cancellation +bounds. Builtins receive selected fields only and never receive a raw journal +handle, target path, or arbitrary field-match capability. The only deletion exception is `JournalCleaner.VacuumJournal`. It is available only through trusted systemd target configuration and a validated operator @@ -93,8 +94,8 @@ checks directory identity across open, accepts only strict systemd archived-file names, excludes symlinks and hardlinks, and revalidates file identity immediately before rooted removal. Active, malformed, recently modified, and policy-retained files must never be deleted. Every cleanup call has both file-count and byte -ceilings in addition to the shared `journal:storage/clean` authorization and -remediation-mode requirement. +ceilings in addition to the exact `systemd-journald.service:clean` authorization +and remediation-mode requirement. `JournalRotator.RotateJournal` is the only journal-daemon mutation exception. It may call only the fixed `io.systemd.Journal.Rotate` Varlink method through From c4fa6e6554184c34b457039c62627077a4864ec7 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 14:55:49 -0400 Subject: [PATCH 33/53] test(systemd): isolate journal fixture generation --- internal/systemd/journal_fixture_test.go | 8 ++++- internal/systemd/testdata/journal/README.md | 29 +++++++++++++------ internal/systemd/testdata/journal/SHA256SUMS | 2 -- internal/systemd/testdata/journal/generate.sh | 16 +++++++--- 4 files changed, 39 insertions(+), 16 deletions(-) delete mode 100644 internal/systemd/testdata/journal/SHA256SUMS diff --git a/internal/systemd/journal_fixture_test.go b/internal/systemd/journal_fixture_test.go index e11110af..8804360a 100644 --- a/internal/systemd/journal_fixture_test.go +++ b/internal/systemd/journal_fixture_test.go @@ -26,6 +26,8 @@ import ( const journalFixtureSize = 8 * 1024 * 1024 +const journalFixtureDirectoryEnv = "RSHELL_JOURNAL_FIXTURE_DIR" + func TestRealJournalFixtures(t *testing.T) { longMessage := journalFixtureLongMessage() bootID := repeatedJournalID(0xbb) @@ -176,7 +178,11 @@ func TestRealJournalFixtureRejectsCorruptedCompressedData(t *testing.T) { func readJournalFixture(t *testing.T, name string) []byte { t.Helper() - file, err := os.Open(filepath.Join("testdata", "journal", name)) + directory := os.Getenv(journalFixtureDirectoryEnv) + if directory == "" { + directory = filepath.Join("testdata", "journal") + } + file, err := os.Open(filepath.Join(directory, name)) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, file.Close()) }) reader, err := gzip.NewReader(file) diff --git a/internal/systemd/testdata/journal/README.md b/internal/systemd/testdata/journal/README.md index 4ad45d7c..c895f613 100644 --- a/internal/systemd/testdata/journal/README.md +++ b/internal/systemd/testdata/journal/README.md @@ -5,18 +5,29 @@ timestamps. They contain no logs or identifiers copied from a real host. The generation script also replaces the journal header's generator machine ID with the fixed synthetic machine ID before verification. -The committed files were generated and verified with systemd 255.4 on Ubuntu -24.04. `regular-uncompressed.journal.gz` exercises the original regular layout. -`compact-keyed-zstd.journal.gz` exercises compact offsets, keyed hashes, and a -Zstandard-compressed DATA object. Gzip only keeps the sparse 8 MiB journal files -small in Git; tests decompress them before parsing. +The two committed golden files were generated and verified with systemd 255.4 +on Ubuntu 24.04. `regular-uncompressed.journal.gz` exercises the original +regular layout. `compact-keyed-zstd.journal.gz` exercises compact offsets, +keyed hashes, and a Zstandard-compressed DATA object. Gzip only keeps the sparse +8 MiB journal files small in Git; tests decompress them in memory or into a +per-test temporary directory. Test runs do not write generated files here. -On a Linux system with `systemd-journal-remote`, regenerate them with: +The golden files remain committed so tests can validate files emitted by real +systemd on hosts and CI workers that do not provide `systemd-journal-remote`. +`fixture.export` is their synthetic source rather than generated output. + +On a Linux system with `systemd-journal-remote`, generate a fresh corpus in an +explicit temporary output directory and test it with: ```sh -./generate.sh +output_dir=$(mktemp -d) +./generate.sh "$output_dir" +RSHELL_JOURNAL_FIXTURE_DIR="$output_dir" \ + go test ../.. \ + -run 'TestRealJournalFixtures|TestJournalFixtureMatchesJournalctl|TestReadJournalUsesPureGoFixtureBackend' ``` Journal file and sequence IDs are randomized by systemd, so regeneration is -semantically deterministic but does not reproduce the same checksums. Review -the semantic test output and commit the updated `SHA256SUMS` with the fixtures. +semantically deterministic but does not reproduce byte-identical files. The +generator never replaces the committed golden corpus unless this directory is +passed explicitly as its output directory. diff --git a/internal/systemd/testdata/journal/SHA256SUMS b/internal/systemd/testdata/journal/SHA256SUMS deleted file mode 100644 index 034f2a82..00000000 --- a/internal/systemd/testdata/journal/SHA256SUMS +++ /dev/null @@ -1,2 +0,0 @@ -9024a795f8dcb533277056d911ec7bf0f01e0e05e94ffba044e254fa6adc20eb compact-keyed-zstd.journal.gz -ace04706878f35355d7d94f246576f1a9d12083b583ae8bab4605bd4ac9a6b76 regular-uncompressed.journal.gz diff --git a/internal/systemd/testdata/journal/generate.sh b/internal/systemd/testdata/journal/generate.sh index 33b1a131..d5223924 100755 --- a/internal/systemd/testdata/journal/generate.sh +++ b/internal/systemd/testdata/journal/generate.sh @@ -5,11 +5,20 @@ set -eu fixture_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) remote=${SYSTEMD_JOURNAL_REMOTE:-/usr/lib/systemd/systemd-journal-remote} +if [ "$#" -ne 1 ]; then + echo "usage: $0 OUTPUT_DIR" >&2 + exit 2 +fi + if [ ! -x "$remote" ]; then echo "systemd-journal-remote not found at $remote" >&2 exit 1 fi +output_dir=$1 +mkdir -p "$output_dir" +output_dir=$(CDPATH= cd -- "$output_dir" && pwd) + tmpdir=$(mktemp -d) trap 'rm -rf "$tmpdir"' EXIT HUP INT TERM export_file="$tmpdir/fixture.export" @@ -51,8 +60,7 @@ journalctl --verify --file="$tmpdir/compact-keyed-zstd.journal" gzip -n "$tmpdir/regular-uncompressed.journal" gzip -n "$tmpdir/compact-keyed-zstd.journal" -mv "$tmpdir/regular-uncompressed.journal.gz" "$fixture_dir/regular-uncompressed.journal.gz" -mv "$tmpdir/compact-keyed-zstd.journal.gz" "$fixture_dir/compact-keyed-zstd.journal.gz" +mv "$tmpdir/regular-uncompressed.journal.gz" "$output_dir/regular-uncompressed.journal.gz" +mv "$tmpdir/compact-keyed-zstd.journal.gz" "$output_dir/compact-keyed-zstd.journal.gz" -cd "$fixture_dir" -sha256sum compact-keyed-zstd.journal.gz regular-uncompressed.journal.gz >SHA256SUMS +printf 'generated journal fixtures in %s\n' "$output_dir" From 40c797735d594c5e74b8ce42e90a2b21846b95e1 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 16:18:14 -0400 Subject: [PATCH 34/53] fix(journalctl): protect recent archives during vacuum --- README.md | 2 +- SHELL_FEATURES.md | 2 +- builtins/journalctl/journalctl.go | 14 ++++++++---- builtins/journalctl/journalctl_test.go | 22 +++++++++++++++++++ builtins/systemd.go | 3 ++- internal/systemd/journal_vacuum_unix.go | 7 ++++-- internal/systemd/journal_vacuum_unix_test.go | 16 ++++++++++---- .../cmd/journalctl/errors/vacuum_denied.yaml | 2 +- .../errors/vacuum_size_without_time.yaml | 13 +++++++++++ 9 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml diff --git a/README.md b/README.md index 7de40707..f7a29fd4 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Log queries default to 100 entries and are capped at 1,000 entries and 32 exact Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. -The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--rotate` may be combined with vacuum flags; rotation completes first so the newly archived files are eligible for cleanup. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. +The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--vacuum-size` requires `--vacuum-time`; size cleanup deletes only archives at least that old and stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index f0f3bf4c..6ad36af6 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; size vacuum requires a time floor and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index ccac388e..09a2c7a6 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -23,8 +23,10 @@ // -o, --output=FORMAT output format: short (default) or cat // --disk-usage show allocated journal storage and exit // --rotate archive active journal files before returning -// --vacuum-size=SIZE remove oldest archives toward allocated SIZE -// --vacuum-time=AGE remove archives older than Go duration AGE +// --vacuum-size=SIZE remove oldest eligible archives toward allocated SIZE; +// requires --vacuum-time +// --vacuum-time=AGE remove archives older than Go duration AGE and limit +// size vacuum eligibility // --dry-run report cleanup without deleting archives // -h, --help print usage and exit package journalctl @@ -80,8 +82,8 @@ func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { output: fs.StringP("output", "o", "short", "output format: short or cat"), usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), rotate: fs.Bool("rotate", false, "archive active journal files and wait for completion"), - vacuumSize: fs.String("vacuum-size", "", "remove oldest archives toward allocated SIZE"), - vacuumTime: fs.String("vacuum-time", "", "remove archives older than Go duration AGE"), + vacuumSize: fs.String("vacuum-size", "", "remove oldest eligible archives toward allocated SIZE; requires --vacuum-time"), + vacuumTime: fs.String("vacuum-time", "", "remove archives older than Go duration AGE and limit size vacuum eligibility"), dryRun: fs.Bool("dry-run", false, "report cleanup without deleting archives"), help: fs.BoolP("help", "h", false, "print usage and exit"), } @@ -258,6 +260,10 @@ func (options flags) runMaintenance(ctx context.Context, callCtx *builtins.CallC } request.MaxBytes = uint64(size) } + if sizeSet && !timeSet { + callCtx.Errf("journalctl: --vacuum-size requires --vacuum-time to protect recently modified archives\n") + return builtins.Result{Code: 1} + } if timeSet { age, err := time.ParseDuration(*options.vacuumTime) if err != nil || age <= 0 { diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index f818efe9..7a3e1b24 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -364,6 +364,28 @@ func TestJournalctlVacuumFormatsSingularResult(t *testing.T) { assert.Equal(t, "Vacuuming done, freed 1.0K from 1 archived journal file.\n", stdout.String()) } +func TestJournalctlVacuumSizeRequiresTimeFloorBeforeAuthorization(t *testing.T) { + cleaner := &fakeJournalCleaner{} + var stdout, stderr bytes.Buffer + authorized := 0 + result := runJournalctl(t, []string{"--vacuum-size", "64M"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + Now: time.Now(), + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + authorized++ + return nil + }, + Systemd: &builtins.SystemdServices{JournalCleaner: cleaner}, + }) + + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) + assert.Equal(t, "journalctl: --vacuum-size requires --vacuum-time to protect recently modified archives\n", stderr.String()) + assert.Zero(t, authorized) + assert.Empty(t, cleaner.requests) +} + func TestJournalctlMaintenanceRejectsInvalidOrMixedOptionsBeforeAuthorization(t *testing.T) { tests := [][]string{ {"--dry-run"}, diff --git a/builtins/systemd.go b/builtins/systemd.go index 189063a0..c47de256 100644 --- a/builtins/systemd.go +++ b/builtins/systemd.go @@ -61,7 +61,8 @@ type JournalStorageReader interface { } // JournalVacuumRequest contains only bounded cleanup predicates. Before is an -// absolute archive mtime cutoff; MaxBytes is an allocated archived-byte target. +// absolute archive mtime cutoff. MaxBytes is an allocated archived-byte target +// that may only delete archives at or before the cutoff. type JournalVacuumRequest struct { Now time.Time Before time.Time diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go index 084e5e86..88513822 100644 --- a/internal/systemd/journal_vacuum_unix.go +++ b/internal/systemd/journal_vacuum_unix.go @@ -43,6 +43,9 @@ func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacu if request.MaxBytes == 0 && request.Before.IsZero() { return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum requires a size or time limit") } + if request.MaxBytes > 0 && request.Before.IsZero() { + return builtins.JournalVacuumResult{}, fmt.Errorf("journal size vacuum requires a time cutoff") + } if !request.Before.IsZero() && request.Before.After(request.Now) { return builtins.JournalVacuumResult{}, fmt.Errorf("journal vacuum time cutoff cannot be in the future") } @@ -64,9 +67,9 @@ func (c *Client) VacuumJournal(ctx context.Context, request builtins.JournalVacu if err := ctx.Err(); err != nil { return result, vacuumPartialError(result, err) } - expired := !request.Before.IsZero() && !candidate.modTime.After(request.Before) + oldEnough := !candidate.modTime.After(request.Before) overSize := request.MaxBytes > 0 && remainingBytes > request.MaxBytes - if !expired && !overSize { + if !oldEnough || (request.MaxBytes > 0 && !overSize) { break } if err := revalidateVacuumCandidate(candidate); err != nil { diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go index 7520d418..a3f6fff9 100644 --- a/internal/systemd/journal_vacuum_unix_test.go +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -88,24 +88,28 @@ func TestVacuumJournalDryRunDoesNotDelete(t *testing.T) { assert.FileExists(t, second) } -func TestVacuumJournalHonorsAllocatedSizeTarget(t *testing.T) { +func TestVacuumJournalHonorsAllocatedSizeTargetAndTimeFloor(t *testing.T) { now := time.Now() client, directory := newVacuumTestClient(t) oldest := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) second := writeVacuumFile(t, directory, archivedJournalName(2), now.Add(-9*24*time.Hour)) + recent := writeVacuumFile(t, directory, archivedJournalName(3), now.Add(-time.Hour)) info, err := os.Lstat(oldest) require.NoError(t, err) stat, err := journalStat(info) require.NoError(t, err) + require.Greater(t, stat.allocated, uint64(1)) result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ Now: now, - MaxBytes: stat.allocated, + MaxBytes: stat.allocated - 1, + Before: now.Add(-48 * time.Hour), }) require.NoError(t, err) - assert.Equal(t, 1, result.Files) + assert.Equal(t, 2, result.Files) assert.NoFileExists(t, oldest) - assert.FileExists(t, second) + assert.NoFileExists(t, second) + assert.FileExists(t, recent) } func TestVacuumJournalSkipsHardlinksAndSymlinks(t *testing.T) { @@ -162,6 +166,10 @@ func TestVacuumJournalRequiresRequestBounds(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "requires a size or time limit") + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, MaxBytes: 1}) + require.Error(t, err) + assert.Contains(t, err.Error(), "size vacuum requires a time cutoff") + _, err = client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{Now: now, Before: now.Add(time.Hour)}) require.Error(t, err) assert.Contains(t, err.Error(), "cannot be in the future") diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml index fc54ee24..8b1d784b 100644 --- a/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_denied.yaml @@ -4,7 +4,7 @@ description: journalctl vacuum requires a journald clean grant input: mode: remediation script: |+ - journalctl --vacuum-size=64M + journalctl --vacuum-size=64M --vacuum-time=168h expect: stdout: |+ stderr: |+ diff --git a/tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml b/tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml new file mode 100644 index 00000000..5db472d2 --- /dev/null +++ b/tests/scenarios/cmd/journalctl/errors/vacuum_size_without_time.yaml @@ -0,0 +1,13 @@ +# Size-only cleanup could delete newly rotated archives, so rshell requires an +# explicit retention floor before checking mutation authorization. +skip_assert_against_bash: true +description: journalctl size vacuum requires a time retention floor +input: + mode: remediation + script: |+ + journalctl --vacuum-size=64M +expect: + stdout: |+ + stderr: |+ + journalctl: --vacuum-size requires --vacuum-time to protect recently modified archives + exit_code: 1 From 86bd4888bf258406372bcdd0345476e8d94a7492 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Wed, 15 Jul 2026 16:37:43 -0400 Subject: [PATCH 35/53] fix(cli): align systemd grant parsing with API --- README.md | 2 +- SHELL_FEATURES.md | 2 +- cmd/rshell/main.go | 25 +++++++++-------- cmd/rshell/main_test.go | 60 +++++++++++++++++++++-------------------- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index f7a29fd4..cad51a70 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Every access path is default-deny: **AllowedCommands** restricts which commands (builtins or external) the interpreter may execute. Commands must be specified with the `rshell:` namespace prefix (e.g. `rshell:cat`, `rshell:echo`). If not set, no commands are allowed. -**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact service with generic actions (`read`, `clean`, `reload`, or `restart`), using `SERVICE:ACTION[+ACTION...]` syntax. Invalid services and unsupported actions are skipped with warnings. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different services. Empty service names, service names containing `:`, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. +**AllowedSystemServices** is the single capability policy shared by systemd-aware builtins. Grants pair one exact service with generic actions (`read`, `clean`, `reload`, or `restart`), using `SERVICE:ACTION[+ACTION...]` syntax. Grants without actions are ignored; invalid services and unsupported actions are skipped with warnings. Service names are matched exactly without adding suffixes, resolving aliases, changing case, or otherwise normalizing them: `mysql` and `mysql.service` are different services. Empty service names, service names containing `:`, whitespace, path-like names, and glob patterns are also skipped with warnings. The policy defaults to denying every operation and remains enforced when all commands are allowed. `read` is available in read-only mode; mutating actions require remediation mode. ```go interp.AllowedSystemServices([]interp.SystemdControlGrant{ diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 6ad36af6..85ee2497 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -122,7 +122,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c ## Execution - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed -- ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services with generic `read`, `clean`, `reload`, and `restart` actions; invalid services/actions are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` +- ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services with generic `read`, `clean`, `reload`, and `restart` actions; actionless grants are ignored, invalid services/actions are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` - ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command diff --git a/cmd/rshell/main.go b/cmd/rshell/main.go index aa62ccd1..fd41174d 100644 --- a/cmd/rshell/main.go +++ b/cmd/rshell/main.go @@ -90,10 +90,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. cmds = strings.Split(allowedCommands, ",") } - serviceGrants, err := parseAllowedServices(allowedServices) - if err != nil { - return err - } + serviceGrants := parseAllowedServices(allowedServices) parsedMode := interp.Mode(mode) if parsedMode != interp.ModeReadOnly && parsedMode != interp.ModeRemediation { return fmt.Errorf("--mode must be one of: read-only, remediation") @@ -297,29 +294,31 @@ func execute(ctx context.Context, script, name string, opts executeOpts, stdin i return runner.Run(ctx, prog) } -func parseAllowedServices(value string) ([]interp.SystemdControlGrant, error) { +func parseAllowedServices(value string) []interp.SystemdControlGrant { if value == "" { - return nil, nil + return nil } entries := strings.Split(value, ",") grants := make([]interp.SystemdControlGrant, 0, len(entries)) for _, entry := range entries { separator := strings.LastIndexByte(entry, ':') - if separator <= 0 || separator == len(entry)-1 { - return nil, fmt.Errorf("--allowed-services: invalid grant %q (expected SERVICE:ACTION[+ACTION...])", entry) + if separator < 0 { + grants = append(grants, interp.SystemdControlGrant{Service: entry}) + continue } - actionNames := strings.Split(entry[separator+1:], "+") + actionSpec := entry[separator+1:] + var actionNames []string + if actionSpec != "" { + actionNames = strings.Split(actionSpec, "+") + } actions := make([]interp.SystemdAction, len(actionNames)) for i, action := range actionNames { actions[i] = interp.SystemdAction(action) } selector := entry[:separator] - if strings.ContainsRune(selector, ':') { - return nil, fmt.Errorf("--allowed-services: invalid service selector %q (service names must not contain ':')", selector) - } grants = append(grants, interp.SystemdControlGrant{Service: selector, Actions: actions}) } - return grants, nil + return grants } diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index cb86b023..a06c4396 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -305,14 +305,19 @@ func TestAllowedServicesFlag(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedServicesFlagRejectsInvalidGrant(t *testing.T) { - code, _, stderr := runCLI(t, - "--allow-all-commands", - "--allowed-services", "mysql.service", - "-c", `echo hello`, - ) - assert.Equal(t, 1, code) - assert.Contains(t, stderr, "invalid grant") +func TestAllowedServicesFlagIgnoresGrantsWithoutActions(t *testing.T) { + for _, grant := range []string{"mysql.service", "mysql.service:"} { + t.Run(grant, func(t *testing.T) { + code, stdout, stderr := runCLI(t, + "--allow-all-commands", + "--allowed-services", grant, + "-c", `echo hello`, + ) + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Empty(t, stderr) + }) + } } func TestAllowedServicesFlagWarnsAndSkipsUnknownAction(t *testing.T) { @@ -339,17 +344,24 @@ func TestAllowedServicesFlagWarnsAndSkipsInvalidService(t *testing.T) { } func TestParseAllowedServicesParsesServiceActions(t *testing.T) { - grants, err := parseAllowedServices("systemd-journald.service:read+clean") - require.NoError(t, err) + grants := parseAllowedServices("systemd-journald.service:read+clean") require.Len(t, grants, 1) assert.Equal(t, "systemd-journald.service", grants[0].Service) assert.Equal(t, []interp.SystemServiceAction{interp.SystemServiceRead, interp.SystemdActionClean}, grants[0].Actions) } -func TestParseAllowedServicesRejectsColonInServiceSelector(t *testing.T) { - _, err := parseAllowedServices("tenant:mysql.service:read") - require.Error(t, err) - assert.Contains(t, err.Error(), `invalid service selector "tenant:mysql.service"`) +func TestParseAllowedServicesPreservesEntriesForPolicyValidation(t *testing.T) { + grants := parseAllowedServices("ignored.service,tenant:mysql.service:read,mysql.service:read+stop") + require.Len(t, grants, 3) + assert.Equal(t, interp.SystemdControlGrant{Service: "ignored.service"}, grants[0]) + assert.Equal(t, interp.SystemdControlGrant{ + Service: "tenant:mysql.service", + Actions: []interp.SystemServiceAction{interp.SystemServiceRead}, + }, grants[1]) + assert.Equal(t, interp.SystemdControlGrant{ + Service: "mysql.service", + Actions: []interp.SystemServiceAction{interp.SystemServiceRead, "stop"}, + }, grants[2]) } func TestAllowedServicesFlagAcceptsServiceGrants(t *testing.T) { @@ -364,26 +376,16 @@ func TestAllowedServicesFlagAcceptsServiceGrants(t *testing.T) { assert.Empty(t, stderr) } -func TestAllowedServicesFlagRejectsColonInSelector(t *testing.T) { - code, stdout, stderr := runCLI(t, - "--allow-all-commands", - "--allowed-services", "tenant:mysql.service:read", - "-c", `echo hello`, - ) - assert.Equal(t, 1, code) - assert.Empty(t, stdout) - assert.Contains(t, stderr, `invalid service selector "tenant:mysql.service"`) -} - -func TestAllowedServicesFlagRejectsColonInServiceSelector(t *testing.T) { +func TestAllowedServicesFlagWarnsAndSkipsColonInSelector(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", "--allowed-services", "tenant:mysql.service:read", "-c", `echo hello`, ) - assert.Equal(t, 1, code) - assert.Empty(t, stdout) - assert.Contains(t, stderr, "service names must not contain ':'") + assert.Equal(t, 0, code) + assert.Equal(t, "hello\n", stdout) + assert.Contains(t, stderr, "AllowedSystemServices: skipping") + assert.Contains(t, stderr, "must not contain ':'") } func TestSystemdRootFlag(t *testing.T) { From c3aea985d56fb9e47e95231b4ff9ebcb1ecaa869 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 09:44:48 -0400 Subject: [PATCH 36/53] test(cli): use portable systemd root --- cmd/rshell/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/rshell/main_test.go b/cmd/rshell/main_test.go index a06c4396..6ff8f00c 100644 --- a/cmd/rshell/main_test.go +++ b/cmd/rshell/main_test.go @@ -391,7 +391,7 @@ func TestAllowedServicesFlagWarnsAndSkipsColonInSelector(t *testing.T) { func TestSystemdRootFlag(t *testing.T) { code, stdout, stderr := runCLI(t, "--allow-all-commands", - "--systemd-root", "/host", + "--systemd-root", t.TempDir(), "-c", `echo hello`, ) assert.Equal(t, 0, code) From 10fe842844928e15777ae57bec8ba36cbf22e042 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 10:09:47 -0400 Subject: [PATCH 37/53] fix(journalctl): clarify combined vacuum semantics --- README.md | 2 +- SHELL_FEATURES.md | 2 +- builtins/journalctl/journalctl.go | 6 +++--- builtins/journalctl/journalctl_test.go | 2 ++ internal/systemd/journal_vacuum_unix_test.go | 19 +++++++++++++++++++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cad51a70..3b344393 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Log queries default to 100 entries and are capped at 1,000 entries and 32 exact Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. -The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. `--vacuum-size` requires `--vacuum-time`; size cleanup deletes only archives at least that old and stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. +The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. Used alone, `--vacuum-time` removes every eligible archive at least that old. `--vacuum-size` requires `--vacuum-time`; when combined, the age becomes a minimum deletion age and cleanup stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 85ee2497..0e30403f 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; size vacuum requires a time floor and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index 09a2c7a6..48b85258 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -25,8 +25,8 @@ // --rotate archive active journal files before returning // --vacuum-size=SIZE remove oldest eligible archives toward allocated SIZE; // requires --vacuum-time -// --vacuum-time=AGE remove archives older than Go duration AGE and limit -// size vacuum eligibility +// --vacuum-time=AGE alone, remove archives older than AGE; with +// --vacuum-size, set the minimum deletion age // --dry-run report cleanup without deleting archives // -h, --help print usage and exit package journalctl @@ -83,7 +83,7 @@ func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), rotate: fs.Bool("rotate", false, "archive active journal files and wait for completion"), vacuumSize: fs.String("vacuum-size", "", "remove oldest eligible archives toward allocated SIZE; requires --vacuum-time"), - vacuumTime: fs.String("vacuum-time", "", "remove archives older than Go duration AGE and limit size vacuum eligibility"), + vacuumTime: fs.String("vacuum-time", "", "alone, remove archives older than AGE; with --vacuum-size, set the minimum deletion age"), dryRun: fs.Bool("dry-run", false, "report cleanup without deleting archives"), help: fs.BoolP("help", "h", false, "print usage and exit"), } diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index 7a3e1b24..5a29126f 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -572,6 +572,8 @@ func TestJournalctlHelpDoesNotRequireSystemdCapability(t *testing.T) { assert.Contains(t, stdout.String(), "Usage: journalctl") assert.Contains(t, stdout.String(), "--unit") assert.Contains(t, stdout.String(), "--rotate") + assert.Contains(t, stdout.String(), "--vacuum-time") + assert.Contains(t, stdout.String(), "alone, remove archives older than AGE; with --vacuum-size, set the minimum deletion age") assert.Empty(t, stderr.String()) } diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go index a3f6fff9..02cfecf7 100644 --- a/internal/systemd/journal_vacuum_unix_test.go +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -112,6 +112,25 @@ func TestVacuumJournalHonorsAllocatedSizeTargetAndTimeFloor(t *testing.T) { assert.FileExists(t, recent) } +func TestVacuumJournalCombinedCleanupStopsAtSizeTarget(t *testing.T) { + now := time.Now() + client, directory := newVacuumTestClient(t) + archive := writeVacuumFile(t, directory, archivedJournalName(1), now.Add(-10*24*time.Hour)) + info, err := os.Lstat(archive) + require.NoError(t, err) + stat, err := journalStat(info) + require.NoError(t, err) + + result, err := client.VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + MaxBytes: stat.allocated, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Zero(t, result.Files) + assert.FileExists(t, archive) +} + func TestVacuumJournalSkipsHardlinksAndSymlinks(t *testing.T) { now := time.Now() client, directory := newVacuumTestClient(t) From c91e640a1a806ac5d81e518f0243687faa1f97c4 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 10:26:36 -0400 Subject: [PATCH 38/53] fix(systemd): stop bounded journal scans --- internal/systemd/journal_query_file.go | 6 ++- internal/systemd/journal_query_file_test.go | 48 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/internal/systemd/journal_query_file.go b/internal/systemd/journal_query_file.go index e1ede5c3..0e345402 100644 --- a/internal/systemd/journal_query_file.go +++ b/internal/systemd/journal_query_file.go @@ -190,11 +190,13 @@ func (i *journalFileQueryIterator) previous(ctx context.Context) (journalQueryEn if err != nil || !found { return journalQueryEntry{}, found, err } + // Candidate indexes follow journal append order. Treat boot and time + // bounds as lower seek boundaries once reverse traversal crosses them. if i.filterBoot && entry.bootID != i.bootID { - continue + return journalQueryEntry{}, false, nil } if i.sinceUsec != 0 && entry.realtime < i.sinceUsec { - continue + return journalQueryEntry{}, false, nil } if entry.seqnum == 0 { return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY sequence number is zero") diff --git a/internal/systemd/journal_query_file_test.go b/internal/systemd/journal_query_file_test.go index e2259fd1..d800f416 100644 --- a/internal/systemd/journal_query_file_test.go +++ b/internal/systemd/journal_query_file_test.go @@ -122,6 +122,54 @@ func TestJournalFileQuerySupportsCompactKeyedLayout(t *testing.T) { assert.Equal(t, "first", entries[1].selected.Message) } +func TestJournalFileQueryStopsAtCurrentBootBoundaryBeforeScanLimit(t *testing.T) { + oldBoot := repeatedJournalID(0x11) + currentBoot := repeatedJournalID(0x22) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: oldBoot, realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=oldest"}}, + {bootID: oldBoot, realtime: 1_700_000_000_000_001, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=old"}}, + {bootID: currentBoot, realtime: 1_700_000_000_000_002, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=current"}}, + }) + view, err := newJournalFileView("boot-boundary.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"api.service"}, + CurrentBoot: true, + MaxEntries: 10, + }, ¤tBoot) + require.NoError(t, err) + iterator.scanned = maxJournalCandidatesScanned - 2 + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 1) + assert.Equal(t, "current", entries[0].selected.Message) + assert.Equal(t, maxJournalCandidatesScanned, iterator.scanned) +} + +func TestJournalFileQueryStopsAtSinceBoundaryBeforeScanLimit(t *testing.T) { + bootID := repeatedJournalID(0x33) + start := time.Unix(1_700_000_000, 0) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: bootID, realtime: uint64(start.UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=oldest"}}, + {bootID: bootID, realtime: uint64(start.Add(time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=old"}}, + {bootID: bootID, realtime: uint64(start.Add(2 * time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=current"}}, + }) + view, err := newJournalFileView("since-boundary.journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ + Units: []string{"api.service"}, + Since: start.Add(2 * time.Second), + MaxEntries: 10, + }, nil) + require.NoError(t, err) + iterator.scanned = maxJournalCandidatesScanned - 2 + + entries := collectJournalQueryEntries(t, iterator) + require.Len(t, entries, 1) + assert.Equal(t, "current", entries[0].selected.Message) + assert.Equal(t, maxJournalCandidatesScanned, iterator.scanned) +} + func TestJournalFileQueryHonorsCancellation(t *testing.T) { bootID := repeatedJournalID(0x44) contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ From 1d8089a1a3732d2d55b2af83b0e94ffa14dfa8d5 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 11:31:29 -0400 Subject: [PATCH 39/53] fix(systemd): reject symlinked journal directories --- internal/systemd/journal_files.go | 32 +++++++++++++++++++++++++- internal/systemd/journal_files_test.go | 22 ++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/internal/systemd/journal_files.go b/internal/systemd/journal_files.go index de90ab1c..60c6d2af 100644 --- a/internal/systemd/journal_files.go +++ b/internal/systemd/journal_files.go @@ -36,7 +36,7 @@ func (c *Client) journalFiles() (string, []string, error) { files := make([]string, 0) for _, baseDir := range c.target.JournalDirs { dirPath := filepath.Join(baseDir, machineID) - dir, err := os.Open(dirPath) + dir, err := openJournalMachineDirectory(dirPath) if err != nil { if os.IsNotExist(err) { continue @@ -78,6 +78,36 @@ func (c *Client) journalFiles() (string, []string, error) { return machineID, files, nil } +func openJournalMachineDirectory(path string) (*os.File, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !before.IsDir() || before.Mode()&fs.ModeSymlink != 0 { + return nil, fmt.Errorf("journal machine directory %q is not a real directory", path) + } + + dir, err := os.Open(path) + if err != nil { + return nil, err + } + opened, err := dir.Stat() + if err != nil { + dir.Close() + return nil, err + } + after, err := os.Lstat(path) + if err != nil { + dir.Close() + return nil, err + } + if !opened.IsDir() || !after.IsDir() || after.Mode()&fs.ModeSymlink != 0 || !os.SameFile(before, opened) || !os.SameFile(after, opened) { + dir.Close() + return nil, fmt.Errorf("journal machine directory %q changed while opening", path) + } + return dir, nil +} + func readMachineID(path string) (string, error) { file, err := os.Open(path) if err != nil { diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go index ecb09111..53584231 100644 --- a/internal/systemd/journal_files_test.go +++ b/internal/systemd/journal_files_test.go @@ -75,3 +75,25 @@ func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "journal directories are not configured") } + +func TestJournalFilesRejectsSymlinkedMachineDirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires elevated privileges on Windows") + } + + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + machineIDPath := filepath.Join(root, "machine-id") + require.NoError(t, os.WriteFile(machineIDPath, []byte(machineID+"\n"), 0o600)) + baseDir := filepath.Join(root, "journal") + outsideDir := filepath.Join(root, "outside") + require.NoError(t, os.Mkdir(baseDir, 0o700)) + require.NoError(t, os.Mkdir(outsideDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(outsideDir, "system.journal"), nil, 0o600)) + require.NoError(t, os.Symlink(outsideDir, filepath.Join(baseDir, machineID))) + + client := NewClient(Target{JournalDirs: []string{baseDir}, MachineIDPath: machineIDPath}) + _, _, err := client.journalFiles() + require.Error(t, err) + assert.Contains(t, err.Error(), "is not a real directory") +} From a61fe6addf95a9f2c64a24cd4cde940b241269a4 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 11:44:28 -0400 Subject: [PATCH 40/53] fix(systemd): preserve entries across clock steps --- internal/systemd/journal_query_file.go | 6 +++--- internal/systemd/journal_query_file_test.go | 18 ++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/internal/systemd/journal_query_file.go b/internal/systemd/journal_query_file.go index 0e345402..d42fca55 100644 --- a/internal/systemd/journal_query_file.go +++ b/internal/systemd/journal_query_file.go @@ -190,13 +190,13 @@ func (i *journalFileQueryIterator) previous(ctx context.Context) (journalQueryEn if err != nil || !found { return journalQueryEntry{}, found, err } - // Candidate indexes follow journal append order. Treat boot and time - // bounds as lower seek boundaries once reverse traversal crosses them. + // Boot IDs form append-order segments, so earlier candidates cannot + // re-enter the current boot once reverse traversal leaves it. if i.filterBoot && entry.bootID != i.bootID { return journalQueryEntry{}, false, nil } if i.sinceUsec != 0 && entry.realtime < i.sinceUsec { - return journalQueryEntry{}, false, nil + continue } if entry.seqnum == 0 { return journalQueryEntry{}, false, journalCorrupt(i.file.name, entry.offset, "ENTRY sequence number is zero") diff --git a/internal/systemd/journal_query_file_test.go b/internal/systemd/journal_query_file_test.go index d800f416..a6257232 100644 --- a/internal/systemd/journal_query_file_test.go +++ b/internal/systemd/journal_query_file_test.go @@ -146,28 +146,26 @@ func TestJournalFileQueryStopsAtCurrentBootBoundaryBeforeScanLimit(t *testing.T) assert.Equal(t, maxJournalCandidatesScanned, iterator.scanned) } -func TestJournalFileQueryStopsAtSinceBoundaryBeforeScanLimit(t *testing.T) { +func TestJournalFileQueryContinuesPastRealtimeClockStep(t *testing.T) { bootID := repeatedJournalID(0x33) start := time.Unix(1_700_000_000, 0) contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ - {bootID: bootID, realtime: uint64(start.UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=oldest"}}, - {bootID: bootID, realtime: uint64(start.Add(time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=old"}}, - {bootID: bootID, realtime: uint64(start.Add(2 * time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=current"}}, + {bootID: bootID, realtime: uint64(start.Add(10 * time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=before clock step"}}, + {bootID: bootID, realtime: uint64(start.Add(5 * time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=before since"}}, + {bootID: bootID, realtime: uint64(start.Add(12 * time.Second).UnixMicro()), fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=latest"}}, }) - view, err := newJournalFileView("since-boundary.journal", bytes.NewReader(contents), uint64(len(contents))) + view, err := newJournalFileView("realtime-clock-step.journal", bytes.NewReader(contents), uint64(len(contents))) require.NoError(t, err) iterator, err := newJournalFileQueryIterator(view, builtins.JournalQuery{ Units: []string{"api.service"}, - Since: start.Add(2 * time.Second), + Since: start.Add(8 * time.Second), MaxEntries: 10, }, nil) require.NoError(t, err) - iterator.scanned = maxJournalCandidatesScanned - 2 entries := collectJournalQueryEntries(t, iterator) - require.Len(t, entries, 1) - assert.Equal(t, "current", entries[0].selected.Message) - assert.Equal(t, maxJournalCandidatesScanned, iterator.scanned) + require.Len(t, entries, 2) + assert.Equal(t, []string{"latest", "before clock step"}, []string{entries[0].selected.Message, entries[1].selected.Message}) } func TestJournalFileQueryHonorsCancellation(t *testing.T) { From da013522716fb1e518dbed65ef3d7568a79423dd Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 12:09:12 -0400 Subject: [PATCH 41/53] docs(systemd): align vacuum safety rules --- builtins/systemd.go | 2 +- docs/RULES.md | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/builtins/systemd.go b/builtins/systemd.go index c47de256..a689fe48 100644 --- a/builtins/systemd.go +++ b/builtins/systemd.go @@ -77,7 +77,7 @@ type JournalVacuumResult struct { Bytes uint64 } -// JournalCleaner performs policy-bounded cleanup of archived journal files. +// JournalCleaner performs request-bounded cleanup of archived journal files. type JournalCleaner interface { VacuumJournal(ctx context.Context, request JournalVacuumRequest) (JournalVacuumResult, error) } diff --git a/docs/RULES.md b/docs/RULES.md index 64449122..cebb282b 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -88,14 +88,18 @@ bounds. Builtins receive selected fields only and never receive a raw journal handle, target path, or arbitrary field-match capability. The only deletion exception is `JournalCleaner.VacuumJournal`. It is available -only through trusted systemd target configuration and a validated operator -vacuum policy. The backend pins each configured journal directory with `os.Root`, -checks directory identity across open, accepts only strict systemd archived-file -names, excludes symlinks and hardlinks, and revalidates file identity immediately -before rooted removal. Active, malformed, recently modified, and policy-retained -files must never be deleted. Every cleanup call has both file-count and byte -ceilings in addition to the exact `systemd-journald.service:clean` authorization -and remediation-mode requirement. +only through trusted systemd target configuration and a validated +`JournalVacuumRequest`. Vacuum thresholds come from the command request rather +than a separate operator policy. Every deletion request includes an absolute +modification-time cutoff, and size-based cleanup additionally includes an +allocated-byte target and is rejected without the cutoff. The backend pins each +configured journal directory with `os.Root`, checks directory identity across +open, accepts only strict systemd archived-file names, excludes symlinks and +hardlinks, and revalidates file identity immediately before rooted removal. +Active files, malformed files, and files newer than the request cutoff must +never be deleted. Fixed discovery bounds, cancellation checks, and +partial-progress errors bound each cleanup invocation, in addition to the exact +`systemd-journald.service:clean` authorization and remediation-mode requirement. `JournalRotator.RotateJournal` is the only journal-daemon mutation exception. It may call only the fixed `io.systemd.Journal.Rotate` Varlink method through From f646b968032fdfb2e6aa2d9ba70b8eb036e9aaa9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 12:22:01 -0400 Subject: [PATCH 42/53] fix(journalctl): cap unique unit scopes --- builtins/journalctl/journalctl.go | 4 ++-- builtins/journalctl/journalctl_test.go | 30 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index 48b85258..09a50ecc 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -110,12 +110,12 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { if *options.rotate || fs.Changed("vacuum-size") || fs.Changed("vacuum-time") || *options.dryRun { return options.runMaintenance(ctx, callCtx, fs) } - if len(*options.units) > builtins.MaxJournalQueryUnits { + units := uniqueUnits(*options.units) + if len(units) > builtins.MaxJournalQueryUnits { callCtx.Errf("journalctl: too many unit scopes (maximum %d)\n", builtins.MaxJournalQueryUnits) return builtins.Result{Code: 1} } - units := uniqueUnits(*options.units) if *options.kernel && len(units) > 0 { callCtx.Errf("journalctl: --dmesg cannot be combined with --unit\n") return builtins.Result{Code: 1} diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index 5a29126f..d5ac55dd 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -554,14 +554,42 @@ func TestJournalctlRejectsDangerousJournalctlOptions(t *testing.T) { } } -func TestJournalctlLimitsRepeatedUnitScopes(t *testing.T) { +func TestJournalctlDeduplicatesRepeatedUnitScopesBeforeLimit(t *testing.T) { args := make([]string, 0, (builtins.MaxJournalQueryUnits+1)*2) for i := 0; i <= builtins.MaxJournalQueryUnits; i++ { args = append(args, "-u", "api.service") } + reader := &fakeJournalReader{} + var stdout, stderr bytes.Buffer + var authorized []builtins.SystemdOperation + result := runJournalctl(t, args, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { + authorized = append(authorized, operations...) + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stdout.String()) + assert.Empty(t, stderr.String()) + assert.Equal(t, []builtins.SystemdOperation{{Service: "api.service", Action: builtins.SystemdActionRead}}, authorized) + require.Len(t, reader.queries, 1) + assert.Equal(t, []string{"api.service"}, reader.queries[0].Units) +} + +func TestJournalctlLimitsUniqueUnitScopes(t *testing.T) { + args := make([]string, 0, (builtins.MaxJournalQueryUnits+1)*2) + for i := 0; i <= builtins.MaxJournalQueryUnits; i++ { + args = append(args, "-u", strings.Repeat("a", i+1)+".service") + } var stdout, stderr bytes.Buffer result := runJournalctl(t, args, &builtins.CallContext{Stdout: &stdout, Stderr: &stderr}) + assert.Equal(t, uint8(1), result.Code) + assert.Empty(t, stdout.String()) assert.Contains(t, stderr.String(), "too many unit scopes") } From e7f18a3fd4f45fc9974a54a1ea059cf607def220 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 12:22:29 -0400 Subject: [PATCH 43/53] ci: remove obsolete systemd headers --- .github/workflows/compliance.yml | 2 -- .github/workflows/test.yml | 5 ----- 2 files changed, 7 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 04c66313..6294fd04 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -18,8 +18,6 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version - - name: Install systemd development headers - run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run compliance checks env: RSHELL_COMPLIANCE_TEST: "1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3ab86d2..1d909e9b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,9 +23,6 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version - - name: Install systemd development headers - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run tests with race detector run: go test -race -v -timeout 10m ./... @@ -72,8 +69,6 @@ jobs: - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: .go-version - - name: Install systemd development headers - run: sudo apt-get update && sudo apt-get install -y libsystemd-dev - name: Run bash comparison tests env: RSHELL_BASH_TEST: "1" From 2da0538d2bdbe533a1673eb92cf75d8802bf8675 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 12:35:31 -0400 Subject: [PATCH 44/53] docs(journalctl): clarify sanitized cat output --- README.md | 2 ++ SHELL_FEATURES.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b344393..97fa1523 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ Root targets derive those paths below `Root`; explicit targets may map them to a Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. +Both `short` and `cat` escape embedded newlines, carriage returns, tabs, non-graphic Unicode code points, and invalid UTF-8 bytes before writing output. Rshell's `-o cat` therefore emits only the sanitized `MESSAGE` field; unlike host `journalctl -o cat`, it never writes raw message bytes. + Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. Used alone, `--vacuum-time` removes every eligible archive at least that old. `--vacuum-size` requires `--vacuum-time`; when combined, the age becomes a minimum deletion age and cleanup stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 0e30403f..71411a8e 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; both output formats sanitize controls, non-graphic Unicode, and invalid UTF-8, so rshell's `cat` output is not raw-output-compatible with host `journalctl`; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected From 5bed79e29bc41df4878f7ca87a64a00c9b414752 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 12:48:53 -0400 Subject: [PATCH 45/53] fix(journalctl): preserve raw cat output --- README.md | 2 +- SHELL_FEATURES.md | 2 +- builtins/journalctl/journalctl.go | 50 ++++++++++++++------------ builtins/journalctl/journalctl_test.go | 24 +++++++++++-- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 97fa1523..1af218dd 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Root targets derive those paths below `Root`; explicit targets may map them to a Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. -Both `short` and `cat` escape embedded newlines, carriage returns, tabs, non-graphic Unicode code points, and invalid UTF-8 bytes before writing output. Rshell's `-o cat` therefore emits only the sanitized `MESSAGE` field; unlike host `journalctl -o cat`, it never writes raw message bytes. +Selected journal fields are capped at 64 KiB. `short` escapes embedded newlines, carriage returns, tabs, non-graphic Unicode code points, and invalid UTF-8 bytes before writing output. Within that bound, `cat` matches host `journalctl -o cat` by writing each `MESSAGE` value unchanged and appending one newline; callers must treat that raw output as untrusted log content. Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 71411a8e..13a4c4f1 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries and 32 service scopes; both output formats sanitize controls, non-graphic Unicode, and invalid UTF-8, so rshell's `cat` output is not raw-output-compatible with host `journalctl`; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries, 32 service scopes, and 64 KiB per selected field; `short` sanitizes controls, non-graphic Unicode, and invalid UTF-8, while `cat` matches host `journalctl` within the field bound by writing each `MESSAGE` value unchanged and appending one newline; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index 09a50ecc..c21be6b7 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -20,7 +20,7 @@ // -n, --lines=COUNT show at most COUNT entries (default 100, maximum 1000) // -S, --since=TIME show entries since an RFC3339 timestamp, local // YYYY-MM-DD HH:MM:SS timestamp, or lookback duration -// -o, --output=FORMAT output format: short (default) or cat +// -o, --output=FORMAT output format: escaped short (default) or raw cat // --disk-usage show allocated journal storage and exit // --rotate archive active journal files before returning // --vacuum-size=SIZE remove oldest eligible archives toward allocated SIZE; @@ -79,7 +79,7 @@ func makeFlags(fs *builtins.FlagSet) builtins.HandlerFunc { boot: fs.BoolP("boot", "b", false, "show messages from the current boot"), lines: fs.StringP("lines", "n", "100", "show at most COUNT entries (maximum 1000)"), since: fs.StringP("since", "S", "", "show entries newer than TIME or lookback duration"), - output: fs.StringP("output", "o", "short", "output format: short or cat"), + output: fs.StringP("output", "o", "short", "output format: escaped short or raw cat"), usage: fs.Bool("disk-usage", false, "show allocated journal storage and exit"), rotate: fs.Bool("rotate", false, "archive active journal files and wait for completion"), vacuumSize: fs.String("vacuum-size", "", "remove oldest eligible archives toward allocated SIZE; requires --vacuum-time"), @@ -382,28 +382,34 @@ func parseSince(value string, now time.Time) (time.Time, bool) { } func writeEntry(writer io.Writer, entry builtins.JournalEntry, output string) error { - var line strings.Builder - if output == "short" { - line.WriteString(entry.Timestamp.Format(shortTime)) - line.WriteByte(' ') - if entry.Hostname == "" { - line.WriteByte('-') - } else { - appendEscaped(&line, entry.Hostname) - } - line.WriteByte(' ') - if entry.Identifier == "" { - line.WriteString("journal") - } else { - appendEscaped(&line, entry.Identifier) + if output == "cat" { + if _, err := io.WriteString(writer, entry.Message); err != nil { + return err } - if entry.PID != "" { - line.WriteByte('[') - appendEscaped(&line, entry.PID) - line.WriteByte(']') - } - line.WriteString(": ") + _, err := io.WriteString(writer, "\n") + return err } + + var line strings.Builder + line.WriteString(entry.Timestamp.Format(shortTime)) + line.WriteByte(' ') + if entry.Hostname == "" { + line.WriteByte('-') + } else { + appendEscaped(&line, entry.Hostname) + } + line.WriteByte(' ') + if entry.Identifier == "" { + line.WriteString("journal") + } else { + appendEscaped(&line, entry.Identifier) + } + if entry.PID != "" { + line.WriteByte('[') + appendEscaped(&line, entry.PID) + line.WriteByte(']') + } + line.WriteString(": ") appendEscaped(&line, entry.Message) line.WriteByte('\n') _, err := io.WriteString(writer, line.String()) diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index d5ac55dd..cb08efcc 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -129,7 +129,7 @@ func TestJournalctlBuildsBoundedUnitQuery(t *testing.T) { }, reader.queries[0]) } -func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { +func TestJournalctlShortOutputEscapesTerminalControls(t *testing.T) { reader := &fakeJournalReader{entries: []builtins.JournalEntry{{ Timestamp: time.Date(2026, time.July, 14, 12, 34, 56, 0, time.UTC), Hostname: "host\tname", @@ -139,7 +139,7 @@ func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { }}} var stdout, stderr bytes.Buffer var authorized []builtins.SystemdOperation - result := runJournalctl(t, []string{"-k", "-n1"}, &builtins.CallContext{ + result := runJournalctl(t, []string{"-k", "-n1", "--output", "short"}, &builtins.CallContext{ Stdout: &stdout, Stderr: &stderr, AuthorizeSystemd: func(operations ...builtins.SystemdOperation) error { @@ -161,6 +161,25 @@ func TestJournalctlKernelQueryEscapesTerminalControls(t *testing.T) { assert.True(t, reader.queries[0].CurrentBoot) } +func TestJournalctlCatOutputPreservesRawMessageBytes(t *testing.T) { + message := "first\nsecond\t\x1b[31mred\x1b[0m\x00\xff\n" + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{Message: message}}} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"-k", "--output", "cat"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, []byte(message+"\n"), stdout.Bytes()) + require.Len(t, reader.queries, 1) +} + func TestJournalctlDiskUsageUsesStorageReadCapability(t *testing.T) { storage := &fakeJournalStorage{usage: builtins.JournalUsage{Bytes: 5 * 1024 * 1024, Files: 3}} reader := &fakeJournalReader{} @@ -601,6 +620,7 @@ func TestJournalctlHelpDoesNotRequireSystemdCapability(t *testing.T) { assert.Contains(t, stdout.String(), "--unit") assert.Contains(t, stdout.String(), "--rotate") assert.Contains(t, stdout.String(), "--vacuum-time") + assert.Contains(t, stdout.String(), "output format: escaped short or raw cat") assert.Contains(t, stdout.String(), "alone, remove archives older than AGE; with --vacuum-size, set the minimum deletion age") assert.Empty(t, stderr.String()) } From 060d5a590653249f599065c87da09ab94b5dc789 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 13:09:02 -0400 Subject: [PATCH 46/53] perf(systemd): avoid journal prefix allocation --- internal/systemd/journal_query_file.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/systemd/journal_query_file.go b/internal/systemd/journal_query_file.go index d42fca55..7d00b96d 100644 --- a/internal/systemd/journal_query_file.go +++ b/internal/systemd/journal_query_file.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "math" - "strings" "time" "github.com/DataDog/rshell/builtins" @@ -333,19 +332,19 @@ func (f *journalFileView) selectedDataField(data journalDataObject) (string, str return field, "", false, nil } - limit := separator + 1 + maxJournalFieldSize + valueOffset := separator + 1 + limit := valueOffset + maxJournalFieldSize payload, truncated, err := f.readDataPayload(data, limit) if err != nil { return "", "", false, err } - fieldPrefix := field + "=" - if !strings.HasPrefix(string(payload), fieldPrefix) { + if !bytes.HasPrefix(payload, prefix[:valueOffset]) { return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload changed while being read") } if !truncated && f.dataHash(payload) != data.hash { return "", "", false, journalCorrupt(f.name, data.payloadOffset, "DATA payload hash does not match its object") } - return field, string(payload[len(fieldPrefix):]), true, nil + return field, string(payload[valueOffset:]), true, nil } func validJournalFieldName(field []byte) bool { From e57213841510035abd7d54f41ec92eac18e01744 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 13:38:18 -0400 Subject: [PATCH 47/53] fix(systemd): pin journal control socket --- README.md | 2 +- SHELL_FEATURES.md | 2 +- docs/RULES.md | 9 +-- internal/systemd/journal_varlink.go | 23 +------- .../systemd/journal_varlink_dial_linux.go | 57 ++++++++++++++++++ .../journal_varlink_dial_linux_test.go | 59 +++++++++++++++++++ .../journal_varlink_dial_unsupported.go | 29 +++++++++ 7 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 internal/systemd/journal_varlink_dial_linux.go create mode 100644 internal/systemd/journal_varlink_dial_linux_test.go create mode 100644 internal/systemd/journal_varlink_dial_unsupported.go diff --git a/README.md b/README.md index 1af218dd..6876a9ef 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Selected journal fields are capped at 64 KiB. `short` escapes embedded newlines, Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. -The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux and the configured journald Varlink socket. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. Used alone, `--vacuum-time` removes every eligible archive at least that old. `--vacuum-size` requires `--vacuum-time`; when combined, the age becomes a minimum deletion age and cleanup stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. +The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux, the configured journald Varlink socket, and procfs descriptor links at `/proc/self/fd`. The backend pins the socket inode before connecting through its descriptor, so a concurrent path replacement cannot redirect the request; rotation fails closed when descriptor links are unavailable. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. Used alone, `--vacuum-time` removes every eligible archive at least that old. `--vacuum-size` requires `--vacuum-time`; when combined, the age becomes a minimum deletion age and cleanup stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 13a4c4f1..58a031f0 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries, 32 service scopes, and 64 KiB per selected field; `short` sanitizes controls, non-graphic Unicode, and invalid UTF-8, while `cat` matches host `journalctl` within the field bound by writing each `MESSAGE` value unchanged and appending one newline; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries, 32 service scopes, and 64 KiB per selected field; `short` sanitizes controls, non-graphic Unicode, and invalid UTF-8, while `cat` matches host `journalctl` within the field bound by writing each `MESSAGE` value unchanged and appending one newline; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux with procfs descriptor links at `/proc/self/fd`, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected diff --git a/docs/RULES.md b/docs/RULES.md index cebb282b..dcb36c56 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -104,10 +104,11 @@ partial-progress errors bound each cleanup invocation, in addition to the exact `JournalRotator.RotateJournal` is the only journal-daemon mutation exception. It may call only the fixed `io.systemd.Journal.Rotate` Varlink method through the configured journal control socket. The backend validates the target machine -ID before connecting, rejects symlink and non-socket endpoints, verifies socket -identity across the connection, and applies fixed response-size and execution -time bounds. A generic Varlink method or parameter interface must not be exposed -to builtins. +ID before connecting, pins the socket inode without following a final symlink, +and connects through the pinned descriptor so a concurrent path replacement +cannot redirect the request. It applies fixed response-size and execution time +bounds. A generic Varlink method or parameter interface must not be exposed to +builtins. --- diff --git a/internal/systemd/journal_varlink.go b/internal/systemd/journal_varlink.go index c622da4f..c14c1088 100644 --- a/internal/systemd/journal_varlink.go +++ b/internal/systemd/journal_varlink.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "net" - "os" "time" ) @@ -33,29 +32,11 @@ type varlinkReply struct { } func rotateJournalControl(ctx context.Context, path string) error { - before, err := os.Lstat(path) + conn, err := dialJournalControl(ctx, path) if err != nil { - return fmt.Errorf("inspect journal control socket: %w", err) - } - if before.Mode()&os.ModeSymlink != 0 || before.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("journal control endpoint is not a Unix socket") - } - - var dialer net.Dialer - conn, err := dialer.DialContext(ctx, "unix", path) - if err != nil { - return contextIOError(ctx, "connect to journal control socket", err) + return err } defer conn.Close() - - after, err := os.Lstat(path) - if err != nil { - return fmt.Errorf("reinspect journal control socket: %w", err) - } - if !os.SameFile(before, after) { - return fmt.Errorf("journal control socket changed while connecting") - } - return callJournalRotateVarlink(ctx, conn) } diff --git a/internal/systemd/journal_varlink_dial_linux.go b/internal/systemd/journal_varlink_dial_linux.go new file mode 100644 index 00000000..f780d212 --- /dev/null +++ b/internal/systemd/journal_varlink_dial_linux.go @@ -0,0 +1,57 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux + +package systemd + +import ( + "context" + "fmt" + "net" + "path/filepath" + "strconv" + + "golang.org/x/sys/unix" +) + +const journalControlFDDir = "/proc/self/fd" + +func dialJournalControl(ctx context.Context, path string) (net.Conn, error) { + fd, err := openJournalControlSocket(path) + if err != nil { + return nil, err + } + defer unix.Close(fd) + return dialPinnedJournalControl(ctx, fd) +} + +func openJournalControlSocket(path string) (int, error) { + fd, err := unix.Open(path, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return -1, fmt.Errorf("inspect journal control socket: %w", err) + } + + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + _ = unix.Close(fd) + return -1, fmt.Errorf("inspect journal control socket: %w", err) + } + if stat.Mode&unix.S_IFMT != unix.S_IFSOCK { + _ = unix.Close(fd) + return -1, fmt.Errorf("journal control endpoint is not a Unix socket") + } + return fd, nil +} + +func dialPinnedJournalControl(ctx context.Context, fd int) (net.Conn, error) { + endpoint := filepath.Join(journalControlFDDir, strconv.Itoa(fd)) + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "unix", endpoint) + if err != nil { + return nil, contextIOError(ctx, "connect to pinned journal control socket", err) + } + return conn, nil +} diff --git a/internal/systemd/journal_varlink_dial_linux_test.go b/internal/systemd/journal_varlink_dial_linux_test.go new file mode 100644 index 00000000..c4b7e11c --- /dev/null +++ b/internal/systemd/journal_varlink_dial_linux_test.go @@ -0,0 +1,59 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build linux + +package systemd + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestDialPinnedJournalControlIgnoresPathReplacement(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "journal.sock") + original := listenUnixSocket(t, path) + + fd, err := openJournalControlSocket(path) + require.NoError(t, err) + t.Cleanup(func() { _ = unix.Close(fd) }) + + require.NoError(t, os.Rename(path, filepath.Join(dir, "original.sock"))) + attacker := listenUnixSocket(t, path) + + conn, err := dialPinnedJournalControl(context.Background(), fd) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + require.NoError(t, original.SetDeadline(time.Now().Add(time.Second))) + accepted, err := original.AcceptUnix() + require.NoError(t, err) + require.NoError(t, accepted.Close()) + + require.NoError(t, attacker.SetDeadline(time.Now().Add(100*time.Millisecond))) + _, err = attacker.AcceptUnix() + require.Error(t, err) + var netErr net.Error + require.ErrorAs(t, err, &netErr) + assert.True(t, netErr.Timeout()) +} + +func listenUnixSocket(t *testing.T, path string) *net.UnixListener { + t.Helper() + listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) + require.NoError(t, err) + listener.SetUnlinkOnClose(false) + t.Cleanup(func() { _ = listener.Close() }) + return listener +} diff --git a/internal/systemd/journal_varlink_dial_unsupported.go b/internal/systemd/journal_varlink_dial_unsupported.go new file mode 100644 index 00000000..45fb2a1e --- /dev/null +++ b/internal/systemd/journal_varlink_dial_unsupported.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +//go:build !linux + +package systemd + +import ( + "context" + "fmt" + "net" + "os" +) + +func dialJournalControl(ctx context.Context, path string) (net.Conn, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect journal control socket: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || info.Mode()&os.ModeSocket == 0 { + return nil, fmt.Errorf("journal control endpoint is not a Unix socket") + } + return nil, fmt.Errorf("journal control connections require Linux") +} From d3bb3b91a548a2b2d6006a5595a37a6cc65b2fe1 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 13:49:37 -0400 Subject: [PATCH 48/53] fix(systemd): root mounted machine ID reads --- README.md | 2 +- SHELL_FEATURES.md | 2 +- docs/RULES.md | 4 + internal/systemd/journal_files.go | 96 +++++++++++++++++++++++- internal/systemd/journal_files_test.go | 44 +++++++++++ internal/systemd/journal_rotate_linux.go | 2 +- internal/systemd/journal_vacuum_unix.go | 2 +- internal/systemd/target.go | 9 ++- internal/systemd/target_test.go | 3 + 9 files changed, 154 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6876a9ef..557ad92c 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ interp.AllowedSystemServices([]interp.SystemdControlGrant{ The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,systemd-journald.service:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. -**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. +**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`; host-absolute machine-ID symlinks such as `/run/machine-id` are resolved inside that root. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Every selected journal file header is checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 58a031f0..869563c8 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -123,7 +123,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ AllowedCommands — restricts which commands (builtins or external) may be executed; commands require the `rshell:` namespace prefix (e.g. `rshell:cat`); if not set, no commands are allowed - ✅ AllowedSystemServices policy — one shared default-deny capability map for exact services with generic `read`, `clean`, `reload`, and `restart` actions; actionless grants are ignored, invalid services/actions are skipped, service names are case-sensitive, are not normalized, and cannot contain `:`, mutating actions require remediation mode, and allowing all commands does not bypass this policy; configure services through `interp.AllowedSystemServices` or CLI `--allowed-services SERVICE:ACTION[+ACTION...]` -- ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host +- ✅ SystemdTargetConfig — systemd-aware builtins use standard local paths by default; `Root` derives journal directories, machine ID, journald Varlink socket, and system D-Bus socket beneath one mounted-host root and resolves host-absolute machine-ID symlinks within that root, while explicit absolute paths support split mount layouts without falling back to local endpoints; target paths are trusted configuration and bypass `AllowedPaths`; explicit mounts must all refer to the same host - ✅ AllowedPaths filesystem sandboxing — restricts all file access (read and write) to specified directories. Entries may end with `:ro` or `:rw` to indicate read-only and read-write permissions, respectively; entries without a suffix default to read-only. In remediation mode, write operations are accepted only inside the most-specific matching `:rw` root. Cross-root symlink fallback is read-only to avoid TOCTOU on writes; on Unix, symlink components in write targets are rejected with `symlinks are not supported as write targets` via a no-follow `openat` walk - ✅ Whole-run execution timeout — callers can bound a `Run()` call via `context.Context`, `interp.MaxExecutionTime`, or the CLI `--timeout` flag; the deadline applies to the entire script, not each individual command - ✅ ProcPath — overrides the proc filesystem path used by `ps` (default `/proc`; Linux-only; useful for testing/container environments); `ps` does not read `/proc//cmdline` diff --git a/docs/RULES.md b/docs/RULES.md index dcb36c56..5a941eea 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -79,6 +79,10 @@ read paths selected by `interp.WithSystemdTarget`; those paths intentionally bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding application and cannot be supplied by shell scripts. +Root-derived targets MUST resolve host-absolute machine-ID symlinks within the +configured root rather than in the rshell process namespace. Symlink traversal +must be bounded and must reject relative paths that escape the target root. + Journal reads and storage metadata are limited to regular, non-symlink `.journal` files directly under the configured machine-ID directories. The pure-Go reader verifies each file header's machine ID, snapshots the file set and identities, diff --git a/internal/systemd/journal_files.go b/internal/systemd/journal_files.go index 60c6d2af..41b77b19 100644 --- a/internal/systemd/journal_files.go +++ b/internal/systemd/journal_files.go @@ -16,8 +16,9 @@ import ( ) const ( - maxJournalFiles = 4096 - maxMachineIDFileSize = 64 + maxJournalFiles = 4096 + maxMachineIDFileSize = 64 + maxMachineIDSymlinkHops = 40 ) func (c *Client) journalFiles() (string, []string, error) { @@ -28,7 +29,7 @@ func (c *Client) journalFiles() (string, []string, error) { return "", nil, fmt.Errorf("systemd target journal directories are not configured") } - machineID, err := readMachineID(c.target.MachineIDPath) + machineID, err := c.readMachineID() if err != nil { return "", nil, err } @@ -113,6 +114,95 @@ func readMachineID(path string) (string, error) { if err != nil { return "", fmt.Errorf("open systemd machine ID %q: %w", path, err) } + return readMachineIDFile(path, file) +} + +func (c *Client) readMachineID() (string, error) { + if c.target.root == "" { + return readMachineID(c.target.MachineIDPath) + } + relPath, err := filepath.Rel(c.target.root, c.target.MachineIDPath) + if err != nil || !validRootRelativePath(relPath) { + return "", fmt.Errorf("systemd machine ID path %q is outside target root %q", c.target.MachineIDPath, c.target.root) + } + + root, err := os.OpenRoot(c.target.root) + if err != nil { + return "", fmt.Errorf("open systemd target root %q: %w", c.target.root, err) + } + file, openErr := openRootedMachineID(root, relPath) + closeErr := root.Close() + if openErr != nil { + return "", fmt.Errorf("open systemd machine ID %q: %w", c.target.MachineIDPath, openErr) + } + if closeErr != nil { + file.Close() + return "", fmt.Errorf("close systemd target root %q: %w", c.target.root, closeErr) + } + return readMachineIDFile(c.target.MachineIDPath, file) +} + +func openRootedMachineID(root *os.Root, path string) (*os.File, error) { + path = filepath.Clean(path) + // Resolve one link per pass so host-absolute targets can be rebased under root. + for range maxMachineIDSymlinkHops + 1 { + components := strings.Split(path, string(filepath.Separator)) + partial := "." + followed := false + for index, component := range components { + if component == "" || component == "." { + continue + } + partial = filepath.Join(partial, component) + info, err := root.Lstat(partial) + if err != nil { + return nil, err + } + if info.Mode()&fs.ModeSymlink == 0 { + continue + } + + target, err := root.Readlink(partial) + if err != nil { + return nil, err + } + if filepath.IsAbs(target) { + target = rootRelativeAbsolutePath(target) + } else { + target = filepath.Join(filepath.Dir(partial), target) + } + if index+1 < len(components) { + target = filepath.Join(target, filepath.Join(components[index+1:]...)) + } + path = filepath.Clean(target) + if !validRootRelativePath(path) { + return nil, fmt.Errorf("machine ID symbolic link escapes the systemd target root") + } + followed = true + break + } + if !followed { + return root.Open(path) + } + } + return nil, fmt.Errorf("machine ID path has too many symbolic links") +} + +func rootRelativeAbsolutePath(path string) string { + path = strings.TrimPrefix(path, filepath.VolumeName(path)) + path = strings.TrimLeft(path, `/\`) + if path == "" { + return "." + } + return path +} + +func validRootRelativePath(path string) bool { + path = filepath.Clean(path) + return !filepath.IsAbs(path) && path != ".." && !strings.HasPrefix(path, ".."+string(filepath.Separator)) +} + +func readMachineIDFile(path string, file *os.File) (string, error) { data, readErr := io.ReadAll(io.LimitReader(file, maxMachineIDFileSize+1)) closeErr := file.Close() if readErr != nil { diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go index 53584231..14c04377 100644 --- a/internal/systemd/journal_files_test.go +++ b/internal/systemd/journal_files_test.go @@ -69,6 +69,50 @@ func TestJournalFilesSelectsRegularFilesForTargetMachine(t *testing.T) { assert.Equal(t, []string{firstJournal, secondJournal}, files) } +func TestJournalFilesResolveAbsoluteMachineIDSymlinkWithinTargetRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires elevated privileges on Windows") + } + + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + relMachineID := filepath.Join("run", "rshell-machine-id") + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "run"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, relMachineID), []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.Symlink(filepath.FromSlash("/run/rshell-machine-id"), filepath.Join(root, "etc", "machine-id"))) + + machineDir := filepath.Join(root, "var", "log", "journal", machineID) + require.NoError(t, os.MkdirAll(machineDir, 0o700)) + journalPath := filepath.Join(machineDir, "system.journal") + require.NoError(t, os.WriteFile(journalPath, nil, 0o600)) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + gotMachineID, files, err := NewClient(target).journalFiles() + require.NoError(t, err) + assert.Equal(t, machineID, gotMachineID) + assert.Equal(t, []string{journalPath}, files) +} + +func TestJournalFilesRejectMachineIDSymlinkEscapeFromTargetRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires elevated privileges on Windows") + } + + parent := t.TempDir() + root := filepath.Join(parent, "host") + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(parent, "outside-machine-id"), []byte("0123456789abcdef0123456789abcdef\n"), 0o600)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "outside-machine-id"), filepath.Join(root, "etc", "machine-id"))) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + _, _, err = NewClient(target).journalFiles() + require.Error(t, err) + assert.Contains(t, err.Error(), "escapes the systemd target root") +} + func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}) _, _, err := client.journalFiles() diff --git a/internal/systemd/journal_rotate_linux.go b/internal/systemd/journal_rotate_linux.go index 30660f4c..d67d7367 100644 --- a/internal/systemd/journal_rotate_linux.go +++ b/internal/systemd/journal_rotate_linux.go @@ -22,7 +22,7 @@ func (c *Client) RotateJournal(ctx context.Context) error { if c.target.MachineIDPath == "" { return fmt.Errorf("systemd target machine ID path is unavailable") } - if _, err := readMachineID(c.target.MachineIDPath); err != nil { + if _, err := c.readMachineID(); err != nil { return fmt.Errorf("validate systemd target machine ID: %w", err) } if c.target.JournalControlSocket == "" { diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go index 88513822..ab8dfb1a 100644 --- a/internal/systemd/journal_vacuum_unix.go +++ b/internal/systemd/journal_vacuum_unix.go @@ -94,7 +94,7 @@ func (c *Client) openVacuumDirectories() ([]*vacuumDirectory, error) { if len(c.target.JournalDirs) == 0 { return nil, fmt.Errorf("systemd target journal directories are not configured") } - machineID, err := readMachineID(c.target.MachineIDPath) + machineID, err := c.readMachineID() if err != nil { return nil, err } diff --git a/internal/systemd/target.go b/internal/systemd/target.go index 2d1b3355..7f85710b 100644 --- a/internal/systemd/target.go +++ b/internal/systemd/target.go @@ -16,14 +16,15 @@ import ( const MaxJournalDirs = 8 // Target identifies one systemd host. Root is accepted only as input to -// ResolveTarget; resolved targets contain explicit paths in the remaining -// fields. +// ResolveTarget; resolved targets retain it internally for rooted symlink +// resolution and expose explicit paths in the remaining fields. type Target struct { Root string JournalDirs []string MachineIDPath string JournalControlSocket string SystemBusSocket string + root string } // LocalTarget returns the standard paths for the local systemd host. @@ -44,7 +45,9 @@ func ResolveTarget(target Target) (Target, error) { if err != nil { return Target{}, err } - return targetFromRoot(root), nil + resolved := targetFromRoot(root) + resolved.root = root + return resolved, nil } if len(target.JournalDirs) == 0 && target.MachineIDPath == "" && target.JournalControlSocket == "" && target.SystemBusSocket == "" { diff --git a/internal/systemd/target_test.go b/internal/systemd/target_test.go index c3a37782..3f95af52 100644 --- a/internal/systemd/target_test.go +++ b/internal/systemd/target_test.go @@ -19,6 +19,7 @@ func TestResolveTargetDefaultsToLocalPaths(t *testing.T) { target, err := ResolveTarget(Target{}) require.NoError(t, err) + assert.Empty(t, target.Root) assert.Equal(t, []string{"/var/log/journal", "/run/log/journal"}, target.JournalDirs) assert.Equal(t, "/etc/machine-id", target.MachineIDPath) assert.Equal(t, "/run/systemd/journal/io.systemd.journal", target.JournalControlSocket) @@ -29,6 +30,8 @@ func TestResolveTargetDerivesMountedRootPaths(t *testing.T) { target, err := ResolveTarget(Target{Root: "/host"}) require.NoError(t, err) + assert.Empty(t, target.Root) + assert.Equal(t, filepath.FromSlash("/host"), target.root) assert.Equal(t, []string{filepath.FromSlash("/host/var/log/journal"), filepath.FromSlash("/host/run/log/journal")}, target.JournalDirs) assert.Equal(t, filepath.FromSlash("/host/etc/machine-id"), target.MachineIDPath) assert.Equal(t, filepath.FromSlash("/host/run/systemd/journal/io.systemd.journal"), target.JournalControlSocket) From 40c5a1e86576d864c684e8c18ecfa3d82be7b39c Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 14:06:20 -0400 Subject: [PATCH 49/53] fix(journalctl): label kernel output --- README.md | 2 +- SHELL_FEATURES.md | 2 +- builtins/journalctl/journalctl.go | 3 +++ builtins/journalctl/journalctl_test.go | 22 ++++++++++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 557ad92c..dbaeb9f5 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Root targets derive those paths below `Root`; explicit targets may map them to a Log queries default to 100 entries and are capped at 1,000 entries and 32 exact unit scopes per invocation. `--since` accepts RFC 3339, local `YYYY-MM-DD HH:MM:SS`, or a non-negative Go lookback duration such as `15m`. `-b` means the newest boot present in the selected target journal, so it remains correct for a mounted host. Bare journal reads, globbed units, arbitrary field matches, follow mode, raw structured output, alternate sources, and arbitrary boot selection are unavailable. -Selected journal fields are capped at 64 KiB. `short` escapes embedded newlines, carriage returns, tabs, non-graphic Unicode code points, and invalid UTF-8 bytes before writing output. Within that bound, `cat` matches host `journalctl -o cat` by writing each `MESSAGE` value unchanged and appending one newline; callers must treat that raw output as untrusted log content. +Selected journal fields are capped at 64 KiB. `short` escapes embedded newlines, carriage returns, tabs, non-graphic Unicode code points, and invalid UTF-8 bytes before writing output; kernel entries without an identifier are labeled `kernel`, matching host `journalctl -k`. Within that bound, `cat` matches host `journalctl -o cat` by writing each `MESSAGE` value unchanged and appending one newline; callers must treat that raw output as untrusted log content. Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index 869563c8..bc02c7d0 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -24,7 +24,7 @@ The in-shell `help` command mirrors these feature categories: run `help` for a c - ✅ `ip [-o|-4|-6|--brief] addr|link [show] [dev IFNAME]` — show network interface addresses and link-layer info (read-only); write ops (`add`, `del`, `flush`, `set`), namespace ops (`netns`, `-n`), and batch mode (`-b`/`-B`/`--force`) are blocked - ✅ `ip route [show|list]` — show IPv4 routing table (Linux only; reads `/proc/net/route` directly via `os.Open`, bypassing `AllowedPaths`); at most 10 000 entries loaded; lines longer than 1 MiB abort parsing with an error (exit 1) - ✅ `ip route get ADDRESS` — show the route selected by longest-prefix-match for ADDRESS (Linux only); write ops (`add`, `del`, `flush`, `replace`, `change`, `save`, `restore`) are blocked; `-6` (IPv6 routing) is not supported -- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries, 32 service scopes, and 64 KiB per selected field; `short` sanitizes controls, non-graphic Unicode, and invalid UTF-8, while `cat` matches host `journalctl` within the field bound by writing each `MESSAGE` value unchanged and appending one newline; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux with procfs descriptor links at `/proc/self/fd`, and mounted-file usage/vacuum operations support Linux/macOS +- ✅ `journalctl (-u SERVICE...|-k) [-b] [-n COUNT] [--since TIME] [-o short|cat]` — bounded journal investigation for exact system services or current-boot kernel messages; defaults to 100 and caps at 1,000 entries, 32 service scopes, and 64 KiB per selected field; `short` sanitizes controls, non-graphic Unicode, and invalid UTF-8 and labels unidentified kernel entries `kernel`, while `cat` matches host `journalctl` within the field bound by writing each `MESSAGE` value unchanged and appending one newline; service reads require exact `SERVICE:read` grants and kernel reads require `systemd-journald.service:read`; also supports `--disk-usage` with `systemd-journald.service:read`, plus remediation-only `--rotate`, `--vacuum-size SIZE`, `--vacuum-time AGE`, and `--dry-run` with `systemd-journald.service:clean`; time-only vacuum removes every eligible archive at least `AGE` old, while size vacuum requires `AGE` as a minimum deletion age and never deletes newer archives merely to reach its target; rotation may precede vacuuming, while `--dry-run` cannot rotate; bare/unrestricted reads, globbed services, arbitrary matches, follow mode, raw structured output, alternate sources, cursors, machines, namespaces, and arbitrary boot selection are unavailable; log reads use a bounded pure-Go parser for regular/compact and XZ/LZ4/Zstandard journal files with no host `journalctl`, cgo, or libsystemd dependency; rotation requires Linux with procfs descriptor links at `/proc/self/fd`, and mounted-file usage/vacuum operations support Linux/macOS - ✅ `logrotate (-s SIZE|-f) [-n] [-v] FILE...` — remediation-mode helper that truncates log files to zero bytes through `AllowedPaths`; `--size SIZE` truncates only files at least SIZE bytes, `--force` explicitly truncates without a threshold, `--dry-run` reports without modifying files, and `--verbose` reports per-file actions. Symlinked write targets are rejected with `symlinks are not supported as write targets`; pass the real log path instead. This is not a full `logrotate(8)` replacement: no config parsing, rename-based rotation, retained copies, compression, state files, or rotate scripts. - ✅ `sort [-rnhubfds] [-k KEYDEF] [-t SEP] [-c|-C] [FILE]...` — sort lines of text files; `-h`/`--human-numeric-sort` orders by SI suffix (none < K/k < M < G < T < P < E < Z < Y < R < Q) then by numeric value (single-letter suffixes only — `Ki`, `Mi`, etc. are not recognised); `-o`, `--compress-program`, and `-T` are rejected (filesystem write / exec) - ✅ `ss [-tuaxlans4689Hoehs] [OPTION]...` — display network socket statistics; reads kernel socket state directly via `os.Open` (bypassing `AllowedPaths`) from: Linux: `/proc/net/`; macOS: sysctl; Windows: iphlpapi.dll; `-F`/`--filter` (GTFOBins file-read), `-p`/`--processes` (PID disclosure), `-K`/`--kill`, `-E`/`--events`, and `-N`/`--net` are rejected diff --git a/builtins/journalctl/journalctl.go b/builtins/journalctl/journalctl.go index c21be6b7..a83372e7 100644 --- a/builtins/journalctl/journalctl.go +++ b/builtins/journalctl/journalctl.go @@ -178,6 +178,9 @@ func (options flags) run(fs *builtins.FlagSet) builtins.HandlerFunc { MaxEntries: lines, } err := callCtx.Systemd.Journal.ReadJournal(ctx, query, func(entry builtins.JournalEntry) error { + if query.Kernel && entry.Identifier == "" { + entry.Identifier = "kernel" + } return writeEntry(callCtx.Stdout, entry, *options.output) }) if err == nil || builtins.IsBrokenPipe(err) { diff --git a/builtins/journalctl/journalctl_test.go b/builtins/journalctl/journalctl_test.go index cb08efcc..9a5a902b 100644 --- a/builtins/journalctl/journalctl_test.go +++ b/builtins/journalctl/journalctl_test.go @@ -161,6 +161,28 @@ func TestJournalctlShortOutputEscapesTerminalControls(t *testing.T) { assert.True(t, reader.queries[0].CurrentBoot) } +func TestJournalctlShortOutputLabelsUnidentifiedKernelEntries(t *testing.T) { + reader := &fakeJournalReader{entries: []builtins.JournalEntry{{ + Timestamp: time.Date(2026, time.July, 14, 12, 34, 56, 0, time.UTC), + Message: "booting", + }}} + var stdout, stderr bytes.Buffer + result := runJournalctl(t, []string{"-k"}, &builtins.CallContext{ + Stdout: &stdout, + Stderr: &stderr, + AuthorizeSystemd: func(...builtins.SystemdOperation) error { + return nil + }, + Systemd: &builtins.SystemdServices{Journal: reader}, + }) + + assert.Equal(t, uint8(0), result.Code) + assert.Empty(t, stderr.String()) + assert.Equal(t, "Jul 14 12:34:56 - kernel: booting\n", stdout.String()) + require.Len(t, reader.queries, 1) + assert.True(t, reader.queries[0].Kernel) +} + func TestJournalctlCatOutputPreservesRawMessageBytes(t *testing.T) { message := "first\nsecond\t\x1b[31mred\x1b[0m\x00\xff\n" reader := &fakeJournalReader{entries: []builtins.JournalEntry{{Message: message}}} From ddd17834115b3cc018f765821528a0ba4cfba683 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 14:25:39 -0400 Subject: [PATCH 50/53] fix(systemd): contain rooted journal paths --- README.md | 2 +- docs/RULES.md | 8 +- internal/systemd/journal_files.go | 96 +---------- internal/systemd/journal_files_test.go | 25 +++ internal/systemd/journal_fixture_test.go | 36 ++++ internal/systemd/journal_snapshot.go | 12 +- internal/systemd/journal_usage_unix.go | 2 +- internal/systemd/journal_usage_unix_test.go | 20 +++ internal/systemd/journal_vacuum_unix.go | 2 +- internal/systemd/journal_vacuum_unix_test.go | 24 +++ internal/systemd/target_path.go | 165 +++++++++++++++++++ 11 files changed, 293 insertions(+), 99 deletions(-) create mode 100644 internal/systemd/target_path.go diff --git a/README.md b/README.md index dbaeb9f5..6091e654 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ interp.AllowedSystemServices([]interp.SystemdControlGrant{ The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,systemd-journald.service:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. -**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`; host-absolute machine-ID symlinks such as `/run/machine-id` are resolved inside that root. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. +**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`; host-absolute machine-ID and journal-directory symlinks are resolved inside that root, and journal reads, usage scans, and vacuuming remain rooted throughout file access. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Every selected journal file header is checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. diff --git a/docs/RULES.md b/docs/RULES.md index 5a941eea..efb9d98d 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -79,9 +79,11 @@ read paths selected by `interp.WithSystemdTarget`; those paths intentionally bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding application and cannot be supplied by shell scripts. -Root-derived targets MUST resolve host-absolute machine-ID symlinks within the -configured root rather than in the rshell process namespace. Symlink traversal -must be bounded and must reject relative paths that escape the target root. +Root-derived targets MUST resolve host-absolute machine-ID and journal-directory +symlinks within the configured root rather than in the rshell process namespace. +Journal discovery, reads, metadata scans, and deletion MUST remain rooted for the +entire operation. Symlink traversal must be bounded and must reject relative paths +that escape the target root. Journal reads and storage metadata are limited to regular, non-symlink `.journal` files directly under the configured machine-ID directories. The pure-Go reader diff --git a/internal/systemd/journal_files.go b/internal/systemd/journal_files.go index 41b77b19..ffcb80ba 100644 --- a/internal/systemd/journal_files.go +++ b/internal/systemd/journal_files.go @@ -16,9 +16,8 @@ import ( ) const ( - maxJournalFiles = 4096 - maxMachineIDFileSize = 64 - maxMachineIDSymlinkHops = 40 + maxJournalFiles = 4096 + maxMachineIDFileSize = 64 ) func (c *Client) journalFiles() (string, []string, error) { @@ -37,7 +36,7 @@ func (c *Client) journalFiles() (string, []string, error) { files := make([]string, 0) for _, baseDir := range c.target.JournalDirs { dirPath := filepath.Join(baseDir, machineID) - dir, err := openJournalMachineDirectory(dirPath) + dir, err := c.openJournalMachineDirectory(dirPath) if err != nil { if os.IsNotExist(err) { continue @@ -79,8 +78,8 @@ func (c *Client) journalFiles() (string, []string, error) { return machineID, files, nil } -func openJournalMachineDirectory(path string) (*os.File, error) { - before, err := os.Lstat(path) +func (c *Client) openJournalMachineDirectory(path string) (*os.File, error) { + before, err := c.lstatTargetPath(path) if err != nil { return nil, err } @@ -88,7 +87,7 @@ func openJournalMachineDirectory(path string) (*os.File, error) { return nil, fmt.Errorf("journal machine directory %q is not a real directory", path) } - dir, err := os.Open(path) + dir, err := c.openTargetFile(path, false) if err != nil { return nil, err } @@ -97,7 +96,7 @@ func openJournalMachineDirectory(path string) (*os.File, error) { dir.Close() return nil, err } - after, err := os.Lstat(path) + after, err := c.lstatTargetPath(path) if err != nil { dir.Close() return nil, err @@ -118,90 +117,13 @@ func readMachineID(path string) (string, error) { } func (c *Client) readMachineID() (string, error) { - if c.target.root == "" { - return readMachineID(c.target.MachineIDPath) - } - relPath, err := filepath.Rel(c.target.root, c.target.MachineIDPath) - if err != nil || !validRootRelativePath(relPath) { - return "", fmt.Errorf("systemd machine ID path %q is outside target root %q", c.target.MachineIDPath, c.target.root) - } - - root, err := os.OpenRoot(c.target.root) + file, err := c.openTargetFile(c.target.MachineIDPath, true) if err != nil { - return "", fmt.Errorf("open systemd target root %q: %w", c.target.root, err) - } - file, openErr := openRootedMachineID(root, relPath) - closeErr := root.Close() - if openErr != nil { - return "", fmt.Errorf("open systemd machine ID %q: %w", c.target.MachineIDPath, openErr) - } - if closeErr != nil { - file.Close() - return "", fmt.Errorf("close systemd target root %q: %w", c.target.root, closeErr) + return "", fmt.Errorf("open systemd machine ID %q: %w", c.target.MachineIDPath, err) } return readMachineIDFile(c.target.MachineIDPath, file) } -func openRootedMachineID(root *os.Root, path string) (*os.File, error) { - path = filepath.Clean(path) - // Resolve one link per pass so host-absolute targets can be rebased under root. - for range maxMachineIDSymlinkHops + 1 { - components := strings.Split(path, string(filepath.Separator)) - partial := "." - followed := false - for index, component := range components { - if component == "" || component == "." { - continue - } - partial = filepath.Join(partial, component) - info, err := root.Lstat(partial) - if err != nil { - return nil, err - } - if info.Mode()&fs.ModeSymlink == 0 { - continue - } - - target, err := root.Readlink(partial) - if err != nil { - return nil, err - } - if filepath.IsAbs(target) { - target = rootRelativeAbsolutePath(target) - } else { - target = filepath.Join(filepath.Dir(partial), target) - } - if index+1 < len(components) { - target = filepath.Join(target, filepath.Join(components[index+1:]...)) - } - path = filepath.Clean(target) - if !validRootRelativePath(path) { - return nil, fmt.Errorf("machine ID symbolic link escapes the systemd target root") - } - followed = true - break - } - if !followed { - return root.Open(path) - } - } - return nil, fmt.Errorf("machine ID path has too many symbolic links") -} - -func rootRelativeAbsolutePath(path string) string { - path = strings.TrimPrefix(path, filepath.VolumeName(path)) - path = strings.TrimLeft(path, `/\`) - if path == "" { - return "." - } - return path -} - -func validRootRelativePath(path string) bool { - path = filepath.Clean(path) - return !filepath.IsAbs(path) && path != ".." && !strings.HasPrefix(path, ".."+string(filepath.Separator)) -} - func readMachineIDFile(path string, file *os.File) (string, error) { data, readErr := io.ReadAll(io.LimitReader(file, maxMachineIDFileSize+1)) closeErr := file.Close() diff --git a/internal/systemd/journal_files_test.go b/internal/systemd/journal_files_test.go index 14c04377..6f9fcabb 100644 --- a/internal/systemd/journal_files_test.go +++ b/internal/systemd/journal_files_test.go @@ -113,6 +113,31 @@ func TestJournalFilesRejectMachineIDSymlinkEscapeFromTargetRoot(t *testing.T) { assert.Contains(t, err.Error(), "escapes the systemd target root") } +func TestJournalFilesKeepAbsoluteJournalSymlinkWithinTargetRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires elevated privileges on Windows") + } + + parent := t.TempDir() + root := filepath.Join(parent, "host") + machineID := "0123456789abcdef0123456789abcdef" + realBase := filepath.Join(parent, "outside-journal") + realMachineDir := filepath.Join(realBase, machineID) + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "var", "log"), 0o700)) + require.NoError(t, os.MkdirAll(realMachineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(realMachineDir, "system.journal"), nil, 0o600)) + require.NoError(t, os.Symlink(realBase, filepath.Join(root, "var", "log", "journal"))) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + gotMachineID, files, err := NewClient(target).journalFiles() + require.NoError(t, err) + assert.Equal(t, machineID, gotMachineID) + assert.Empty(t, files) +} + func TestJournalFilesRejectsMissingJournalConfiguration(t *testing.T) { client := NewClient(Target{MachineIDPath: filepath.Join(t.TempDir(), "machine-id")}) _, _, err := client.journalFiles() diff --git a/internal/systemd/journal_fixture_test.go b/internal/systemd/journal_fixture_test.go index 8804360a..f162bcb0 100644 --- a/internal/systemd/journal_fixture_test.go +++ b/internal/systemd/journal_fixture_test.go @@ -144,6 +144,42 @@ func TestReadJournalUsesPureGoFixtureBackend(t *testing.T) { assert.Equal(t, []string{"manager noticed service", journalFixtureLongMessage()}, []string{entries[0].Message, entries[1].Message}) } +func TestReadJournalResolvesAbsoluteJournalSymlinkWithinTargetRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires elevated privileges on Windows") + } + + root := t.TempDir() + machineID := repeatedJournalID(0xaa) + realBase := filepath.Join(root, "custom", "journal") + realMachineDir := filepath.Join(realBase, machineID.String()) + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "var", "log"), 0o700)) + require.NoError(t, os.MkdirAll(realMachineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte(machineID.String()+"\n"), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(realMachineDir, "system.journal"), + readJournalFixture(t, "compact-keyed-zstd.journal.gz"), + 0o600, + )) + require.NoError(t, os.Symlink(filepath.FromSlash("/custom/journal"), filepath.Join(root, "var", "log", "journal"))) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + var entries []builtins.JournalEntry + err = NewClient(target).ReadJournal(context.Background(), builtins.JournalQuery{ + Units: []string{"rshell-fixture.service"}, + CurrentBoot: true, + MaxEntries: 1, + }, func(entry builtins.JournalEntry) error { + entries = append(entries, entry) + return nil + }) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, journalFixtureLongMessage(), entries[0].Message) +} + func TestReadJournalReturnsCanceledContextBeforeFilesystemAccess(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/internal/systemd/journal_snapshot.go b/internal/systemd/journal_snapshot.go index 13eedf55..d5196d8f 100644 --- a/internal/systemd/journal_snapshot.go +++ b/internal/systemd/journal_snapshot.go @@ -58,7 +58,7 @@ func (c *Client) openJournalSnapshot() (*journalSnapshot, error) { } fileIDs := make(map[journalID]string, len(paths)) for _, path := range paths { - opened, err := openJournalSnapshotFile(path) + opened, err := c.openJournalSnapshotFile(path) if err != nil { snapshot.close() return nil, err @@ -93,8 +93,8 @@ func (c *Client) openJournalSnapshot() (*journalSnapshot, error) { return snapshot, nil } -func openJournalSnapshotFile(path string) (*journalSnapshotFile, error) { - before, err := os.Lstat(path) +func (c *Client) openJournalSnapshotFile(path string) (*journalSnapshotFile, error) { + before, err := c.lstatTargetPath(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, journalChanged("journal file %q disappeared before open", path) @@ -105,7 +105,7 @@ func openJournalSnapshotFile(path string) (*journalSnapshotFile, error) { return nil, journalChanged("journal file %q is no longer a regular non-symlink file", path) } - file, err := os.Open(path) + file, err := c.openTargetFile(path, false) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, journalChanged("journal file %q disappeared during open", path) @@ -117,7 +117,7 @@ func openJournalSnapshotFile(path string) (*journalSnapshotFile, error) { file.Close() return nil, fmt.Errorf("inspect open journal file %q: %w", path, err) } - after, err := os.Lstat(path) + after, err := c.lstatTargetPath(path) if err != nil { file.Close() if errors.Is(err, fs.ErrNotExist) { @@ -153,7 +153,7 @@ func (s *journalSnapshot) stable(client *Client) error { return journalChanged("journal file set changed during snapshot") } for _, opened := range s.files { - current, err := os.Lstat(opened.path) + current, err := client.lstatTargetPath(opened.path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return journalChanged("journal file %q disappeared during snapshot", opened.path) diff --git a/internal/systemd/journal_usage_unix.go b/internal/systemd/journal_usage_unix.go index ffbdad76..3c819de2 100644 --- a/internal/systemd/journal_usage_unix.go +++ b/internal/systemd/journal_usage_unix.go @@ -28,7 +28,7 @@ func (c *Client) JournalDiskUsage(ctx context.Context) (builtins.JournalUsage, e if err := ctx.Err(); err != nil { return builtins.JournalUsage{}, err } - info, err := os.Lstat(path) + info, err := c.lstatTargetPath(path) if err != nil { return builtins.JournalUsage{}, fmt.Errorf("inspect journal file %q: %w", path, err) } diff --git a/internal/systemd/journal_usage_unix_test.go b/internal/systemd/journal_usage_unix_test.go index ce71fc46..ab20e2be 100644 --- a/internal/systemd/journal_usage_unix_test.go +++ b/internal/systemd/journal_usage_unix_test.go @@ -48,6 +48,26 @@ func TestJournalDiskUsageReturnsZeroForEmptyTarget(t *testing.T) { assert.Zero(t, usage.Bytes) } +func TestJournalDiskUsageResolvesAbsoluteJournalSymlinkWithinTargetRoot(t *testing.T) { + root := t.TempDir() + machineID := "0123456789abcdef0123456789abcdef" + realBase := filepath.Join(root, "custom", "journal") + realMachineDir := filepath.Join(realBase, machineID) + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "var", "log"), 0o700)) + require.NoError(t, os.MkdirAll(realMachineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte(machineID+"\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(realMachineDir, "system.journal"), make([]byte, 8192), 0o600)) + require.NoError(t, os.Symlink(filepath.FromSlash("/custom/journal"), filepath.Join(root, "var", "log", "journal"))) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + usage, err := NewClient(target).JournalDiskUsage(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, usage.Files) + assert.Greater(t, usage.Bytes, uint64(0)) +} + func TestJournalDiskUsageHonorsCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/internal/systemd/journal_vacuum_unix.go b/internal/systemd/journal_vacuum_unix.go index ab8dfb1a..1a602882 100644 --- a/internal/systemd/journal_vacuum_unix.go +++ b/internal/systemd/journal_vacuum_unix.go @@ -101,7 +101,7 @@ func (c *Client) openVacuumDirectories() ([]*vacuumDirectory, error) { directories := make([]*vacuumDirectory, 0, len(c.target.JournalDirs)) for _, basePath := range c.target.JournalDirs { - baseRoot, err := os.OpenRoot(basePath) + baseRoot, err := c.openTargetDirectory(basePath) if err != nil { if os.IsNotExist(err) { continue diff --git a/internal/systemd/journal_vacuum_unix_test.go b/internal/systemd/journal_vacuum_unix_test.go index 02cfecf7..0525f8e9 100644 --- a/internal/systemd/journal_vacuum_unix_test.go +++ b/internal/systemd/journal_vacuum_unix_test.go @@ -174,6 +174,30 @@ func TestVacuumJournalRejectsSymlinkedMachineDirectory(t *testing.T) { assert.FileExists(t, archive) } +func TestVacuumJournalKeepsAbsoluteJournalSymlinkWithinTargetRoot(t *testing.T) { + now := time.Now() + parent := t.TempDir() + root := filepath.Join(parent, "host") + outsideBase := filepath.Join(parent, "outside-journal") + outsideMachineDir := filepath.Join(outsideBase, testMachineID) + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "var", "log"), 0o700)) + require.NoError(t, os.MkdirAll(outsideMachineDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte(testMachineID+"\n"), 0o600)) + archive := writeVacuumFile(t, outsideMachineDir, archivedJournalName(1), now.Add(-10*24*time.Hour)) + require.NoError(t, os.Symlink(outsideBase, filepath.Join(root, "var", "log", "journal"))) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + result, err := NewClient(target).VacuumJournal(context.Background(), builtins.JournalVacuumRequest{ + Now: now, + Before: now.Add(-48 * time.Hour), + }) + require.NoError(t, err) + assert.Zero(t, result.Files) + assert.FileExists(t, archive) +} + func TestVacuumJournalRequiresRequestBounds(t *testing.T) { now := time.Now() client, _ := newVacuumTestClient(t) diff --git a/internal/systemd/target_path.go b/internal/systemd/target_path.go new file mode 100644 index 00000000..01d3bbe4 --- /dev/null +++ b/internal/systemd/target_path.go @@ -0,0 +1,165 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package systemd + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const maxTargetSymlinkHops = 40 + +func (c *Client) lstatTargetPath(path string) (fs.FileInfo, error) { + if c.target.root == "" { + return os.Lstat(path) + } + root, relative, err := c.openRootedTargetPath(path, false) + if err != nil { + return nil, err + } + info, operationErr := root.Lstat(relative) + return info, combineTargetPathErrors(operationErr, c.closeTargetRoot(root)) +} + +func (c *Client) openTargetFile(path string, followFinalSymlink bool) (*os.File, error) { + if c.target.root == "" { + return os.Open(path) + } + root, relative, err := c.openRootedTargetPath(path, followFinalSymlink) + if err != nil { + return nil, err + } + file, operationErr := root.Open(relative) + closeErr := c.closeTargetRoot(root) + if operationErr != nil { + return nil, combineTargetPathErrors(operationErr, closeErr) + } + if closeErr != nil { + return nil, errors.Join(closeErr, file.Close()) + } + return file, nil +} + +func (c *Client) openTargetDirectory(path string) (*os.Root, error) { + if c.target.root == "" { + return os.OpenRoot(path) + } + root, relative, err := c.openRootedTargetPath(path, true) + if err != nil { + return nil, err + } + directory, operationErr := root.OpenRoot(relative) + closeErr := c.closeTargetRoot(root) + if operationErr != nil { + return nil, combineTargetPathErrors(operationErr, closeErr) + } + if closeErr != nil { + return nil, errors.Join(closeErr, directory.Close()) + } + return directory, nil +} + +func (c *Client) openRootedTargetPath(path string, followFinalSymlink bool) (*os.Root, string, error) { + relative, err := filepath.Rel(c.target.root, path) + if err != nil || !validRootRelativePath(relative) { + return nil, "", fmt.Errorf("systemd target path %q is outside target root %q", path, c.target.root) + } + + root, err := os.OpenRoot(c.target.root) + if err != nil { + return nil, "", fmt.Errorf("open systemd target root %q: %w", c.target.root, err) + } + resolved, resolveErr := resolveRootedPath(root, relative, followFinalSymlink) + if resolveErr != nil { + return nil, "", combineTargetPathErrors(resolveErr, c.closeTargetRoot(root)) + } + return root, resolved, nil +} + +func (c *Client) closeTargetRoot(root *os.Root) error { + if err := root.Close(); err != nil { + return fmt.Errorf("close systemd target root %q: %w", c.target.root, err) + } + return nil +} + +func combineTargetPathErrors(operationErr, closeErr error) error { + if operationErr == nil { + return closeErr + } + if closeErr == nil { + return operationErr + } + return errors.Join(operationErr, closeErr) +} + +func resolveRootedPath(root *os.Root, path string, followFinalSymlink bool) (string, error) { + path = filepath.Clean(path) + if !validRootRelativePath(path) { + return "", fmt.Errorf("systemd target path escapes the configured root") + } + + // Resolve one link per pass so host-absolute targets can be rebased under root. + for range maxTargetSymlinkHops + 1 { + components := strings.Split(path, string(filepath.Separator)) + partial := "." + followed := false + for index, component := range components { + if component == "" || component == "." { + continue + } + partial = filepath.Join(partial, component) + info, err := root.Lstat(partial) + if err != nil { + return "", err + } + if info.Mode()&fs.ModeSymlink == 0 || (!followFinalSymlink && index == len(components)-1) { + continue + } + + target, err := root.Readlink(partial) + if err != nil { + return "", err + } + if filepath.IsAbs(target) { + target = rootRelativeAbsolutePath(target) + } else { + target = filepath.Join(filepath.Dir(partial), target) + } + if index+1 < len(components) { + target = filepath.Join(target, filepath.Join(components[index+1:]...)) + } + path = filepath.Clean(target) + if !validRootRelativePath(path) { + return "", fmt.Errorf("symbolic link escapes the systemd target root") + } + followed = true + break + } + if !followed { + return path, nil + } + } + return "", fmt.Errorf("systemd target path has too many symbolic links") +} + +func rootRelativeAbsolutePath(path string) string { + path = strings.TrimPrefix(path, filepath.VolumeName(path)) + path = strings.TrimLeft(path, `/\`) + if path == "" { + return "." + } + return path +} + +func validRootRelativePath(path string) bool { + path = filepath.Clean(path) + return filepath.VolumeName(path) == "" && !filepath.IsAbs(path) && path != ".." && !strings.HasPrefix(path, ".."+string(filepath.Separator)) +} From da04b2926b25b3dcd642b700d97d008401d11cbf Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 14:43:05 -0400 Subject: [PATCH 51/53] fix(systemd): root journal rotation socket --- README.md | 2 +- docs/RULES.md | 11 ++-- internal/systemd/journal_rotate_linux.go | 2 +- internal/systemd/journal_varlink.go | 4 +- .../systemd/journal_varlink_dial_linux.go | 31 +++++----- .../journal_varlink_dial_linux_test.go | 57 +++++++++++++++++-- .../journal_varlink_dial_unsupported.go | 4 +- internal/systemd/journal_varlink_test.go | 5 +- internal/systemd/target_path.go | 8 ++- 9 files changed, 90 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 6091e654..5b6774e7 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ interp.AllowedSystemServices([]interp.SystemdControlGrant{ The development CLI accepts equivalent grants through `--allowed-services mysql.service:restart+reload+read,systemd-journald.service:read+clean`. Service selectors are always exact service names; `:` separates a selector from its actions and is not allowed inside service names. The restricted `journalctl` builtin is available as described below; `systemctl` is not yet implemented. -**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`; host-absolute machine-ID and journal-directory symlinks are resolved inside that root, and journal reads, usage scans, and vacuuming remain rooted throughout file access. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. +**SystemdTargetConfig** selects which Linux host systemd-aware builtins address. With no option, standard local paths are used. `Root` derives all paths beneath a mounted host root such as `/host`; host-absolute symlinks in machine-ID, journal-directory, and journald control-socket paths are resolved inside that root. Journal reads, usage scans, vacuuming, and socket pinning remain rooted throughout access. For unusual container mount layouts, callers may instead provide explicit journal directories, machine-id path, journald Varlink socket, and system bus socket. `Root` and explicit fields cannot be mixed. Once any explicit field is supplied, omitted fields stay unavailable and never fall back to local endpoints. The machine-id path is mandatory for every explicit target. The development CLI exposes the equivalent `--systemd-root` and `--systemd-*` path flags. The target paths intentionally bypass `AllowedPaths`: they are trusted runner configuration and cannot be supplied by shell scripts. `Root` is the strongest way to keep files and endpoints on one host because every path is derived from the same mounted root. With explicit paths, the embedding application is responsible for mounting every supplied path from the same host. Every selected journal file header is checked against the configured machine ID, but journald's Rotate Varlink method does not return a machine ID that can independently attest its socket. diff --git a/docs/RULES.md b/docs/RULES.md index efb9d98d..2b821d9a 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -79,11 +79,12 @@ read paths selected by `interp.WithSystemdTarget`; those paths intentionally bypass `AllowedPaths`, like `ProcPath`, because they are fixed by the embedding application and cannot be supplied by shell scripts. -Root-derived targets MUST resolve host-absolute machine-ID and journal-directory -symlinks within the configured root rather than in the rshell process namespace. -Journal discovery, reads, metadata scans, and deletion MUST remain rooted for the -entire operation. Symlink traversal must be bounded and must reject relative paths -that escape the target root. +Root-derived targets MUST resolve host-absolute symlinks in machine-ID, +journal-directory, and journald control-socket paths within the configured root +rather than in the rshell process namespace. Journal discovery, reads, metadata +scans, deletion, and socket pinning MUST remain rooted for the entire operation. +Symlink traversal must be bounded and must reject relative paths that escape the +target root. Journal reads and storage metadata are limited to regular, non-symlink `.journal` files directly under the configured machine-ID directories. The pure-Go reader diff --git a/internal/systemd/journal_rotate_linux.go b/internal/systemd/journal_rotate_linux.go index d67d7367..f7e6ca6a 100644 --- a/internal/systemd/journal_rotate_linux.go +++ b/internal/systemd/journal_rotate_linux.go @@ -31,7 +31,7 @@ func (c *Client) RotateJournal(ctx context.Context) error { rotationCtx, cancel := context.WithTimeout(ctx, journalRotationTimeout) defer cancel() - if err := rotateJournalControl(rotationCtx, c.target.JournalControlSocket); err != nil { + if err := c.rotateJournalControl(rotationCtx, c.target.JournalControlSocket); err != nil { if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { return fmt.Errorf("journal rotation timed out after %s", journalRotationTimeout) } diff --git a/internal/systemd/journal_varlink.go b/internal/systemd/journal_varlink.go index c14c1088..0b066179 100644 --- a/internal/systemd/journal_varlink.go +++ b/internal/systemd/journal_varlink.go @@ -31,8 +31,8 @@ type varlinkReply struct { Continues bool `json:"continues"` } -func rotateJournalControl(ctx context.Context, path string) error { - conn, err := dialJournalControl(ctx, path) +func (c *Client) rotateJournalControl(ctx context.Context, path string) error { + conn, err := c.dialJournalControl(ctx, path) if err != nil { return err } diff --git a/internal/systemd/journal_varlink_dial_linux.go b/internal/systemd/journal_varlink_dial_linux.go index f780d212..65952ee4 100644 --- a/internal/systemd/journal_varlink_dial_linux.go +++ b/internal/systemd/journal_varlink_dial_linux.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "net" + "os" "path/filepath" "strconv" @@ -19,31 +20,31 @@ import ( const journalControlFDDir = "/proc/self/fd" -func dialJournalControl(ctx context.Context, path string) (net.Conn, error) { - fd, err := openJournalControlSocket(path) +func (c *Client) dialJournalControl(ctx context.Context, path string) (net.Conn, error) { + socket, err := c.openJournalControlSocket(path) if err != nil { return nil, err } - defer unix.Close(fd) - return dialPinnedJournalControl(ctx, fd) + defer socket.Close() + return dialPinnedJournalControl(ctx, int(socket.Fd())) } -func openJournalControlSocket(path string) (int, error) { - fd, err := unix.Open(path, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) +func (c *Client) openJournalControlSocket(path string) (*os.File, error) { + socket, err := c.openTargetFileFlags(path, false, unix.O_PATH|unix.O_NOFOLLOW) if err != nil { - return -1, fmt.Errorf("inspect journal control socket: %w", err) + return nil, fmt.Errorf("inspect journal control socket: %w", err) } - var stat unix.Stat_t - if err := unix.Fstat(fd, &stat); err != nil { - _ = unix.Close(fd) - return -1, fmt.Errorf("inspect journal control socket: %w", err) + info, err := socket.Stat() + if err != nil { + _ = socket.Close() + return nil, fmt.Errorf("inspect journal control socket: %w", err) } - if stat.Mode&unix.S_IFMT != unix.S_IFSOCK { - _ = unix.Close(fd) - return -1, fmt.Errorf("journal control endpoint is not a Unix socket") + if info.Mode()&os.ModeSocket == 0 { + _ = socket.Close() + return nil, fmt.Errorf("journal control endpoint is not a Unix socket") } - return fd, nil + return socket, nil } func dialPinnedJournalControl(ctx context.Context, fd int) (net.Conn, error) { diff --git a/internal/systemd/journal_varlink_dial_linux_test.go b/internal/systemd/journal_varlink_dial_linux_test.go index c4b7e11c..1d0c10f5 100644 --- a/internal/systemd/journal_varlink_dial_linux_test.go +++ b/internal/systemd/journal_varlink_dial_linux_test.go @@ -17,7 +17,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/sys/unix" ) func TestDialPinnedJournalControlIgnoresPathReplacement(t *testing.T) { @@ -25,14 +24,14 @@ func TestDialPinnedJournalControlIgnoresPathReplacement(t *testing.T) { path := filepath.Join(dir, "journal.sock") original := listenUnixSocket(t, path) - fd, err := openJournalControlSocket(path) + socket, err := (&Client{}).openJournalControlSocket(path) require.NoError(t, err) - t.Cleanup(func() { _ = unix.Close(fd) }) + t.Cleanup(func() { _ = socket.Close() }) require.NoError(t, os.Rename(path, filepath.Join(dir, "original.sock"))) attacker := listenUnixSocket(t, path) - conn, err := dialPinnedJournalControl(context.Background(), fd) + conn, err := dialPinnedJournalControl(context.Background(), int(socket.Fd())) require.NoError(t, err) t.Cleanup(func() { _ = conn.Close() }) @@ -49,6 +48,56 @@ func TestDialPinnedJournalControlIgnoresPathReplacement(t *testing.T) { assert.True(t, netErr.Timeout()) } +func TestRotateJournalResolvesAbsoluteSocketSymlinkWithinTargetRoot(t *testing.T) { + root := shortSocketTempDir(t) + realDirectory := filepath.Join(root, "custom", "systemd", "journal") + require.NoError(t, os.MkdirAll(filepath.Join(root, "etc"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "run"), 0o700)) + require.NoError(t, os.MkdirAll(realDirectory, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "etc", "machine-id"), []byte("0123456789abcdef0123456789abcdef\n"), 0o600)) + require.NoError(t, os.Symlink(filepath.FromSlash("/custom/systemd"), filepath.Join(root, "run", "systemd"))) + listener := listenUnixSocket(t, filepath.Join(realDirectory, "io.systemd.journal")) + serverFinished := make(chan error, 1) + go func() { + conn, err := listener.AcceptUnix() + if err != nil { + serverFinished <- err + return + } + _, finished := serveVarlinkResponse(conn, []byte(`{"parameters":{}}`), nil) + serverFinished <- <-finished + }() + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + require.NoError(t, NewClient(target).RotateJournal(context.Background())) + require.NoError(t, <-serverFinished) +} + +func TestDialJournalControlKeepsAbsoluteParentSymlinkWithinTargetRoot(t *testing.T) { + parent := shortSocketTempDir(t) + root := filepath.Join(parent, "host") + outsideDirectory := filepath.Join(parent, "outside-systemd", "journal") + require.NoError(t, os.MkdirAll(filepath.Join(root, "run"), 0o700)) + require.NoError(t, os.MkdirAll(outsideDirectory, 0o700)) + require.NoError(t, os.Symlink(filepath.Dir(outsideDirectory), filepath.Join(root, "run", "systemd"))) + listenUnixSocket(t, filepath.Join(outsideDirectory, "io.systemd.journal")) + + target, err := ResolveTarget(Target{Root: root}) + require.NoError(t, err) + _, err = NewClient(target).dialJournalControl(context.Background(), target.JournalControlSocket) + require.Error(t, err) + assert.ErrorContains(t, err, "inspect journal control socket") +} + +func shortSocketTempDir(t *testing.T) string { + t.Helper() + directory, err := os.MkdirTemp("", "rsock-") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(directory)) }) + return directory +} + func listenUnixSocket(t *testing.T, path string) *net.UnixListener { t.Helper() listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) diff --git a/internal/systemd/journal_varlink_dial_unsupported.go b/internal/systemd/journal_varlink_dial_unsupported.go index 45fb2a1e..67861642 100644 --- a/internal/systemd/journal_varlink_dial_unsupported.go +++ b/internal/systemd/journal_varlink_dial_unsupported.go @@ -14,11 +14,11 @@ import ( "os" ) -func dialJournalControl(ctx context.Context, path string) (net.Conn, error) { +func (c *Client) dialJournalControl(ctx context.Context, path string) (net.Conn, error) { if err := ctx.Err(); err != nil { return nil, err } - info, err := os.Lstat(path) + info, err := c.lstatTargetPath(path) if err != nil { return nil, fmt.Errorf("inspect journal control socket: %w", err) } diff --git a/internal/systemd/journal_varlink_test.go b/internal/systemd/journal_varlink_test.go index 86bec11d..6b76a5ff 100644 --- a/internal/systemd/journal_varlink_test.go +++ b/internal/systemd/journal_varlink_test.go @@ -112,9 +112,10 @@ func TestRotateJournalControlHonorsCancellation(t *testing.T) { } func TestRotateJournalControlRejectsNonSocketEndpoints(t *testing.T) { + client := &Client{} regular := filepath.Join(t.TempDir(), "journal.sock") require.NoError(t, os.WriteFile(regular, []byte("not a socket"), 0o600)) - err := rotateJournalControl(context.Background(), regular) + err := client.rotateJournalControl(context.Background(), regular) assert.ErrorContains(t, err, "not a Unix socket") if runtime.GOOS == "windows" { @@ -122,7 +123,7 @@ func TestRotateJournalControlRejectsNonSocketEndpoints(t *testing.T) { } symlink := filepath.Join(t.TempDir(), "journal-link.sock") require.NoError(t, os.Symlink(regular, symlink)) - err = rotateJournalControl(context.Background(), symlink) + err = client.rotateJournalControl(context.Background(), symlink) assert.ErrorContains(t, err, "not a Unix socket") } diff --git a/internal/systemd/target_path.go b/internal/systemd/target_path.go index 01d3bbe4..58d9723c 100644 --- a/internal/systemd/target_path.go +++ b/internal/systemd/target_path.go @@ -29,14 +29,18 @@ func (c *Client) lstatTargetPath(path string) (fs.FileInfo, error) { } func (c *Client) openTargetFile(path string, followFinalSymlink bool) (*os.File, error) { + return c.openTargetFileFlags(path, followFinalSymlink, os.O_RDONLY) +} + +func (c *Client) openTargetFileFlags(path string, followFinalSymlink bool, flag int) (*os.File, error) { if c.target.root == "" { - return os.Open(path) + return os.OpenFile(path, flag, 0) } root, relative, err := c.openRootedTargetPath(path, followFinalSymlink) if err != nil { return nil, err } - file, operationErr := root.Open(relative) + file, operationErr := root.OpenFile(relative, flag, 0) closeErr := c.closeTargetRoot(root) if operationErr != nil { return nil, combineTargetPathErrors(operationErr, closeErr) From 3495891a40664608895a5c891eda4b3a3e80d866 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 14:58:53 -0400 Subject: [PATCH 52/53] fix(systemd): detect in-place journal changes --- README.md | 2 +- internal/systemd/journal_snapshot.go | 50 ++++++++++++++++++++++- internal/systemd/journal_snapshot_test.go | 33 +++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5b6774e7..09dab149 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Selected journal fields are capped at 64 KiB. `short` escapes embedded newlines, Log reading uses a bounded pure-Go journal-file parser and does not execute host `journalctl` or require cgo or `libsystemd`. It supports regular and compact journal layouts, legacy and keyed DATA hashes, and XZ, LZ4, and Zstandard-compressed DATA objects. Unknown incompatible format features fail with a clear error. -The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID and file identities before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux, the configured journald Varlink socket, and procfs descriptor links at `/proc/self/fd`. The backend pins the socket inode before connecting through its descriptor, so a concurrent path replacement cannot redirect the request; rotation fails closed when descriptor links are unavailable. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. Used alone, `--vacuum-time` removes every eligible archive at least that old. `--vacuum-size` requires `--vacuum-time`; when combined, the age becomes a minimum deletion age and cleanup stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. +The reader opens at most 128 regular, non-symlink journal files, verifies the machine ID plus each file's identity, size, modification time, and parsed header before and after each query, and retries once if concurrent rotation or an in-progress write makes the first snapshot inconsistent. Rotation requires Linux, the configured journald Varlink socket, and procfs descriptor links at `/proc/self/fd`. The backend pins the socket inode before connecting through its descriptor, so a concurrent path replacement cannot redirect the request; rotation fails closed when descriptor links are unavailable. Disk usage and vacuuming use the mounted journal files directly on Linux or macOS. Used alone, `--vacuum-time` removes every eligible archive at least that old. `--vacuum-size` requires `--vacuum-time`; when combined, the age becomes a minimum deletion age and cleanup stops once the size target is reached, leaving newer archives intact even when usage remains above the target. `--rotate` may be combined with vacuum flags, but newly rotated archives remain protected by the time floor. `--dry-run` cannot be combined with `--rotate` and still requires the clean grant and remediation mode. Vacuum thresholds are provided by the `journalctl` command itself. The backend removes only strict systemd archived-journal filenames, never active files, symlinks, hardlinks, or malformed names, and it revalidates each file immediately before deletion. diff --git a/internal/systemd/journal_snapshot.go b/internal/systemd/journal_snapshot.go index d5196d8f..ae0f505f 100644 --- a/internal/systemd/journal_snapshot.go +++ b/internal/systemd/journal_snapshot.go @@ -163,13 +163,59 @@ func (s *journalSnapshot) stable(client *Client) error { if current.Mode()&fs.ModeSymlink != 0 || !current.Mode().IsRegular() || !os.SameFile(current, opened.info) { return journalChanged("journal file %q was replaced during snapshot", opened.path) } - if current.Size() < opened.info.Size() { - return journalChanged("journal file %q shrank during snapshot", opened.path) + if err := opened.stable(); err != nil { + return err } } return nil } +func (f *journalSnapshotFile) stable() error { + before, err := f.file.Stat() + if err != nil { + return fmt.Errorf("inspect open journal file %q before snapshot verification: %w", f.path, err) + } + if err := f.stableMetadata(before); err != nil { + return err + } + + header, err := readJournalHeader(f.path, f.file, uint64(before.Size())) + if err != nil { + if errors.Is(err, errJournalCorrupt) || errors.Is(err, errJournalUnsupported) { + return journalChanged("journal file %q header changed during snapshot verification: %v", f.path, err) + } + return fmt.Errorf("verify journal file %q header: %w", f.path, err) + } + + after, err := f.file.Stat() + if err != nil { + return fmt.Errorf("inspect open journal file %q after snapshot verification: %w", f.path, err) + } + if err := f.stableMetadata(after); err != nil { + return err + } + if header != f.view.header { + return journalChanged("journal file %q header changed during snapshot", f.path) + } + return nil +} + +func (f *journalSnapshotFile) stableMetadata(current fs.FileInfo) error { + if !current.Mode().IsRegular() || !os.SameFile(current, f.info) { + return journalChanged("journal file %q was replaced during snapshot", f.path) + } + if current.Size() < f.info.Size() { + return journalChanged("journal file %q shrank during snapshot", f.path) + } + if current.Size() > f.info.Size() { + return journalChanged("journal file %q grew during snapshot", f.path) + } + if !current.ModTime().Equal(f.info.ModTime()) { + return journalChanged("journal file %q modification time changed during snapshot", f.path) + } + return nil +} + func (s *journalSnapshot) close() error { var result error for _, opened := range s.files { diff --git a/internal/systemd/journal_snapshot_test.go b/internal/systemd/journal_snapshot_test.go index b0b31d66..a785b033 100644 --- a/internal/systemd/journal_snapshot_test.go +++ b/internal/systemd/journal_snapshot_test.go @@ -6,6 +6,7 @@ package systemd import ( + "encoding/binary" "fmt" "os" "path/filepath" @@ -41,6 +42,38 @@ func TestJournalSnapshotDetectsFileReplacement(t *testing.T) { assert.Contains(t, err.Error(), "replaced during snapshot") } +func TestJournalSnapshotDetectsInPlaceHeaderChange(t *testing.T) { + client, journalDir, machineID := newJournalSnapshotTestClient(t) + contents := buildJournalQueryFixture(t, []journalQueryFixtureEntry{ + {bootID: repeatedJournalID(0x22), realtime: 1_700_000_000_000_000, fields: []string{"_SYSTEMD_UNIT=api.service", "MESSAGE=ready"}}, + }) + path := writeJournalSnapshotFixture(t, journalDir, "system.journal", contents, machineID, repeatedJournalID(0x31), repeatedJournalID(0x44)) + + snapshot, err := client.openJournalSnapshot() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, snapshot.close()) }) + opened := snapshot.files[0] + + var changedRealtime [8]byte + binary.LittleEndian.PutUint64(changedRealtime[:], opened.view.header.tailEntryRealtime+1) + writer, err := os.OpenFile(path, os.O_WRONLY, 0) + require.NoError(t, err) + _, err = writer.WriteAt(changedRealtime[:], 192) + require.NoError(t, err) + require.NoError(t, writer.Close()) + require.NoError(t, os.Chtimes(path, opened.info.ModTime(), opened.info.ModTime())) + + current, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, opened.info.Size(), current.Size()) + require.True(t, opened.info.ModTime().Equal(current.ModTime())) + + err = snapshot.stable(client) + require.Error(t, err) + assert.ErrorIs(t, err, errJournalChanged) + assert.Contains(t, err.Error(), "header changed during snapshot") +} + func TestOpenJournalSnapshotRejectsTooManyFilesBeforeParsing(t *testing.T) { client, journalDir, _ := newJournalSnapshotTestClient(t) for index := 0; index <= maxJournalQueryFiles; index++ { From 3cfccbd66040175f7fb3d8dd6d9b1cf7951d9bdc Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Jul 2026 15:16:46 -0400 Subject: [PATCH 53/53] fix(systemd): bound LZ4 journal decoding --- internal/systemd/journal_data.go | 189 +++++++++++++++++++++++--- internal/systemd/journal_data_test.go | 67 +++++++++ internal/systemd/journal_fuzz_test.go | 14 ++ 3 files changed, 251 insertions(+), 19 deletions(-) diff --git a/internal/systemd/journal_data.go b/internal/systemd/journal_data.go index aff5380e..aefa8ea0 100644 --- a/internal/systemd/journal_data.go +++ b/internal/systemd/journal_data.go @@ -6,6 +6,7 @@ package systemd import ( + "bufio" "encoding/binary" "errors" "fmt" @@ -13,7 +14,6 @@ import ( "math" "github.com/klauspost/compress/zstd" - "github.com/pierrec/lz4/v4" "github.com/ulikunitz/xz" ) @@ -21,10 +21,11 @@ const ( journalDataRegularHeaderSize = 64 journalDataCompactHeaderSize = 72 - maxJournalDecodeWindow = 8 * 1024 * 1024 - maxJournalEncodedRead = 8 * 1024 * 1024 - maxJournalLZ4DataSize = 8 * 1024 * 1024 - maxJournalPayloadRead = maxJournalFieldSize + 512 + maxJournalDecodeWindow = 8 * 1024 * 1024 + maxJournalEncodedRead = 8 * 1024 * 1024 + maxJournalLZ4DataSize = 8 * 1024 * 1024 + maxJournalPayloadRead = maxJournalFieldSize + 512 + journalLZ4ReadBufferSize = 4 * 1024 ) type journalDataObject struct { @@ -226,29 +227,179 @@ func (f *journalFileView) readLZ4DataPayload(data journalDataObject, limit int) if decodedSize > maxJournalLZ4DataSize { return nil, false, journalLimit(f.name, data.payloadOffset, "LZ4 DATA payload expands to %d bytes; maximum is %d", decodedSize, maxJournalLZ4DataSize) } - if data.payloadSize-8 > maxJournalEncodedRead { + compressedSize := data.payloadSize - 8 + if compressedSize > maxJournalEncodedRead { return nil, false, journalLimit(f.name, data.payloadOffset, "compressed LZ4 DATA input exceeds %d bytes", maxJournalEncodedRead) } - if decodedSize > uint64(math.MaxInt) || data.payloadSize-8 > uint64(math.MaxInt) { - return nil, false, journalLimit(f.name, data.payloadOffset, "LZ4 DATA payload exceeds the platform allocation range") + + section := io.NewSectionReader(f.reader, int64(data.payloadOffset+8), int64(compressedSize)) + decoded, truncated, err := decodeJournalLZ4Prefix(section, compressedSize, decodedSize, limit) + if err != nil { + return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: %v", err) } + return decoded, truncated, nil +} - compressed := make([]byte, int(data.payloadSize-8)) - if err := readJournalAt(f.name, f.reader, f.size, data.payloadOffset+8, compressed); err != nil { - return nil, false, err +type journalLZ4Source struct { + reader *bufio.Reader + remaining uint64 +} + +// Journal LZ4 payloads are raw blocks, while the public block decoder requires +// a full-size destination. Decode sequences directly to materialize limit+1. +func decodeJournalLZ4Prefix(reader io.Reader, compressedSize, decodedSize uint64, limit int) ([]byte, bool, error) { + outputSize := decodedSize + truncated := decodedSize > uint64(limit) + if truncated { + outputSize = uint64(limit) + 1 + } + if outputSize > uint64(math.MaxInt) { + return nil, false, fmt.Errorf("decoded prefix exceeds the platform allocation range") + } + + output := make([]byte, int(outputSize)) + source := journalLZ4Source{ + reader: bufio.NewReaderSize(reader, journalLZ4ReadBufferSize), + remaining: compressedSize, + } + written := 0 + for source.remaining > 0 { + token, err := source.readByte() + if err != nil { + return nil, false, err + } + + remainingDecoded := decodedSize - uint64(written) + literalLength, err := source.readLength(uint64(token>>4), token>>4 == 15, remainingDecoded) + if err != nil { + return nil, false, fmt.Errorf("literal length: %w", err) + } + literalRead := min(literalLength, uint64(len(output)-written)) + if err := source.readFull(output[written : written+int(literalRead)]); err != nil { + return nil, false, fmt.Errorf("literal data: %w", err) + } + written += int(literalRead) + if literalRead < literalLength { + return output[:limit], true, nil + } + if truncated && written == len(output) { + return output[:limit], true, nil + } + if source.remaining == 0 { + if uint64(written) != decodedSize { + return nil, false, fmt.Errorf("decoded %d of %d bytes", written, decodedSize) + } + return output, false, nil + } + + var offsetBytes [2]byte + if err := source.readFull(offsetBytes[:]); err != nil { + return nil, false, fmt.Errorf("match offset: %w", err) + } + offset := uint64(binary.LittleEndian.Uint16(offsetBytes[:])) + if offset == 0 || offset > uint64(written) { + return nil, false, fmt.Errorf("invalid match offset %d after %d decoded bytes", offset, written) + } + + remainingDecoded = decodedSize - uint64(written) + if remainingDecoded < 4 { + return nil, false, fmt.Errorf("match exceeds the declared %d-byte decoded size", decodedSize) + } + matchLength, complete, err := source.readMatchLength( + uint64(token&0x0f), + token&0x0f == 15, + remainingDecoded-4, + uint64(len(output)-written), + truncated, + ) + if err != nil { + return nil, false, fmt.Errorf("match length: %w", err) + } + matchRead := min(matchLength, uint64(len(output)-written)) + for index := uint64(0); index < matchRead; index++ { + output[written+int(index)] = output[written+int(index)-int(offset)] + } + written += int(matchRead) + if !complete || matchRead < matchLength || truncated && written == len(output) { + return output[:limit], true, nil + } + if uint64(written) == decodedSize { + if source.remaining != 0 { + return nil, false, fmt.Errorf("compressed input continues after %d decoded bytes", decodedSize) + } + return output, false, nil + } + } + + return nil, false, fmt.Errorf("decoded %d of %d bytes", written, decodedSize) +} + +func (s *journalLZ4Source) readByte() (byte, error) { + if s.remaining == 0 { + return 0, io.ErrUnexpectedEOF } - decoded := make([]byte, int(decodedSize)) - n, err := lz4.UncompressBlock(compressed, decoded) + value, err := s.reader.ReadByte() if err != nil { - return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: %v", err) + return 0, err + } + s.remaining-- + return value, nil +} + +func (s *journalLZ4Source) readFull(destination []byte) error { + if uint64(len(destination)) > s.remaining { + return io.ErrUnexpectedEOF + } + n, err := io.ReadFull(s.reader, destination) + s.remaining -= uint64(n) + return err +} + +func (s *journalLZ4Source) readLength(base uint64, extended bool, maximum uint64) (uint64, error) { + if base > maximum { + return 0, fmt.Errorf("length exceeds the remaining %d decoded bytes", maximum) } - if n != len(decoded) { - return nil, false, journalCorrupt(f.name, data.payloadOffset, "decode LZ4 DATA payload: got %d of %d bytes", n, len(decoded)) + if !extended { + return base, nil } - if len(decoded) > limit { - return decoded[:limit], true, nil + for { + next, err := s.readByte() + if err != nil { + return 0, err + } + if uint64(next) > maximum-base { + return 0, fmt.Errorf("length exceeds the remaining %d decoded bytes", maximum) + } + base += uint64(next) + if next != 0xff { + return base, nil + } + } +} + +func (s *journalLZ4Source) readMatchLength(base uint64, extended bool, maximum, needed uint64, prefixOnly bool) (uint64, bool, error) { + if base > maximum { + return 0, false, fmt.Errorf("length exceeds the remaining %d decoded bytes", maximum+4) + } + if !extended { + return base + 4, true, nil + } + for { + next, err := s.readByte() + if err != nil { + return 0, false, err + } + if uint64(next) > maximum-base { + return 0, false, fmt.Errorf("length exceeds the remaining %d decoded bytes", maximum+4) + } + base += uint64(next) + if next != 0xff { + return base + 4, true, nil + } + if prefixOnly && base+4 >= needed { + return base + 4, false, nil + } } - return decoded, false, nil } func readDecodedJournalPayload(name string, offset uint64, algorithm string, reader io.Reader, limit int) ([]byte, bool, error) { diff --git a/internal/systemd/journal_data_test.go b/internal/systemd/journal_data_test.go index e09dabca..d912ae36 100644 --- a/internal/systemd/journal_data_test.go +++ b/internal/systemd/journal_data_test.go @@ -8,6 +8,7 @@ package systemd import ( "bytes" "encoding/binary" + "io" "strings" "testing" @@ -120,6 +121,61 @@ func TestJournalDataPayloadBoundsLZ4Expansion(t *testing.T) { assert.Contains(t, err.Error(), "expands to 8388609 bytes") } +func TestJournalDataPayloadBoundsLZ4PrefixDecode(t *testing.T) { + payload := append([]byte("MESSAGE="), bytes.Repeat([]byte{'x'}, maxJournalLZ4DataSize-len("MESSAGE="))...) + encoded := encodeJournalLZ4(t, payload) + require.Greater(t, len(encoded), 2*journalLZ4ReadBufferSize) + contents, offset := testJournalDataContents(t, encoded, journalObjectCompressedLZ4, false) + reader := &journalCountingReaderAt{ReaderAt: bytes.NewReader(contents)} + view, err := newJournalFileView("bounded-lz4.journal", reader, uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + reader.bytesRead = 0 + + decoded, truncated, err := view.readDataPayload(data, 64) + require.NoError(t, err) + assert.True(t, truncated) + assert.Equal(t, payload[:64], decoded) + assert.LessOrEqual(t, reader.bytesRead, 8+journalLZ4ReadBufferSize) +} + +func TestJournalDataPayloadLZ4PrefixDecodeMatchesPayload(t *testing.T) { + literalPrefix := make([]byte, 512) + for index := range literalPrefix { + literalPrefix[index] = byte(index) + } + payloads := []struct { + name string + payload []byte + }{ + {name: "single-byte matches", payload: bytes.Repeat([]byte{'x'}, 4*1024)}, + {name: "phrase matches", payload: bytes.Repeat([]byte("journal payload "), 8*1024)}, + {name: "literal prefix", payload: bytes.Repeat(literalPrefix, 32)}, + } + + for _, test := range payloads { + encoded := encodeJournalLZ4(t, test.payload) + contents, offset := testJournalDataContents(t, encoded, journalObjectCompressedLZ4, false) + view, err := newJournalFileView(test.name+".journal", bytes.NewReader(contents), uint64(len(contents))) + require.NoError(t, err) + data, err := view.dataObjectAt(uint64(offset)) + require.NoError(t, err) + + limits := []int{1, 17, 64, 1024, maxJournalPayloadRead, len(test.payload) - 1, len(test.payload), len(test.payload) + 1} + for _, limit := range limits { + if limit <= 0 || limit > maxJournalPayloadRead { + continue + } + decoded, truncated, err := view.readDataPayload(data, limit) + require.NoErrorf(t, err, "%s with limit %d", test.name, limit) + expectedLength := min(limit, len(test.payload)) + assert.Equalf(t, test.payload[:expectedLength], decoded, "%s with limit %d", test.name, limit) + assert.Equalf(t, len(test.payload) > limit, truncated, "%s with limit %d", test.name, limit) + } + } +} + func TestJournalDataPayloadRejectsUnboundedReadLimit(t *testing.T) { contents, offset := testJournalDataContents(t, []byte("MESSAGE=hello"), 0, false) view, err := newJournalFileView("limit.journal", bytes.NewReader(contents), uint64(len(contents))) @@ -207,3 +263,14 @@ func encodeJournalZSTD(t *testing.T, payload []byte) []byte { defer encoder.Close() return encoder.EncodeAll(payload, nil) } + +type journalCountingReaderAt struct { + io.ReaderAt + bytesRead int +} + +func (r *journalCountingReaderAt) ReadAt(destination []byte, offset int64) (int, error) { + n, err := r.ReaderAt.ReadAt(destination, offset) + r.bytesRead += n + return n, err +} diff --git a/internal/systemd/journal_fuzz_test.go b/internal/systemd/journal_fuzz_test.go index 3b52313e..40878fd3 100644 --- a/internal/systemd/journal_fuzz_test.go +++ b/internal/systemd/journal_fuzz_test.go @@ -21,6 +21,20 @@ const ( journalFuzzTimeout = 10 * time.Millisecond ) +func FuzzJournalLZ4Prefix(f *testing.F) { + f.Add(append([]byte{0x90}, []byte("MESSAGE=x")...), uint64(9), uint16(4)) + f.Add([]byte{0x1f, 'a', 0x01, 0x00, 12}, uint64(32), uint16(8)) + + f.Fuzz(func(t *testing.T, compressed []byte, rawDecodedSize uint64, rawLimit uint16) { + if len(compressed) == 0 || len(compressed) > maxJournalFuzzEncodedData { + return + } + decodedSize := rawDecodedSize%maxJournalLZ4DataSize + 1 + limit := int(rawLimit%maxJournalFuzzPayloadRead) + 1 + _, _, _ = decodeJournalLZ4Prefix(bytes.NewReader(compressed), uint64(len(compressed)), decodedSize, limit) + }) +} + // FuzzJournalObjects drives every bounded object decoder reachable from a // parsed header. Corrupt and unsupported inputs are expected; the contract is // that they return without panicking or allocating from untrusted sizes.