diff --git a/.docs-lint-scope b/.docs-lint-scope index 3834a34d..89f83c0a 100644 --- a/.docs-lint-scope +++ b/.docs-lint-scope @@ -25,6 +25,4 @@ README.md docs/architecture.md docs/configuration.md -docs/design/attribution-branch-findings.md -docs/design/attribution-publish-and-join.md -docs/design/attribution-metrics-proposal.md +docs/spec/attribution.md diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 6564aa86..65ba8d68 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -155,10 +155,14 @@ type GitTargetSpec struct { // path identity-complete, never append or co-mingle), not a second placement // namespace the user has to configure. A user routes Secrets the same way they // route anything else — by naming their type in ByType. When a resource's type -// has no ByType entry and no Default, placement falls back to following the layout -// already established by sibling resources in the repository, and finally to the -// canonical, versionless {namespaceOrCluster}/{group}/{resource}/{name}.yaml path -// when there is nothing to follow. Because that fallback omits the API version, +// has no ByType entry and no Default, the new document goes beside the folder's +// kustomization when the whole folder is governed by exactly one supported +// kustomization (so the file is reachable from a render root instead of being +// written where kustomize would never build it), and otherwise at the built-in +// canonical, versionless {namespaceOrCluster}/{group}/{resource}/{name}.yaml path. +// Nothing infers a destination from where the repository keeps other resources of +// the same type: a layout this operator cannot derive from one root is declared +// here or it is canonical. Because the canonical path omits the API version, // objects that differ only by version share a file; a target that watches several // versions of the same group/resource and wants them separated must use a // ByType/Default template that includes {version}. @@ -173,8 +177,9 @@ type GitTargetPlacementSpec struct { ByType map[string]string `json:"byType,omitempty"` // Default is the path template used for a new resource whose type has no ByType - // entry. Omitted, it falls through to sibling-layout inference and then the - // built-in canonical path. A bundling default (one that is not identity-complete, + // entry. Omitted, it falls through to the folder's one supported kustomization + // root, if it has exactly one, and then to the built-in canonical path. + // A bundling default (one that is not identity-complete, // such as "all.yaml") is only valid when a sensitive resource can never reach it // — give every sensitive type an explicit identity-complete ByType entry. // +optional diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 07864f8f..96b7b5e8 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -308,8 +308,9 @@ spec: default: description: |- Default is the path template used for a new resource whose type has no ByType - entry. Omitted, it falls through to sibling-layout inference and then the - built-in canonical path. A bundling default (one that is not identity-complete, + entry. Omitted, it falls through to the folder's one supported kustomization + root, if it has exactly one, and then to the built-in canonical path. + A bundling default (one that is not identity-complete, such as "all.yaml") is only valid when a sensitive resource can never reach it — give every sensitive type an explicit identity-complete ByType entry. type: string diff --git a/docs/INDEX.md b/docs/INDEX.md index 3f82a498..d9a8b8cd 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -37,11 +37,12 @@ misled. Full list in [`spec/README.md`](spec/README.md); the ones that carry a | Spec | The rule | |---|---| | [`manifest-system.md`](spec/manifest-system.md) | the whole live → Git pipeline, and every invariant below in summary | +| [`attribution.md`](spec/attribution.md) | **how a commit gets its author.** Deletion is attributed at intent time; the publish half files a fact under the strongest key it has, the join half walks the tiers strongest-first, and neither branches on the type. The single reference, folded from six design records | | [`current-manifest-support-review.md`](spec/current-manifest-support-review.md) | all-or-nothing folder claim; never half-write a multi-doc file; **refuse rather than prune** | | [`manifestedit-field-ownership-spike.md`](spec/manifestedit-field-ownership-spike.md) | the API wins — full-object ownership, never field-subset | | [`reconcile-via-watchlist-mark-and-sweep.md`](spec/reconcile-via-watchlist-mark-and-sweep.md) | **no bookmark, no sweep** | | [`contextual-namespace-and-kustomize-folder-editing.md`](spec/contextual-namespace-and-kustomize-folder-editing.md) | kustomize namespace inference; the supported subset | -| [`gittarget-new-file-placement-rules.md`](spec/gittarget-new-file-placement-rules.md) | where a new resource's file goes | +| [`gittarget-new-file-placement-rules.md`](spec/gittarget-new-file-placement-rules.md) | where a new resource's file goes: declared, the folder's one kustomize root, canonical. Sibling inference is removed, and kept as history | | [`sops-single-file-no-multidoc.md`](spec/sops-single-file-no-multidoc.md) | one encrypted file is one document | | [`scale-subresource-audit-rehydration.md`](spec/scale-subresource-audit-rehydration.md) | `/scale` only; every other subresource ignored | | [`commit-window-refactor.md`](spec/commit-window-refactor.md) | one grouped commit = one (author, GitTarget) | @@ -65,21 +66,18 @@ says what we support and refuse** — and then its kustomize field taxonomy, the write boundary, the orchestrator/expansion line, and how secrets are handled. -Fifteen other open items: +Sixteen other open items: | Doc | Open question | |---|---| -| [`open-asks-priority.md`](design/open-asks-priority.md) | **the work queue.** Reconciled 2026-07-29 to what the attribution branch shipped: the fact stream, consumer ask #23, the name tier and metrics Phase 1 are struck from it and recorded in "already shipped", the residue they leave (the removal-wait decision, the head-of-line block on the shard, the aggregated create) is ranked, and #5 loses one of its two arguments because #23's fix retired it. Three backlogs are open at once — the gitops-api consumer asks, the maintainer review's unbuilt block (F6, F9, F10), and the config-surface proposal (B1–B6) — and they overlap. Merges them into one ordered queue under four stated tests, and makes one design call against what was asked: **delete Option C sibling inference** rather than ship an off-switch for it, because it lets a human's edit to the repository change the operator's behaviour with nothing in status recording the move, its central guard has already failed once by cascading, and the explainability its own spec made mandatory was never built. That answers the namespace-leak ask by removal, and means `spec.placement.mode` is never built. Open: whether the removal ships with an Event on the first fall-back to canonical | +| [`open-asks-priority.md`](design/open-asks-priority.md) | **the work queue.** Swept 2026-07-30 against the 0.41.0 release, which carries the attribution model and the placement break and deliberately **not** the GitTarget wave: every Tier 2 entry that changes a `GitTarget` field is part of postponed [#294](https://github.com/ConfigButler/gitops-reverser/issues/294) and is not independently schedulable, while the Tier 1 entries are not and must not wait for it. One entry moved up in the sweep, to Tier 1: a declared path into a kustomize subdirectory produces a file that is in Git and rendered by nothing, which is the product being silently wrong. Reconciled 2026-07-29 to what the attribution branch shipped: the fact stream, consumer ask #23, the name tier and metrics Phase 1 are struck from it and recorded in "already shipped", the residue they leave (the removal-wait decision, the head-of-line block on the shard, the aggregated create) is ranked, and #5 loses one of its two arguments because #23's fix retired it. Three backlogs are open at once — the gitops-api consumer asks, the maintainer review's unbuilt block (F6, F9, F10), and the config-surface proposal (B1–B6) — and they overlap. Merges them into one ordered queue under four stated tests, and makes one design call against what was asked: **delete Option C sibling inference** rather than ship an off-switch for it, because it lets a human's edit to the repository change the operator's behaviour with nothing in status recording the move, its central guard has already failed once by cascading, and the explainability its own spec made mandatory was never built. That answers the namespace-leak ask by removal, and means `spec.placement.mode` is never built. **The deletion has shipped**, together with the placement metrics the argument had said to lead *away* from — an objection to their labels, which naming the GitTarget and the type retires — and "what the deletion taught" records the two things building it found: namespace inheritance was a second implementation of a rule that belonged to the governing kustomization, and it was missing the check that the transformer names the resource's own namespace. Open: whether a fall-back to canonical also raises an Event on the GitTarget, and `status.layout` | +| [`placement-visibility-and-declared-defaults.md`](design/placement-visibility-and-declared-defaults.md) | the three questions the inference deletion left, **decided and then not built** — PR #291 shipped the deletion, the counters and the namespace-transformer fix, and none of the eight items this page had queued behind them, which the page now says. The residue is filed as [#295](https://github.com/ConfigButler/gitops-reverser/issues/295) (correctness: a declared path into a kustomize subdirectory is never rendered, and the identity gate rejects the versionless canonical path) and [#296](https://github.com/ConfigButler/gitops-reverser/issues/296) (visibility: `status.layout`, the ambiguous render root, the `declared` metric split, `{kindLower}`). Its Question 2 is superseded outright by the layout model. What still stands: **Keep `canonical`** as the name for the built-in path and split `declared` into `byType`/`default`, because reusing "default" for both a declaration and the absence of one makes the metric unreadable. **No CRD default for `placement.default`**, and the reason is structural rather than the two that look obvious: a defaulted default is never empty, so it shadows the kustomize-root step and every new file in an overlay would take the canonical path, in Git and rendered by nothing. The validation failure is real but the rule doing the rejecting is itself wrong, and the persistence objection is a trade we could take; defaulting the Secret route to work around the first is a floor that vanishes when a user writes any `byType` entry, because map defaults never merge. **`status.layout` instead**, with five worked examples (greenfield, kustomize overlay, brownfield missing one rule, two ambiguous roots, a refusal from an operator-configured sensitive type) over the `MarkTargetRetention` seam, which already enqueues on change and so retires the "the data plane cannot notify the GitTarget" objection. **`{kindLower}`, not a `toLower` function.** Carries three findings that changed a decision: `IdentityCompletePlacementTemplate` demanding `{version}` contradicts the versionless-path decision; two supported kustomizations still produce a file nothing renders and nothing counts; and **a declared path into a subdirectory of a kustomize folder is registered only when render-root scoping happens to be in force**, so one `byType` line reproduces the unrendered-file bug today. Fixing that last one (walk up to the nearest kustomization) also weakens the case against the CRD default from a correctness wall to a legibility trade, which the page says rather than leaving the stronger argument standing | +| [`gittarget-layout-model.md`](design/gittarget-layout-model.md) | **postponed to a later deployment, filed as [#293](https://github.com/ConfigButler/gitops-reverser/issues/293)** — the proposal the placement questions were circling around: a path template is the wrong primitive, so declare what the folder IS. `spec.layout.kind` with the values `Auto`, `Kustomize`, `Tree`, `Flat` and `Template`, plus `byType` overrides valid under every kind, with two rules that carry the value: whatever chose the path, the file is registered with the kustomization that governs it (so F10 becomes unstatable rather than fixed), and a structural kind excludes a blanket `default` (so a declared template can no longer silently disable the render root). `kind: Auto` is a safe CRD default because it NAMES the structural rule instead of standing in front of it, which is why defaulting a mode works where defaulting a path did not, and it is declared inference rather than the undeclared kind that was deleted. `kind: Kustomize` with `create: true` bootstraps an empty repository into a folder `kubectl apply -k` can build. Seven worked examples, a status shape with `declaredKind` beside the resolved `kind`, a mechanical migration for every current configuration, and an argument that the layout should NOT be its own CRD: a shared object changing where N folders write, with nothing on the GitTarget recording it, is the same defect as sibling inference with a different actor, the shared thing is four lines, and generators already solve reuse. Also carries the namespace half: `scope: SingleNamespace` is a STRUCTURAL claim that must agree with the authorization bound `allowedSourceNamespaces`, and it cannot be derived because that matcher may be absent and because the namespaces that arrive come from WatchRule objects that do not own the folder; `writeNamespace` with the values `FromContext`, `Always` and `Never` replaces the inference that decides whether `metadata.namespace` is written, which is the one inference an empty folder cannot perform, and `create: true` lets the operator ESTABLISH the convention by writing `namespace:` into the kustomization it creates. The layout is **immutable** except a widening transition, because GitTarget has no finalizer so recreating one re-adopts every document by identity, and `Auto` resolves once and pins the result so a deleted `kustomization.yaml` cannot silently re-lay-out the folder. Open: whether `scope` should be derived and materialized at creation instead of declared | +| [`gittarget-api-wave.md`](design/gittarget-api-wave.md) | **postponed, filed as [#294](https://github.com/ConfigButler/gitops-reverser/issues/294). Not in 0.41.0**, which already carries the attribution model and the placement break. One breaking wave on GitTarget, sequencing the layout model with the maintainer review's still-open API block (F6, F10, F12's reference nit, §3's pushbacks) and the queue's Tier 2 items (B4, B1, #5, #6). The batching argument is the weaker half; the stronger one is that four of them are the same decision seen from different angles: **the folder is described on the GitTarget and the connection describes only the connection**, which is why `commitWindow` and `commit.message` move off `GitProvider`. Two findings change the layout design rather than accompanying it: `spec.mode: Observe` becomes how a layout is adopted safely (a dry run over `status.layout` instead of declare-and-hope), and `spec.interval` plus an observation pass is what keeps the scan-derived half of that status fresh for a target that writes nothing. `spec.suspend` is a precondition rather than a rider, because a layout that creates a `kustomization.yaml` needs a stop button. Records that F7 already shipped the EventRecorder the placement Event was said to be too expensive for, that layout is mutable like `prune`, that F9 stays OUTSIDE the wave because its answer constrains the enum work, and that the version stays `v1alpha3` with loud rejections rather than paying for a conversion path | | [`docs-linting.md`](design/docs-linting.md) | how to mechanize [`style-guide.md`](style-guide.md) with markdownlint-cli2 and Vale. Both are wired into `task lint`, gated on the files [`.docs-lint-scope`](../.docs-lint-scope) lists rather than the whole tree: 102 of 174 files fail markdownlint and 148 of 174 fail Vale, so the two backlogs need different gates. Open: how the scope list grows to cover the tree, the `MD013` limit, and whether `AGENTS.md` and the chart READMEs are in scope | -| [`attribution-deletion-intent-actor.md`](design/attribution-deletion-intent-actor.md) | a finalized deletion used to be attributed to the controller that cleared the finalizer rather than to the human who asked for it. Reproduced from a two-actor mutation-lab capture (`configmap/deletion-intent-actor`, with a tunable hold between the phases) and pinned by a corpus-driven unit test: the human's `delete` and the controller's finalizer `patch` both return a body carrying the resourceVersion the DELETION stamped, so both facts are filed under the same `(uid, resourceVersion)` key and the index is last-writer-wins — the deleter's fact is not outranked, it is replaced. **Built**: a STICKY removal pointer — a fact about a deletion may not be overwritten by a fact about a write — keyed strictly by uid, consulted ahead of the exact tier for a removal, and bounded by the index's caps rather than the join TTL, because a uid is unique across space and time — in memory, so a restart still re-warms from one TTL of stream retention. It ships the `delete_sticky` value on `attribution_resolutions_total{tier}` and collapses the full-grace wait a `Terminating` object seen on replay used to pay. Records why the cheaper alternatives are second: filing delete facts under the name tier answers at a tier that misstates the evidence, and making the exact entry write-once makes correctness depend on fact arrival order and does nothing for replay. Open: nothing | | [`attribution-removal-wait-options.md`](design/attribution-removal-wait-options.md) | a removal now waits for evidence about the DELETION rather than accepting the object's last write, which stopped it naming whoever last edited the object as the author of a deletion they did not perform. Enumerates the eight situations a resolution can be in and shows the cost is concentrated in exactly one: a removal for which no delete fact will ever arrive (a graceful pod delete, a status-only removal, a type the audit policy skips) spends the whole grace to return the answer it had at t=0, measured at ~3.1s against ~70ms when evidence is present. Prices five options against that, and recommends a per-route watermark — stop waiting once the fact stream has demonstrably moved past this event — over a second timeout flag whose right value lives in the API server's config rather than ours. Open: the decision, and how common the case is outside the e2e suite | -| [`attribution-metrics-proposal.md`](design/attribution-metrics-proposal.md) | a phased attribution metric surface, revised after review cut an earlier draft of thirteen new families down to a first release that covers health and the unseen loss paths. Splits `result` into `tier` and `actor_kind` (how `commits_total` already models it) and `weak` into `latest` and `resource_version`, taking the break in the release that has broken `result` anyway. Adds watch-queue delay, follower error and last-success health, a `no_attribution_fact` outcome on the existing bounded audit vocabulary, and a decode-error counter for the one loss path with no symptom at all: both transports discard an undecodable stream entry and advance past it with no log and no metric. Records what the first draft got wrong and why, including a proposed series that would have been permanently zero and a gauge that would have counted registrations rather than blocked resolvers. Its Phase 1 is now Phase 1 of [`metrics-observability-plan.md`](design/metrics-observability-plan.md), which absorbed the surface and records the drift it had accumulated; this stays as the reasoning trail. **Phase 1 has shipped** — the migration is in [`UPGRADING.md`](UPGRADING.md). Open: nothing structural | -| [`attribution-publish-and-join.md`](design/attribution-publish-and-join.md) | the reference for what attribution's two halves each do, exactly: the publish side that turns one audit event into zero or one fact and files it under the keys it happens to have, and the join side that walks the tiers strongest-first to name an author for a watch event. A flowchart per half, the tier table, and the two rules that are easy to miss (a removal never answers with a write fact without looking further; an exact-capable event may never fall through to the removal tiers). Also answers whether anything special-cases a type: nothing does, every branch is on the verb or on which fields are present, and the two ConfigMap deletes in the corpus — one answered with the object, one with a `Status` — are the standing argument that a type-based rule would be unsound | -| [`attribution-branch-findings.md`](design/attribution-branch-findings.md) | what the attribution switchover's loose ends turned out to be, measured rather than reasoned. The mutation lab was serving `/audit-webhook` as an exact path while the cluster posts to the named `/audit-webhook/default`, so every audit event 404'd and every audit-carrying scenario timed out — a routing mismatch that reads exactly like a broken cluster. With it fixed, the corpus answers the aggregated-API removal question: a proxied delete is audited with a name but **no uid and no resourceVersion**, so the exact and latest tiers can never match it, and a proxied `deletecollection` returns **no response body**, so its fact carries no uid set and the join must fall back to scope. Separates the missing name from the missing body — a `generateName` create recovers both from the response object, an aggregated write has nothing to recover from — and prices a name tier against accepting that aggregated types are collection-only. Open: the tier decision, and whether a CommitRequest missing the window of the write it follows by two seconds is new on this branch | -| [`attribution-fact-identity.md`](design/attribution-fact-identity.md) | several `ClusterProvider`s may name one physical cluster, but a kube-apiserver posts audit to one route, so only one of those names is ever fed and every other one authors `unknown (attribution unresolved)`. Proposes a declared `spec.attribution.auditRoute` that partitions the facts instead of `metadata.name`, so several providers can share one cluster's facts while cloned clusters stay separate, ingestion loses its last Kubernetes read, and a misrouted provider becomes loud. Renames the key infix and the annotation-key flag to the same word | -| [`attribution-wait-poll-vs-push.md`](design/attribution-wait-poll-vs-push.md) | **superseded by the above, kept as the reasoning trail.** a watch event needs its author before it can be routed, and the audit fact naming that author may not have arrived yet, so `ResolveAuthor` polls Redis every 150ms for up to three seconds on the watch shard's own goroutine. Separates the wait (forced: two unordered deliveries out of one kube-apiserver, and the commit window groups by author) from the poll (a choice). Answers which of the two fires first: the watch, nearly always, because audit delivery is batched by the apiserver while the watch is streamed, so the first lookup is a near-guaranteed miss and the loop runs to completion on every attributable event. Six options priced against that, from shifting the first check to the delivery floor and a circuit breaker for an audit route that has never resolved anything, through a Redis publish and subscribe, to the reassembly-buffer design that stops blocking the watch shard. Open: whether the wait population is dominated by resolved-late (favors publish and subscribe) or never-resolved (favors the buffer), plus a proposed per-scenario timing report from the mutation-capture lab | | [`watch-and-catalog-architecture.md`](design/watch-and-catalog-architecture.md) | the target three-layer watch model — **needs a human call before building** | -| [`metrics-observability-plan.md`](design/metrics-observability-plan.md) | the canonical metrics plan, reconciled to the code after the fact-stream switchover and now carrying the attribution surface from [`attribution-metrics-proposal.md`](design/attribution-metrics-proposal.md). Reads the product as one pipeline — watch events arrive, and are processed into commits — and maps a metric to each stage. The attribution join is built and correctly labelled; **watch ingestion, shard queue delay, and the relevance filter are still dark**. **Phase 1 — the attribution relabel plus the loss-path counters — has shipped**; Phase 2 is the watch stage, Phase 3 the filter and push health, Phase 4 the dashboard and alerts. Open: Phases 2-4, and the dashboard JSON is deliberately not written until the watch families exist | +| [`metrics-observability-plan.md`](design/metrics-observability-plan.md) | the canonical metrics plan, reconciled to the code after the fact-stream switchover and now carrying the attribution surface that shipped (documented in [`spec/attribution.md`](spec/attribution.md)). Reads the product as one pipeline — watch events arrive, and are processed into commits — and maps a metric to each stage. The attribution join is built and correctly labelled; **watch ingestion, shard queue delay, and the relevance filter are still dark**. **Phase 1 — the attribution relabel plus the loss-path counters — has shipped**; Phase 2 is the watch stage, Phase 3 the filter and push health, Phase 4 the dashboard and alerts. Open: Phases 2-4, and the dashboard JSON is deliberately not written until the watch families exist | | [`reconcile-triggering.md`](design/reconcile-triggering.md) | which controllers still fail to wake up | | [`multi-source-audit-ingress-hardening.md`](design/multi-source-audit-ingress-hardening.md) | how independent sources authenticate to a named audit route, when annotation routing is trustworthy, and how multi-provider ingestion remains fair | | [`release-image-reuse-plan.md`](design/release-image-reuse-plan.md) | PRs 2–5 unstarted | diff --git a/docs/TODO.md b/docs/TODO.md index 08d2d865..7975bc13 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -64,9 +64,14 @@ This file is meant to track the smaller current backlog, not historical notes. Secrets and CozyStack `tenantsecrets`; resources with sensitive fields under shapes such as `spec.credentials` need an explicit field policy or full-file encryption decision. -- [ ] Revisit output layout. - Think about better control over target folders and whether some use cases should support multiple - resources per file. +- [ ] Revisit output layout. **Now designed and postponed, not open-ended.** The answer is to declare + what the folder *is* rather than a path template: `GitTarget.spec.layout`, designed in + [gittarget-layout-model.md](design/gittarget-layout-model.md) and filed as + [#293](https://github.com/ConfigButler/gitops-reverser/issues/293), sequenced with the rest of the + breaking `GitTarget` work as [#294](https://github.com/ConfigButler/gitops-reverser/issues/294). + Deliberately **not** in 0.41.0, which already carries the new attribution model and the + sibling-inference removal. Multiple resources per file is bundle support, which exists for + match-first today and is a separate question from where a *new* file goes. - [ ] Reduce duplication between `WatchRule` and `ClusterWatchRule` code paths where it makes sense. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 7ccb8be5..b1c3a9f1 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,6 +7,81 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. +## New resources land where you declare, not where the folder's other documents live + +Sibling inference is gone. A resource with no document in Git yet is placed by the first of three +things that applies, and nothing else: + +1. the GitTarget's `spec.placement.byType` entry for its type, or `spec.placement.default`; +2. the folder's one supported `kustomization.yaml`, when the whole folder has exactly one — the file + lands beside it and joins its `resources:` list; +3. the built-in canonical path, `{namespaceOrCluster}/{groupPath}/{resource}/{name}{sensitiveSuffix}`: + a cluster-scoped resource uses the literal `_cluster/` in place of the namespace, a core resource + omits the group segment, there is no version segment, and a sensitive resource gets `.sops.yaml` + instead of `.yaml`. + +Before this, a folder with no declared placement was read for its layout: a new ConfigMap was appended +to the bundle the other ConfigMaps shared, or written beside them one-per-file. That is what changes. + +**Who is affected.** A target whose repository this operator created is **unaffected** — such a folder +already used canonical paths, which inference also produced. A target pointed at a **hand-authored** +folder with a layout of its own is affected: a resource of a type that folder already holds, in a +namespace or with a name it has never held, now gets the canonical path instead of joining the +existing file or directory. Nothing already in Git moves — an existing document is still edited in +place at its current location, forever. + +**What to do about it.** Declare the layout the folder means: + +```yaml +spec: + placement: + byType: + v1/configmaps: "all.yaml" # keep bundling ConfigMaps into one file + v1/secrets: "team-a/secrets/{name}.sops.yaml" # one encrypted file per Secret +``` + +A declared template does everything inference did and says so on the page. A **kustomize** folder needs +no declaration: step 2 already places new files where that folder builds them. + +**One shape worth expecting.** In a folder that kustomize builds, a bundle file is no longer extended +by default. A new resource gets a file of its own beside the `kustomization.yaml` and an entry added to +its `resources:` list, so the build file changes where it previously did not (the bundle was already +listed, so extending it needed no entry). Both outcomes mirror the resource and both render; the new +one keeps a resource the operator placed out of a file a human curated, and it is undone by declaring +the bundle in `byType`. + +**How to tell whether it affects you**, before or after upgrading — every placement is counted, by +GitTarget and by type: + +```promql +sum by (gittarget_namespace, gittarget_name, group, version, resource) ( + increase(gitopsreverser_placements_total{source="canonical"}[24h])) +``` + +Each series is a type that took the built-in path. For a canonical-layout folder that is simply the +layout. For a folder with a convention of its own it is the `byType` line to add. `source="declared"` +and `source="kustomize_root"` need no attention. Two companions ship with it: +`gitopsreverser_placement_refusals_total{reason}` (resources the writer declined to place — each one +is absent from the mirror) and `gitopsreverser_placement_kustomization_entries_total{outcome}`, whose +`failed` value is a new file committed outside every render. See +[interpreting-metrics.md](interpreting-metrics.md). + +**Why the feature was removed rather than made switchable.** It let an edit to the *repository* change +where the operator writes, with no Kubernetes object changing and nothing in status recording the move +— delete enough of one namespace's documents from a shared bundle and the next new one takes a +different path. Its namespace-safety guard had also failed once by cascading: one wrong append made a +per-namespace file look namespace-agnostic, which legitimized it for every later resource, which +collapsed a whole type into one file. There is deliberately **no** `spec.placement.mode` flag to turn +inference back on: an off-switch for a removed feature is a permanent API field bought to solve a +temporary problem. The full argument is in +[`open-asks-priority.md`](design/open-asks-priority.md). + +**One related fix.** A new document in a directory whose kustomization sets `namespace:` omits +`metadata.namespace` only when the transformer names the resource's **own** namespace. When it names a +different one the namespace is now written explicitly — omitting it would have handed the namespace to +kustomize and rendered a different object than the one being mirrored. This also now applies to a path +you declared, which previously wrote a `namespace:` line the rest of that folder omits. + ## 0.41.0 — attribution facts travel on a selectable transport, and Redis is no longer implied Attribution stopped meaning Redis. The audit receiver appends its facts to a per-type **stream**, diff --git a/docs/architecture.md b/docs/architecture.md index 71118150..3f8d6a34 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -538,8 +538,8 @@ The **built-in default** path is `{spec.path}/{namespace}/{group}/{resource}/{na first, the API group omitted for core resources, no version segment, and a `.sops.yaml` suffix for sensitive resources; a cluster-scoped resource uses the literal `_cluster/` in place of the namespace (an illegal Kubernetes namespace name, so it can never clash with a real one). -But that default is only the cold-start seed: a new resource first follows its **siblings'** existing -layout, and a `GitTarget` can declare its own placement policy. Details and the placement policy are in +That default is what a new resource gets unless something more specific applies: a `GitTarget`'s own +declared placement policy, or a folder that kustomize builds from a single root. Details are in [File Placement](#file-placement). *** @@ -614,8 +614,8 @@ per-mutation change log. - **Fact transport (per-type stream)**: [internal/queue/fact_stream.go](../internal/queue/fact_stream.go) - **In-process fact index**: [internal/queue/fact_index.go](../internal/queue/fact_index.go) - **Resolver (grace window join)**: [internal/watch/author_resolver.go](../internal/watch/author_resolver.go) -- **Design**: [attribution-publish-and-join.md](design/attribution-publish-and-join.md), - [attribution-fact-stream.md](finished/attribution-fact-stream.md) +- **Spec**: [spec/attribution.md](spec/attribution.md) (the single reference) +- **Design record**: [attribution-fact-stream.md](finished/attribution-fact-stream.md) Attribution runs only when `--author-attribution=true`. A normal source posts audit `EventList` payloads to `/audit-webhook/`, where the route is @@ -734,8 +734,8 @@ TTL does not bound: a uid is unique across space and time, so the statement cann and its horizon is the index's caps instead. It is still in-memory: a restart re-warms the index from one TTL of stream retention like everything else. Without the slot, a finalizer patch's fact overwrites the deleter's, because both carry the resourceVersion the *deletion* stamped -([attribution-deletion-intent-actor.md](design/attribution-deletion-intent-actor.md)). **A removal -never returns on a write fact without looking further:** the per-object tiers are last-writer-wins, +([spec/attribution.md](spec/attribution.md)). **A removal never returns on a write fact +without looking further:** the per-object tiers are last-writer-wins, so for a removal they hold whoever last *edited* the object, which is not who deleted it; such a match is held as a fallback while the wait continues for evidence about the deletion itself. And **an exact-capable event may not fall through to the removal tiers:** a create or update presents @@ -1111,7 +1111,7 @@ hydrates only touched files into buffers for the commit, and flushes only change - **Upserts:** if a managed document for the resource already exists, patch it in place (preserving siblings in a multi document file); if it is sensitive, encrypt the whole document again at its existing path; if no document exists, place a new file per [File Placement](#file-placement) (declared policy, - then sibling inference, then the canonical default). + then the folder's one kustomize root, then the canonical default). - **Kustomize override edit-through:** a live value produced by a well-formed `images:` or `replicas:` entry in the document's kustomization chain is written back to that entry (comment-preserving, only fields the entry already declares); the source manifest keeps its bytes. Anything the inversion cannot @@ -1135,21 +1135,27 @@ placed never moves a file already in Git. A new resource is placed by the first 1. **Declared policy (`spec.placement`).** A `GitTarget` can declare a `byType` map (exact `[group/]version/resource` → path template) plus a `default` template, rendered from a small brace-variable path language (`{namespace}`, `{group}`, `{resource}`, `{name}`, …). -2. **Sibling inference.** With no matching declared template, the new resource follows the layout its - siblings already use: appended to the bundle its type shares, or placed one-per-file beside them. - so pointing a target at an existing folder continues that folder's convention. When the whole - subtree is governed by one supported kustomization and the type is brand new, the file lands beside - that kustomization and gets a `resources:` entry. -3. **Canonical fallback.** With nothing to follow (an empty repo, a brand-new type), the built-in default +2. **The folder's one kustomize root.** When the whole writable subtree is governed by exactly one + supported `kustomization.yaml`, the file lands beside it and gets a `resources:` entry in the same + commit. This step is a structural fact rather than a reading of the folder's conventions: the + canonical path below is a tree a `resources:` graph cannot reach, so a file written there would never + be rendered. Two supported kustomizations is ambiguous and declines. +3. **Canonical fallback.** Otherwise the built-in default `{spec.path}/{namespace}/{group}/{resource}/{name}.yaml`: namespace-first, group omitted for core, no - version, `_cluster/` for cluster-scoped, `.sops.yaml` for sensitive, so a fresh target is deterministic - and self-propagating. + version, `_cluster/` for cluster-scoped, `.sops.yaml` for sensitive. + +**The layout of the folder's other documents is not an input.** An earlier release followed it (sibling +inference), which made a human's edit to the repository change where the operator wrote next, with no +Kubernetes object changing and nothing in status recording the move. It was removed; a layout the ladder +cannot derive is declared in `spec.placement`, and +`gitopsreverser_placements_total{source="canonical"}` names the target and type that needs the line. Sensitivity is a write-safety classifier, not a placement input: whatever path is chosen, a sensitive resource is written encrypted, is never appended to an existing file, and is never co-mingled with a plaintext document. When those guarantees cannot be honoured (e.g. a bundling `default` would route a -sensitive resource into a shared file), the resource is **skipped fail-safe** (logged per-resource and -counted in the resync summary as `placementSkipped`) rather than written unsafely. +sensitive resource into a shared file), the resource is **refused fail-safe** rather than written +unsafely: logged per-resource, counted in the resync summary as `placementSkipped`, and counted by +`gitopsreverser_placement_refusals_total{reason}`. ### Bootstrap, encryption, and signing @@ -1382,7 +1388,7 @@ Deeper dives live under [docs/design/](design/): - [Watch-first ingestion design record](finished/watch-first-ingestion-architecture.md): historical context for the current watch-only object-state model and optional audit attribution. -- [How attribution works: the publish side and the join side](design/attribution-publish-and-join.md): +- [Attribution: how a commit gets its author](spec/attribution.md): the two halves, the tier ladder, and the wait. - [Attribution facts as a stream, not a keyspace](finished/attribution-fact-stream.md): the shipped transport seam, the in-process index, and what running without Redis costs. diff --git a/docs/configuration.md b/docs/configuration.md index 5c5aaf4a..dcdbf115 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -470,8 +470,8 @@ The important fields are: want the repository root - `spec.encryption`: how `Secret` resources should be encrypted before commit - `spec.placement`: optional policy for where **new** resources are written (see - [Where new resources are written](#where-new-resources-are-written-specplacement)); omit it to follow - the repository's existing layout + [Where new resources are written](#where-new-resources-are-written-specplacement)); omit it and a new + resource takes the folder's one kustomization root, or the built-in canonical path - `spec.prune`: which deletion paths may remove documents from this target's folder (see [Deletion policy](#deletion-policy-specprunemode)); omit it for the safe default @@ -610,64 +610,64 @@ For each new resource the operator walks this order and stops at the first that 1. **`spec.placement.byType[]`:** an explicit template for that resource's type, if you declared one. 2. **`spec.placement.default`:** your explicit catch-all template, if you declared one. -3. **Sibling inference:** follow the layout the repository already uses for resources like this one - (described next). +3. **The folder's one kustomize root:** if the whole folder is governed by exactly one supported + `kustomization.yaml`, the new file goes beside it and is added to its `resources:` list in the same + commit. This is not a guess about your conventions: a file that kustomization cannot reach would + never be rendered, so it would be in Git and applied by nothing. Two or more supported + kustomizations is ambiguous, and the operator declines rather than picking one. 4. **Built-in canonical path:** `{namespace}/{group}/{resource}/{name}.yaml`, namespace first, the group omitted for core resources, no version segment, `_cluster/` in place of the namespace for cluster-scoped resources (an illegal namespace name, so it can never clash with a real one), and a `.sops.yaml` suffix for sensitive resources. -If you set **no** `spec.placement`, only steps 3 and 4 run, which is why pointing a target at an -existing repository "continues" that repo's conventions, and a brand-new empty repo gets the tidy -canonical layout. +**The operator does not read the rest of your repository to place a file.** Where you keep the other +ConfigMaps does not decide where a new ConfigMap goes: a layout the ladder above cannot derive is one +you declare. That is deliberate. An earlier version followed the surrounding layout, which meant +editing the repository silently changed where the operator wrote next, with no Kubernetes object +changing and nothing recording the move. If you are upgrading from a release that had it, see +[`UPGRADING.md`](UPGRADING.md). -#### Following the existing layout (sibling inference) +#### Knowing when you need a rule -This is the part that looks like magic but isn't: the operator never reverse-engineers a template. It -reads the files already in the target and makes two **observed** decisions for the new resource. *Which -directory* (the nearest cohort of resources like it: same type, then same type in any namespace) and -*one-file-or-bundle* (does that cohort keep one resource per file, or share a multi-document file?). +Every placement is counted, labeled with the GitTarget and the resource type, so "does this folder +need a `byType` line?" is a query rather than a folder inspection: -Worked example. A target at `spec.path: clusters/prod` already looks like: - -```text -clusters/prod/ - all.yaml # 9 ConfigMaps in one multi-document file (a "bundle") - team-a/secrets/db.sops.yaml # one Secret, encrypted, one file per Secret +```promql +# types landing on the built-in path, per target: each is a candidate for a byType entry +sum by (gittarget_namespace, gittarget_name, group, version, resource) ( + increase(gitopsreverser_placements_total{source="canonical"}[24h]) +) ``` -- A new **ConfigMap** `cache` arrives: its type-cohort (ConfigMaps) lives entirely in the `all.yaml` - bundle → the new document is **appended to `all.yaml`**. No new file, no canonical tree is created. -- A new **Secret** `api-token` arrives: it is sensitive, so plaintext siblings are ignored; the only - encrypted cohort is `team-a/secrets/` (one-per-file) → a new encrypted file - **`team-a/secrets/api-token.sops.yaml`**. -- A new ConfigMap in a **brand-new namespace** `billing`: the ConfigMap cohort is still the `all.yaml` - bundle, which is namespace-agnostic, so it is **appended to `all.yaml`** too, and the new namespace needs - no new segment. - -The boundaries that keep it predictable: - -- A **sensitive** resource never infers from (or is appended into) a plaintext file; it only follows - encrypted siblings, otherwise it uses the secure canonical path. -- A resource in a **namespace the target has never written before** only joins an existing cohort when - that cohort has *proven* it is namespace-agnostic by already holding more than one namespace. One - directory holding one namespace looks identical to a per-namespace layout whose second namespace has - not arrived yet, so it is not treated as shared. The resource takes the canonical path, which - carries its own namespace segment. Guessing here would file one namespace's objects under another's - folder. -- When a type lives in two layouts at once, the tie-break is deterministic (the cohort with the - most members wins, then the lexically smallest path), and it is never a coin-flip. -- Inference can only **continue** a layout that already exists. It cannot invent a greenfield one. "I - want all ConfigMaps bundled even though none exist yet" is a job for `byType` below. - -The full ladder, tie-break rules, and edge cases are in -[gittarget-new-file-placement-rules.md](spec/gittarget-new-file-placement-rules.md), which also carries -the vision the sibling-inference step serves. +`source="declared"` is a path you asked for, and `source="kustomize_root"` is a folder whose own +structure answered. Neither needs attention, and two companions matter as much: + +- `gitopsreverser_placement_refusals_total{reason}` counts resources the operator **did not write**: + a template that escapes `spec.path` (`invalid_path`), a sensitive resource whose path is already + taken (`sensitive_append`), a plaintext resource routed at an encrypted file + (`plaintext_onto_encrypted`), and two resources of mixed sensitivity landing on one new file + (`mixed_sensitivity_new_file`). Any of these means a policy to fix; the write is retried after you + fix it. +- `gitopsreverser_placement_kustomization_entries_total{outcome="failed"}` counts new files whose + `resources:` entry could not be added. The file is committed and kustomize will never build it, so + this is the one that looks fine in the folder and is not. + +See [interpreting-metrics.md](interpreting-metrics.md) for the full label sets. + +#### Namespaces in a kustomize folder + +If the `kustomization.yaml` governing the destination sets `namespace:`, and it names the resource's own +namespace, the new file omits `metadata.namespace`, because the build supplies it and every other document in +that folder omits it too. This applies to a path you declared as much as to the kustomize-root +fallback. When the transformer names a *different* namespace the namespace is written explicitly: +leaving it out would hand the namespace to kustomize and render a different object than the one being +mirrored. #### Declaring a layout (`byType` / `default`) -Set `spec.placement` when you want to **prescribe** a layout rather than follow the repo (for example a -greenfield repo, or a convention inference can't reach): +Set `spec.placement` when the layout you want is neither of the two things the operator can work out +for itself (a folder's single kustomization root, or the canonical path). For example, a bundle every +ConfigMap joins, or a per-namespace layout: ```yaml spec: @@ -680,8 +680,8 @@ spec: - **`byType`** maps an exact `[group/]version/resource` key (core resources omit the group, e.g. `v1/configmaps`; grouped resources include it, e.g. `apps/v1/deployments`) to a path template. -- **`default`** is the template for any type with no `byType` entry. Omit it to fall through to sibling - inference and then the built-in path. +- **`default`** is the template for any type with no `byType` entry. Omit it to fall through to the + kustomize-root step and then the built-in path. - Templates are small **brace-variable path templates** (see the table below), validated statically as part of the `Validated` gate: an unknown variable, a path that escapes `spec.path` (a leading `/` or `..`), or a non-`.yaml`/`.yml` suffix fails the target *before* any write. diff --git a/docs/design/attribution-branch-findings.md b/docs/design/attribution-branch-findings.md deleted file mode 100644 index 263a6e90..00000000 --- a/docs/design/attribution-branch-findings.md +++ /dev/null @@ -1,316 +0,0 @@ -# Findings on the attribution fact switchover - -What this branch's loose ends turned out to be, and which of them are measured rather than reasoned. -Five findings: one lab defect that hid every other measurement, three results about how an audit event -identifies its object that the corpus now carries, and one product race that the e2e suite was -failing to report and that is now root-caused and fixed. - -The three identity results are measurements, captured by the mutation lab against the e2e cluster and -committed to the corpus. The race was measured from the resolver's own histogram against a live -cluster. Everything here is evidence rather than inference, except where it says otherwise. - -## 1. The lab was serving the wrong audit route, so nothing was audited - -The mutation lab registered exactly one audit path, `/audit-webhook`. The e2e cluster's audit -kubeconfig posts to `/audit-webhook/default`. The route is NAMED after the `default` -ClusterProvider, because the bare path is the product's shared, annotation-routed endpoint. Go's -`ServeMux` treats a pattern without a trailing slash as an exact match, so every audit event the API -server sent got a 404. - -The store held 133 admission records and 10 watch records, and zero audit records. Every scenario -that requires an audit event timed out (including the seventeen long-standing ones), which reads -exactly like a broken cluster rather than a one-line routing mismatch. That is the expensive part: -the lab's failure mode is indistinguishable from the environment's. - -Fixed by serving the whole `/audit-webhook/` subtree. Row 15 went from a 91-second timeout to passing -in 6.6 seconds, and every scenario passes. The seventeen previously committed corpus rows re-captured -byte-identical, which is the positive result for a re-capture. - -## 2. An aggregated-API removal carries no uid - -Corpus `flunder/aggregated-api-delete/`. The kube-apiserver proxies the request to the extension -server and never decodes what came back, so the audit event's `objectRef` carries: - -```yaml -objectRef: - apiGroup: wardle.example.com - apiVersion: v1alpha1 - name: fl-del # from the URL path - namespace: - resource: flunders -verb: delete -``` - -No `uid`. No `resourceVersion`. No `responseObject` to recover either from. Meanwhile the watch -`DELETED` for the same object carries the full body, uid included. - -## 3. An aggregated deletecollection returns no response body - -Corpus `flunder/aggregated-api-deletecollection/`. One audit record, name-less, with the selector -visible only in the `requestURI`, and no response body: the collection fact therefore carries no uid -set and the join can only proceed by scope. This is precisely the case the deleted response-body expander -produced nothing at all for, and it is the case `deletecollection_scope` exists to serve. - -The scenario deletes three flunders to make the asymmetry unmistakable: three watch `DELETED` events -and three admission records against one audit record that names none of them. - -## Where each verb falls out - -```mermaid -flowchart TD - A[Audit event on an aggregated type] --> B{objectRef has a name?} - B -->|no: create| C[No fact published
rejected at the name gate] - B -->|yes: update, patch, delete| D{body present to backfill
uid and resourceVersion?} - B -->|deletecollection: name-less by nature| E[Collection fact
selector + namespace] - - D -->|no: proxied, so no body| F{uid or resourceVersion
on the fact?} - D -->|yes: ordinary bodied type| G[uid and rv recovered] - - F -->|neither: name only| H[name tier
namespace + name] - G --> I[exact tier: uid + rv
latest tier: uid] - E --> J[deletecollection_body_uid if a uid set arrived
deletecollection_scope otherwise] - - C --> K[Committer-authored] - H --> L[Attributed to the actor] - I --> L - J --> L - - style C fill:#7f1d1d,color:#fff - style K fill:#7f1d1d,color:#fff - style H fill:#14532d,color:#fff - style L fill:#14532d,color:#fff -``` - -Every verb but the create now reaches an author. Before the name tier, only the collection delete -did: an update, a patch or a single delete published a fact the index discarded on arrival, because -it could be keyed on nothing. The create still reaches no one, and cannot, because its audit event -never says which object it was about. - -## 4. The missing name changes behavior only where there is no body - -This was worth checking, because "the name is not available yet" describes both a `generateName` -create and an aggregated write, and it would be reasonable to expect them to fail the same way. They -do not, and the reason is the fork in the middle of the diagram. - -`IdentityFromAuditEvent` takes namespace, name and uid from `objectRef`, then backfills whatever is -still missing from the event's body: - -```go -preferred, fallback := bodyPriority(event, op) -backfillIdentityFromBody(&id, preferred) -backfillIdentityFromBody(&id, fallback) -``` - -For a `generateName` create on an ordinary type, `objectRef.name` is empty (the API server assigns -the name), but the policy captures at `RequestResponse`, so the `responseObject` carries the assigned -name and uid and the backfill recovers both. The fact publishes and joins normally. - -For an aggregated write there is no body to backfill from, so nothing is recovered. The same empty -field is fatal in one case and harmless in the other, and the discriminator is the body, not the -name. So: yes, the unavailable name does change behavior, but only where the body cannot cover for -it, and that is exactly the aggregated population. - -This is captured rather than left as reasoning. Corpus `configmap/generate-name-create/` is row 18, -the control for the two aggregated rows: it asserts that the `objectRef` carries no name and that the -response body does, so the recovery is evidence in the tree rather than a claim in a paragraph. Put -the three rows side by side and the discriminator is visible without reading any code. - -## 5. The CommitRequest that missed its window: root-caused - -The e2e spec `finalizes a CommitRequest created with metadata.generateName` failed in CI, in the full -local suite, and in an isolated local run. It was not a flake, and it is now fixed at its cause. - -```mermaid -sequenceDiagram - participant T as e2e spec - participant K as kube-apiserver - participant C as controller - participant W as commit window - - T->>K: create Deployment - Note over T,K: +105 ms - T->>K: create CommitRequest (generateName) - C->>C: author resolved from admission record - loop every 2s for ~8s - C->>W: attach enqueued - W-->>C: no open window - end - C->>T: Ready=True reason=NoWindowInGrace sha="" - Note over C,W: 2 seconds later - C->>W: Opening commit window -``` - -Both runs showed the same two-second miss, so it was not load. Two earlier Deployments against the -same GitTarget opened their windows within a couple of seconds; the third did not open one for about -ten, and the request's grace expired at eight. - -### What it was - -Measured, on the branch, from the resolver's own histogram after one run of the commit-request specs: - -| result | resolutions | total wait | mean | -|---|---|---|---| -| `exact_user` | 6 | 1.06s | 0.18s | -| `weak` | 3 | **20.18s** | **6.73s** | - -(These are the label values of the day. `result` has since become `tier` plus `actor_kind`, so -`exact_user` reads `tier="exact", actor_kind="user"`, and `weak` splits: what was measured here is -the uid tier, now `tier="latest"`. The measurement stands; only the names moved.) - -Three removals spent twenty seconds between them. The e2e cluster runs -`--author-attribution-grace=10s`, so each was waiting out most of a full grace, and `weak` is -precisely the tier that holds a removal matched to a WRITE fact. - -That wait is not free, and this is the part that turns a slow resolution into a stalled pipeline. -`streamLiveTargetWatchEvents` processes one shard's events on a single goroutine, and `attachAuthor` -blocks it: a removal that waits out its grace is **head-of-line blocking** for every later event of -that type. Three of them ahead of a create is ten seconds before the create is even looked at, and -the commit window cannot open until it is. The CommitRequest's own grace expires first, and it -reports `NoWindowInGrace` about a window that had not been allowed to exist yet. - -The two-second miss in the diagram is not a tuning problem between two graces. It is where the -serialized waits happened to land. - -### Why the removals were waiting at all - -A removal holds a per-object write fact as a fallback and keeps waiting for evidence about the -deletion, which is correct and deliberate. The question is why that evidence never arrived, when the -delete IS audited and the audit batch interval is one second. - -Because the delete fact was there, and the lookup could not reach it. - -Whether a delete fact carries a uid depends on what the API server answers the request with, and it -answers differently for different deletes. Both shapes are in the corpus: - -- `configmap/finalizer-delete/audit.delete.yaml`: `responseObject` is the **ConfigMap**, so the uid - is recoverable and the fact is filed under the uid tier, where a removal finds it at once. -- `configmap/owner-ref-cascade/audit.delete.cm-parent.yaml`: `responseObject` is a **`Status`**. - There is no uid in it and none in the `objectRef`, so the fact's only key is its name. A - `kubectl delete deployment` is this shape. - -Before the name tier existed, such a fact was published and dropped as unjoinable, so the removal -had no delete evidence at all and waited out the grace. That is the original bug, and it is as old as -the audit event's shape rather than as old as any commit here. - -Adding the name tier stored the fact but did not make it reachable, which is a defect in that change -rather than in the design. `Lookup` returns as soon as the removal ladder yields anything, and the -uid tier yields the object's last WRITE fact, so the name-keyed DELETE fact below was never -consulted. The caller then held the write fact and waited the full grace for evidence that was -already in the index. - -The fix is an ordering rule, and it is the one the removal path already states elsewhere: a fact -about the DELETION outranks a fact about a write, whichever key each is filed under. `lookupRemoval` -now returns the object's own delete fact when the uid tier has one, then consults the name tier for a -delete fact, and only then falls back to the write fact it was holding. A name-keyed WRITE still does -not jump the queue; only removal verbs do. - -### The same measurement, after - -| result | before: n / total / mean | after: n / total / mean | -|---|---|---| -| `exact_user` | 6 / 1.06s / 0.18s | 6 / 0.74s / 0.12s | -| `name` | none reached | 2 / 0.60s / 0.30s | -| `weak` | 3 / 20.18s / **6.73s** | 2 / 0.28s / **0.14s** | -| **total wait** | **21.24s** | **1.63s** | - -The spec passes, and the histogram says it passes for the diagnosed reason rather than by timing luck. -Two resolutions now land on the `name` tier: those are the delete facts that were being stored and -never read. The removals that still resolve `weak` no longer wait for them, because the lookup -reaches the delete evidence before it settles for a write fact, and the shard is no longer blocked -behind a grace that had nothing to wait for. - -### The assertion that hid it - -The spec asserted `Ready=True` and then a non-empty `status.sha`. But `Ready=True` is also the benign -rejection state: `rejectCommitRequest` sets it deliberately for `NoWindowInGrace`, `WindowMismatch` -and `AlreadyPresent`, so that kstatus reads Current rather than Failed. The Ready assertion therefore -passed on a request that committed nothing, and the spec spent the remaining two minutes re-reading -an empty string before reporting: - -```text -Expected - : -not to be empty -``` - -The reason was sitting in the condition the spec never read. `expectCommitRequestCommitted` now -requires the Ready reason to be `Committed` and gives up as soon as any other terminal outcome -appears, because a terminal outcome is final and re-reading it cannot change the answer. The same -failure now reports in ten seconds, naming `NoWindowInGrace` and its message. - -## The decision: a name tier, built - -A name tier is now built rather than proposed. `AuthorFact.Name` is back on the wire, and the -index files a fact under `(namespace, name)` when it carries neither a uid nor a resourceVersion. -`Lookup` consults that tier last, below the rv-only hatch, and reports `AttributionName`. - -The ordering argument is that a name is the weakest per-object evidence available: it is reused after -a delete and recreate, where a uid never is and an rv identifies one specific write. Ranking it last -costs the stronger tiers nothing, because no fact carrying a uid or an rv is ever filed there, and no -query reaches it until every stronger tier has missed. - -What it fixes is exactly the two rows it can reach: an aggregated update or patch, and an aggregated -single delete. Both used to publish a fact the index then discarded, so they shipped -committer-authored whoever ran them. - -Restoring the field is the part worth being explicit about. It was removed during this work on the -observation that no tier read it. That was true of the code and false of the domain: for a whole -population of writes the name is the only identity the audit event carries, so a fact without it -could not be joined at all. "No code reads it" and "nothing could ever read it" are different claims, -and only the second one justifies deleting a field. - -### What the name tier still does not reach - -**The aggregated create.** Its `objectRef` carries no name, and there is no response body to recover -one from, so `AuthorFactFromEvent` rejects it at the name gate and nothing is published for any tier -to join. This is not a gap in the tier; it is a request the API server logged without ever saying -which object it was about. - -Two options remain open for that population, and they are not exclusive: - -**B. Accept that aggregated creates are unattributable.** Document that per-object attribution does -not apply to them, and let them ship committer-authored. Honest, but it makes the guarantee -type-dependent in a way a user cannot predict from the API surface. - -**C. Shorten the wait for facts that provably are not coming.** An aggregated create can be -recognized as unattributable at publish time rather than after a full grace. This attributes nothing; -it stops paying for evidence that cannot arrive. - -### For the window race - -The measurement answered the question this section used to pose. Neither option was right: the window -was not slow to open and the CommitRequest's grace was not too short. The write's event had not been -processed yet, because three removals ahead of it were sitting out most of a ten-second grace each -(20.2s between them, measured) for evidence the index already held. - -That is fixed at its cause, in the lookup ordering. What remains open is the structural half. - -**The head-of-line block is still there.** Any removal that must wait out its grace (a type the -audit policy excludes, a delete whose fact never arrives) still stalls every later event on its shard -for up to ten seconds. This fix removes the largest population that was hitting it, and -does not change the fact that a blocking resolve on a serial goroutine can do this at all. - -Two directions, and they are the same two the wait-versus-poll record already frames: - -1. **Bound the removal's extra wait separately from the grace.** Once a fallback is in hand, the - fact stream for that scope is demonstrably live; what is outstanding is only whether a delete fact - also lands, which is an audit-batch interval rather than a full grace. Small change, needs a - number chosen with evidence. -2. **Stop blocking the shard.** Resolve attribution off the event loop and reassemble in order. This - is the real answer and the larger one; it also fixes every other cause of a slow resolve. - -Worth noting for whoever picks this up: the e2e default of `--author-attribution-grace=10s` makes the -blocking three times worse than the product default of 3s. That is a test-environment choice -amplifying a product behavior, not a product setting anyone runs. - -### A note on the pre-branch probe - -A run against the merge-base (`7ece7310`) was attempted to establish whether the race was new on this -branch, and it did not produce an answer: its bring-up failed before any spec ran, with the manager -rejecting the API server's audit client certificate (`tls: bad certificate`) so the audit pipeline -never warmed up. That is a worktree/cluster-provisioning problem rather than a product difference: -both trees post to the same named audit route and both serve it. - -The comparison turned out not to be needed. The cause is measured directly on the branch, and the -delete-fact shape that triggers it is a property of the Kubernetes audit event rather than of any -commit here. diff --git a/docs/design/attribution-deletion-intent-actor.md b/docs/design/attribution-deletion-intent-actor.md deleted file mode 100644 index 8ce3ed19..00000000 --- a/docs/design/attribution-deletion-intent-actor.md +++ /dev/null @@ -1,276 +0,0 @@ -# A finalized deletion is attributed to the cleanup controller, not to the deleter - -> **design** — built. Index: [`../INDEX.md`](../INDEX.md) -> -> Status: BUILT. The defect was reproduced from a mutation-lab capture -> (`configmap/deletion-intent-actor`) and pinned by a corpus-driven unit test -> ([fact_index_corpus_test.go](../../internal/queue/fact_index_corpus_test.go)); that test now pins -> the fix. Shipped: the sticky uid-keyed removal slot, its consultation ahead of the exact tier for a -> removal, the count-based horizon, and the `delete_sticky` tier on -> `attribution_resolutions_total{tier}`. The measurement and the reasoning below are kept as -> written — they are why the shape is what it is. - -A human deletes an object that carries a finalizer. A controller clears the finalizer. The commit -that removes the file from Git is authored by **the controller**, and the human who asked for the -deletion appears nowhere. - -```text -onboard: [DELETE] reposelections/attribution-probe (by system:serviceaccount:gitops-api:tenant-operator) -``` - -The `[CREATE]` on the same object, seconds earlier, names the human correctly. This document says -exactly why, with the measurement, and designs the fix. - -## What the deletion-as-intent rule promises - -The spec's §2.2 states the intent: - -> *The first observation of `deletionTimestamp` (or a `DELETED` event) removes the resource from Git -> and attributes the removal to the actor who **requested** the deletion.* - -The first half holds. [`operationForLiveTargetWatchEvent`](../../internal/watch/target_watch.go) -renders any object carrying a `deletionTimestamp` as a `DELETE`, so the file goes at the transition -rather than at the eventual `DELETED`. The second half is the one that does not hold, and the reason -is not in the watch side at all — it is in what the two audit events look like. - -## The measurement - -The lab drives one ConfigMap with two real identities: the deletion is requested by -`alice@example.com` (impersonated), the finalizer is cleared by a real ServiceAccount token, -`system:serviceaccount::finalizer-controller`. A **hold** between the two phases is the knob: -the API server batches audit deliveries (`--audit-webhook-batch-max-wait`, 1s here), so the hold -decides whether both events reach the operator in one batch or two. - -Three facts come out of the capture, and only the third is surprising: - -1. **The removal verb is a `patch`.** There is exactly one audit `delete` — phase one, the human's — - and nothing audits the disappearance itself. This was already known - ([`finalizer-delete`](../../test/mutationlab/corpus/configmap/finalizer-delete), catalog row 8). -2. **The two phases have different actors**, which is the whole premise of the report. -3. **Both audit events carry the same `resourceVersion`.** The human's `delete` returns the object - with `deletionTimestamp` set, at RV *n*. The controller's finalizer `patch` returns the object as - it stood — also at RV *n*. The capture's relational tokens make this visible directly: both - response bodies read `resourceVersion: `, and so does the deletion-pending watch event. - -Fact 3 is the defect. It is not a ranking problem, and no tier ordering can fix it. - -## Why the join then names the controller - -The publish side files a fact under the strongest key it has -([`file`](../../internal/queue/fact_index.go)). Both of these facts carry a uid and a -resourceVersion, so both are filed under **`exact(uid, rv)`** and **`latest(uid)`**. Both index -structures are last-writer-wins ([`putExact`, `putLatest`](../../internal/queue/fact_index_store.go)): - -```go -s.exact[key] = entry // same (uid, rv) → the controller's fact REPLACES the human's -s.latest[uid] = entry // same uid → likewise -``` - -So once the controller's fact is applied, the human's fact is not outranked — **it is gone**. The -watch event that renders as the removal is the deletion-pending `MODIFIED`, whose resourceVersion is -that same colliding value, so it asks the exact tier for exactly the key that was overwritten, gets -the strongest possible match, and never consults a weaker tier or waits for anything. - -The corpus-driven test states both sides of the race: - -| Facts applied when the removal resolves | Named actor | Tier | -|---|---|---| -| the human's `delete` only (a slow finalizer, separate audit batches) | **the human** | `exact` | -| both, in the order the API server produced them (a fast finalizer, one batch) | **the controller** | `exact` | - -Nothing about the join can tell those two apart. Whether a deletion is attributed correctly depends -on how quickly a controller cleans up, which is a property of the cluster's workloads. - -**The more finalizers an object carries, the likelier the wrong answer**, exactly as the report -says: each finalizer controller adds another write that can land inside the batch window. - -### The second trigger: a hung finalizer plus a restart - -A finalizer that hangs — days, when nobody notices — is the *safe* case for as long as the operator -stays up. The file left Git at the transition, attributed to the human, and the eventual finalizer -clear and terminal `DELETED` both fold to no-ops against the already-absent path. Slower is better. - -It stops being safe the moment the operator is not running at the transition. A restart, a rollout, -a `GitTarget` created later, a `410` rebuild — after any of them the watch collapses to CURRENT -state, so there is no transition event to observe. The first observation is of an object already -`Terminating`, the file is still in Git, that observation renders as a `DELETE`, and a real commit -lands. The human's fact aged out hours ago, so that commit is authored `unresolved`, or picks up -whichever fact is still inside the TTL — during cleanup, plausibly the finalizer controller's. - -A hang does not cause this. It widens the window in which an ordinary restart does, from seconds to -days, which turns an unlucky coincidence into an expected one. - -### What the current code does get right - -Three things already move in the right direction, and the fix should not disturb them: - -- **A removal never settles for a write fact.** [`lookupRemoval`](../../internal/queue/fact_index.go) - holds a write match as a fallback and keeps waiting for evidence about the deletion; only a fact - whose own verb is `delete`/`deletecollection` ends the wait. That is already "prefer the actor of - the transition", expressed at the tier level — it just never gets the chance here, because the - exact tier answers first. -- **An object's own delete fact wins from more places** than it used to: by uid, and by - `(namespace, name)` when the delete audit event carried no uid at all. -- **The head-of-line stall that inverted event ordering is fixed at its cause** - ([attribution-branch-findings.md](attribution-branch-findings.md)), so the reported log's - "controller's window commits first" no longer has that mechanism behind it. - -## The fix: a sticky removal pointer - -**A fact about a DELETION must not be replaceable by a fact about a WRITE.** - -That single rule closes the defect, and it is the rule the tier ladder already believes in — the -removal path is built entirely around "a fact about the deletion outranks a fact about a write". It -is simply not enforceable today, because the write can overwrite the deletion's *storage* before any -ranking happens. - -Concretely: file a fact whose verb is a removal into a **removal slot** keyed by uid, and let a -later non-removal fact fill the ordinary tiers without touching that slot. `lookupRemoval` consults -the removal slot first. How long the slot lives is its own decision, and not the join TTL — see -[the horizon](#how-long-it-should-live-and-why-that-is-not-the-ttl). - -Sketch, in [`file`](../../internal/queue/fact_index.go): - -```go -case isRemovalVerb(fact.Verb) && fact.UID != "": - facts.putRemoval(fact.UID, entry) // sticky: only another REMOVAL fact may replace it - // …and the ordinary exact/latest filing as today -``` - -and in [`lookupRemoval`](../../internal/queue/fact_index.go), before the exact tier is consulted for -a non-exact-capable query. - -### Why "sticky" and not "ranked" - -A ranking change alone (consult a removal tier before the exact tier) does not help while both facts -live under one key: there would be nothing left to rank. The stickiness is the load-bearing half. -The ordering change is what makes it reachable. - -### What it costs - -- **One more entry per deleted object**, held until the caps reclaim it rather than until the TTL - does. The index is capped per type and in total, so the cost is bounded by construction and shows - up on `attribution_fact_index_evictions_total` if it ever binds. -- **A deliberate asymmetry**: the removal slot is the only structure in the index that a later fact - may not overwrite. That deserves the comment it will get — the reason is that "who deleted this" - is a question a later write cannot answer, which is not true of any other tier. - -### How long it should live, and why that is not the TTL - -The removal pointer should outlive the join TTL, and be bounded by **count rather than time**. - -Every other structure in the index must expire for correctness. `exact` and `latest` can be -superseded by a later write. The name tier must expire because a **name** is reused after a -delete-and-recreate, which is exactly why it ranks last. A uid-keyed removal statement has neither -failure mode: - -> **A uid is unique across space and time.** "uid X was deleted, and this actor asked for it" can -> never be superseded, because that object can never be written again, deleted again, or recreated -> under the same uid. - -So holding it for a day is not less accurate than holding it for ten minutes, only more expensive, -and the question stops being "how long is safe" and becomes "how much memory will we spend". That -argues for a per-type LRU of recent removals, aged out under pressure rather than by a clock: a -cluster that deletes rarely keeps its removals for a very long time at no cost, and a busy one keeps -the most recent, which are the ones a replay is most likely to need. It fits the index's existing -shape — bounded per type and in total, evictions counted — and it needs no new number for an -operator to tune. - -**Strictly uid-keyed.** The same stickiness on the name tier would be a defect rather than a fix: a -name reused after a recreate would inherit the previous object's deleter, and a longer horizon makes -that more likely, not less. The TTL is what bounds that risk today and must keep bounding it. - -**The longer horizon is a horizon within one process, and the TTL still bounds what a restart can -recover.** The pointer is not persisted: the index is warmed from the fact streams, and both halves -of that are one TTL wide — the transport trims to the retention horizon, and the follower replays -exactly `i.ttl` ([`Run`](../../internal/queue/fact_index.go), -[`FollowFacts`](../../internal/queue/fact_stream.go)). So a pointer for a delete older than the TTL -survives a `410` rebuild, a re-list, and any amount of watch churn, but it does not survive the -operator process going away. The second trigger is therefore *narrowed* rather than closed: it is -closed for a restart that happens after the delete fact was seen and inside the retention window, -and untouched when the operator was not running to see the delete at all. Closing that remainder -needs the pointer to outlive the process, which is a different change — persistence — and is not -proposed here. - -### The payoff that is not correctness - -A `Terminating` object seen on replay resolves `absent` today, which means it waits out the **whole** -grace window for a fact written days ago that can never arrive — on the shard's single goroutine, -with every later event queued behind it. That is the head-of-line cost -[attribution-branch-findings.md](attribution-branch-findings.md) measured. A removal pointer that -outlives the TTL turns that full-grace wait into an immediate hit, so a cluster carrying objects -stuck `Terminating` stops paying a grace per replayed event — for as long as the process that saw -the delete is still running, which is the horizon the section above states exactly. - -### Why not simply refuse to overwrite the exact entry - -The narrower move is to make the exact tier **write-once**: `(uid, rv)` asserts "this actor produced -this exact version", and the finalizer patch did not produce that version — it returned it. Keeping -the first fact filed under a key would name the deleter here without touching the read ladder at all, -which is a real attraction. - -It is not enough, for two reasons. - -**It makes correctness depend on arrival order.** First-writer-wins is still a race, only decided -earlier: it gives the right answer exactly when the deleter's fact is filed first. Within one audit -batch it is, but a fact's arrival order is not guaranteed in general — an HA control plane audits the -`delete` on one API server and the finalizer `patch` on another, and those two audit batches are -independent. "A write may never displace a deletion" needs no ordering assumption, because the -patch's fact can never enter the slot no matter when it lands. - -**It does nothing for the replay case.** The exact tier is TTL-bounded and has to stay that way — it -holds one entry per write, not one per delete — so the [second trigger](#the-second-trigger-a-hung-finalizer-plus-a-restart) -and the [head-of-line payoff](#the-payoff-that-is-not-correctness) both need a structure whose -horizon is a count. That structure is the removal slot, and once it exists, the ordering change is -the cheap half. - -The narrower move remains worth making on its own merits — a mutating request that returns a -resourceVersion it did not produce should not claim that version's exact key, and a no-op patch does -exactly that today. It is a separate change with a separate argument, and it is not this one. - -### The alternative, and why it is second - -**File every delete fact under the name tier as well**, so it survives the uid tiers being -overwritten. It is smaller — one extra `putName` — and needs no new structure or lookup order. - -It is second because it is indirect: it fixes this defect by making a *weaker* tier hold the answer, -so the removal resolves at `tier="name"` rather than at a tier that says what it is. A reader of the -metric would see a name-tier match and reasonably conclude the audit event carried no uid, which -here would be false. It also leaves the exact tier still answering with the controller for the -window before the name tier is consulted. - -## How we will know it worked - -The metrics that shipped in the attribution surface make this observable without a repro: - -- The evidence mix moves. A finalized deletion that resolves today at `tier="exact"` should resolve - at `tier="delete_sticky"` instead: - `sum by (tier) (rate(gitopsreverser_attribution_resolutions_total[5m]))`. -- `commits_total{author_kind}` shifts from `serviceaccount` toward `user` for the affected types — - the bottom line the report is actually about. -- The removal wait collapses for `Terminating` objects seen on replay, which today spend a full - grace each: - `histogram_quantile(0.95, sum by (le) (rate(gitopsreverser_attribution_resolution_wait_seconds_bucket{event_kind="removal"}[5m])))`. -- The corpus test inverts: the "both facts applied" case names the human, and the assertion that - pinned the defect is now the assertion that pins the fix. - -## Reproducing it - -```bash -task lab-corpus-update # captures configmap/deletion-intent-actor (three holds, two actors) -go test ./internal/queue/ -run TestFactIndex_FinalizerRemovalNamesTheDeleter -v -``` - -The hold is tunable — `LAB_FINALIZER_HOLD=0s` makes the controller win every time, a hold longer -than the audit batch window makes the human win — which is the race itself, made deterministic. - -## References - -- [attribution-publish-and-join.md](attribution-publish-and-join.md) — what each half does, and the - tier ladder this fix extends. -- [attribution-removal-wait-options.md](attribution-removal-wait-options.md) — why a removal waits - for evidence about the deletion rather than accepting the last write. -- [mutation-capture-lab-design.md](../spec/mutation-capture-lab-design.md) — the lab, and why a - captured shape beats an asserted one. -- [interpreting-metrics.md](../interpreting-metrics.md) — the tier and actor-kind labels the fix is - measured on. diff --git a/docs/design/attribution-fact-identity.md b/docs/design/attribution-fact-identity.md deleted file mode 100644 index f782132c..00000000 --- a/docs/design/attribution-fact-identity.md +++ /dev/null @@ -1,575 +0,0 @@ -# Attribution when several ClusterProviders name one cluster - -> **design**: implemented. Index: [`../INDEX.md`](../INDEX.md) -> -> Status: the field, the two renames, the dropped ingestion gate, and the warning are built and -> covered by unit and e2e tests. Two deviations from the text below are marked in place: the value -> is validated as a DNS-1123 **subdomain** rather than a label, and the `lastAuditEventTime` status -> field is NOT built (the warning is). -> -> Prompted by a bug report from the gitops-api team (`docs/bug-report.md`): every object mirrored -> through a dedicated in-cluster `ClusterProvider` was committed as -> `unknown (attribution unresolved)`. The report's diagnosis is correct and is reproduced in code -> below. Its recommended fix (make the failure loud) is adopted here as one half of the answer. - -## The decision - -A `ClusterProvider` declares the audit route its cluster's events arrive on, and that declared -value rather than the object's name is what partitions the attribution facts. Several providers may -name one physical cluster by carrying the same route. The field defaults to the object's own name, -so every install that works today keeps working unchanged. - -```yaml -apiVersion: configbutler.ai/v1alpha3 -kind: ClusterProvider -metadata: - name: tenant-acme-delegating # the name humans read and GitTargets reference -spec: - attribution: - auditRoute: prod-eu-1 # the route this cluster's audit events arrive on -``` - -Ingestion does not change, and it loses its only Kubernetes read: an audit event arrives, and its -fact is stored under the route it arrived on, with no lookup, no existence check, and no fan out. -The key keeps its structure, and the rest of the system is renamed to speak the same word. - -## What happens today - -The write side files a fact under the name in the audit route. The read side looks one up under the -name on the `GitTarget`. Both are the provider name, exactly as designed, and they diverge whenever -those two names are not the same object. - -| Side | Code | Key input | -|---|---|---| -| write | [`audit_handler.go`](../../internal/webhook/audit_handler.go) `resolveRoute` then `RecordFact` | the `` in `/audit-webhook/` | -| read | [`target_watch.go`](../../internal/watch/target_watch.go) `attachAuthor` then `LookupAuthorResolution` | `GitTarget.SourceCluster()`, the `spec.clusterProviderRef` name | - -A kube-apiserver takes one `--audit-webhook-config-file`, and that file -[uses the kubeconfig format to specify *the remote address of the service*](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend), -singular. So one physical cluster feeds exactly one provider name through its -[webhook backend](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend), and -that is the constraint everything below follows from. Every other `ClusterProvider` naming that same -cluster reads a partition nothing writes, and each of its commits is authored -`unknown (attribution unresolved)` after waiting out the full grace window. Nothing fails, no -condition changes, and the mirrored content is correct, so the loss is visible only in the commit -author. - -Two providers, one remote cluster, one audit route. Both targets see the same object on their own -watch, and both look for the same write, under two different keys: - -```mermaid -flowchart LR - subgraph SOURCE["One source cluster · one kube-apiserver"] - API["Kubernetes API
ConfigMap web · uid U · rv 101"] - AUD["Webhook backend
one audit webhook config
names one server URL"] - end - - subgraph CONTROL["Control plane · operator cluster"] - CPA["ClusterProvider prod-a
allowSourceNamespaceOverride: true"] - CPB["ClusterProvider prod-b
allowSourceNamespaceOverride: false"] - GTA["GitTarget alpha"] - GTB["GitTarget beta"] - end - - subgraph REDIS["Attribution index"] - FA[("route:prod-a:configmaps:object:U:101
author alice")] - FB[("route:prod-b:configmaps:object:U:101
nothing ever writes this key")] - end - - GTA -->|clusterProviderRef| CPA - GTB -->|clusterProviderRef| CPB - CPA -. "kubeConfig" .-> API - CPB -. "the same kubeConfig, the same cluster" .-> API - AUD ==>|"POST /audit-webhook/prod-a"| FA - API -->|"watch event · uid U · rv 101"| GTA - API -->|"watch event · uid U · rv 101"| GTB - GTA -->|"read under prod-a"| FA - GTB -->|"read under prod-b"| FB - FA -.->|hit| C1["commit author
alice"] - FB -.->|"absent after the full grace"| C2["commit author
unknown (attribution unresolved)"] - - classDef ok fill:#e6f7e6,stroke:#33aa33,color:#000; - classDef miss fill:#ffe6e6,stroke:#cc3333,color:#000; - class FA,C1 ok; - class FB,C2 miss; -``` - -The reported case is this picture with `kubeConfig` omitted from both providers and the route named -`default`: the local apiserver posts to `/audit-webhook/default`, `prod-b` becomes -`srcns-delegating`, and the red path is the one the gitops-api team saw. Nothing about the diagram -depends on the cluster being remote. It depends only on two provider names sharing one audit route. - -The reported run is the clean form of this: five objects mirrored through `srcns-delegating` were -`absent`, the objects mirrored through `default` in the same run resolved, and the fact counters -showed `written=66 / matched=12` with zero expiries. Facts were being written the whole time, under -a key nobody read. - -## Why several providers name one cluster - -`ClusterProvider` carries two jobs on one object. It is the cluster's **identity** (the audit route, -the fact partition, the GVK to GVR registry key), and it is a **policy** holder: -`spec.allowedNamespaces` decides which namespaces may reference it, and -`spec.allowSourceNamespaceOverride` delegates source-namespace selection to admitted `GitTarget`s. - -The identity job wants one object per cluster. The policy job wants one object per delegation -stance, because both fields are provider-wide. A platform administrator who grants the override to -one tenant and withholds it from another has no choice but to create two providers on the same -cluster. That is what [`watchrule-source-namespace/`](watchrule-source-namespace/README.md) shipped, -and its own e2e creates `srcns-delegating` and `srcns-non-delegating` side by side. The -configuration in the bug report is one the product encourages. - -Splitting `metadata.name` from the audit route resolves that tension without splitting the object: -the name serves the policy job (one object per stance, named for the stance), and the route serves -the identity job (one value per cluster, shared by every object that carries it). - -## The constraint on any fix - -Ingestion stays low level and trackable. An audit event arrives, it is reduced to a fact, and the -fact is stored under the route it arrived on. Ingestion does not read a `ClusterProvider`, -does not resolve locality, does not write a fact more than once, and does not refuse an event -because something is missing. A provider may be created later, deleted and recreated, or renamed -around a running audit stream, and none of that is ingestion's problem: the facts carry a short TTL -and a stored fact nobody reads costs one expiring Redis key. - -Whether a fact may be *used* is not a keyspace question. A `GitTarget` mirrors an object only when -`allowedNamespaces`, `allowedSourceNamespaces`, and the `WatchRule` gate all admit it. By the time a -watch event reaches the resolver, the decision to mirror that object has been taken. Attribution -answers a narrower question: who last wrote the object this event carries. - -## The proposal - -### The field - -A new optional field, on the provider that owns the cluster identity: - -```go -// AuditRoute is the route this cluster's audit events arrive on. The sender is the API server's -// webhook backend (https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend), -// and this is the segment its configured URL ends in: /audit-webhook/. When several -// logical clusters share one backend it is instead the value of the audit-event annotation named -// by --author-attribution-audit-route-annotation-key. -// -// It partitions the attribution facts, so two ClusterProviders carrying the same route read one -// cluster's facts, and two carrying different routes can never cross-credit an author. -// -// Empty means metadata.name, which is the single-provider-per-cluster convention and what every -// existing install already does. Set it when several ClusterProviders name one cluster (only one -// of them can be what the API server posts under), or when the audit events are labelled for -// something other than this object, such as a kcp logical cluster. -// +optional -AuditRoute string `json:"auditRoute,omitempty"` -``` - -Resolved through a method, the same shape as -[`GitTarget.SourceCluster()`](../../api/v1alpha3/gittarget_types.go), so no caller ever sees the -empty case: - -```go -func (p *ClusterProvider) AuditRoute() string { - if p.Spec.Attribution == nil || p.Spec.Attribution.AuditRoute == "" { - return p.Name - } - return p.Spec.Attribution.AuditRoute -} -``` - -It is **mutable**, unlike `spec.kubeConfig`. Repointing it changes which partition is read, not -which cluster is mirrored, so it is not the silent-retarget hazard immutability exists to prevent. -Correcting a typo should not require deleting the object. - -### The key keeps its shape and renames its infix - -```text -:author:v1:audit:route:::object:: -:author:v1:audit:route:::object::last -:author:v1:audit:route:::rv: -``` - -The infix stays on all three families, including the rv-only hatch where it is -[correctness work](../finished/multi-cluster-author-attribution.md) rather than partitioning, since -resourceVersions are small cluster-scoped integers that collide freely. Two things change: the -value, which becomes a declared route rather than an object name, and the label, which becomes -`route:` where it was `cluster:`. - -The label is spelled `route:` rather than `auditRoute:` because the key already says `audit` one -segment earlier (`author:v1:audit:`), so the longer form would stutter. It reads as one path: -*author, from audit, on route `prod-eu-1`*. - -The same two providers, now agreeing on one identifier: - -```mermaid -flowchart LR - subgraph SOURCE["One source cluster · one kube-apiserver"] - API["Kubernetes API
ConfigMap web · uid U · rv 101"] - AUD["Webhook backend
names one server URL"] - end - - subgraph CONTROL["Control plane · operator cluster"] - CPA["ClusterProvider prod-a
auditRoute: prod-eu-1"] - CPB["ClusterProvider prod-b
auditRoute: prod-eu-1"] - GTA["GitTarget alpha"] - GTB["GitTarget beta"] - end - - subgraph REDIS["Attribution index"] - F[("route:prod-eu-1:configmaps:object:U:101
author alice")] - end - - GTA -->|clusterProviderRef| CPA - GTB -->|clusterProviderRef| CPB - CPA -. "kubeConfig" .-> API - CPB -. "the same kubeConfig, the same cluster" .-> API - AUD ==>|"POST /audit-webhook/prod-eu-1"| F - API -->|"watch event · uid U · rv 101"| GTA - API -->|"watch event · uid U · rv 101"| GTB - GTA -->|"read under prod-eu-1"| F - GTB -->|"read under prod-eu-1"| F - F -.->|hit| C1["commit author
alice"] - F -.->|hit| C2["commit author
alice"] - - classDef ok fill:#e6f7e6,stroke:#33aa33,color:#000; - class F,C1,C2 ok; -``` - -### Ingestion gets simpler, not harder - -Today a named route 404s when the `ClusterProvider` does not exist, and the annotation-routed bare -endpoint rejects an event whose annotation names no provider. Both checks go away, and with them the -`AuditProviderResolver` interface, its `clusterProviderExistence` implementation in `cmd/main.go`, -and the startup rule that annotation routing requires a resolver. Ingestion becomes: parse the route -or the annotation, record the fact under it. It makes no Kubernetes API call at all. - -That is correct under this design because the route is no longer a claim about an object that must -exist. It is the partition name. An audit batch in flight while a provider is being created or -recreated is currently dropped (and the apiserver does not retry a 404); now it is stored, and the -provider joins it when it arrives. - -The authentication boundary does not move. The audit server still requires a client certificate -signed by the audit CA (`RequireAndVerifyClientCert`), which is what stops an unauthenticated -producer. The existence check was never an authorization check: -[`multi-source-audit-ingress-hardening.md`](multi-source-audit-ingress-hardening.md) already records -that "a holder of a shared, CA-signed client certificate can therefore submit facts for any existing -provider route". Removing it widens that from any existing route to any string, which changes the -blast radius of an already-accepted trust assumption rather than opening a new one. - -> **This touches an open design.** -> [`multi-source-audit-ingress-hardening.md`](multi-source-audit-ingress-hardening.md) lists "an -> event with a missing, unknown, or untrusted annotation records no fact" as an acceptance criterion -> for annotation routing. That criterion needs restating in terms of routes rather than providers. -> Its option A (provider-bound mTLS) becomes route-bound mTLS and gets *stronger* here, because the -> certificate would bind to the route, which is now the real identity. - -### What this does for kcp and shared streams - -Annotation routing becomes coherent rather than coincidental. With -`--author-attribution-audit-route-annotation-key=kcp.io/cluster`, the annotation value on each -event is the logical cluster it came from, and that value is now matched against `auditRoute` -rather than against `metadata.name`. A workspace provider can be called `tenant-acme` and carry -`auditRoute: `. - -Today those installs have to name the Kubernetes object after the hash for attribution to resolve, -which answers open question 2 in the bug report: the remote path works, but only if you accept -hash-named objects. This removes that constraint. - -### Cloned and restored clusters stay separate - -Keeping the partition on every key family is what makes this topology supportable, and it is the -main reason this design was chosen over dropping the partition in favour of the object UID. - -An etcd snapshot restored into a second cluster reproduces every `metadata.uid` exactly, so the -original and the copy contain the same objects with the same identities. Mirroring both from one -operator is then only safe if something separates their facts, and under this design that something -is a value a human sets: - -```yaml -metadata: {name: prod} # the original -spec: {attribution: {auditRoute: prod-eu-1}} ---- -metadata: {name: prod-restored} # an etcd snapshot of it, stood up for a rehearsal -spec: {attribution: {auditRoute: prod-eu-1-restored}} -``` - -Object `U` exists in both with the same UID. Its facts land under -`route:prod-eu-1:…:object:U:…` and `route:prod-eu-1-restored:…:object:U:…`, which never -join. A design keying facts by object identity alone could not make that distinction, because -every identity readable from inside a cloned cluster was cloned along with it. - -The same applies to the narrower case of one provider name being repointed at a different physical -cluster. `spec.kubeConfig` is immutable precisely so that repointing means delete and recreate, and -giving the recreated provider a fresh `auditRoute` also gives it a fresh partition, without the -purge mechanism that was considered and rejected earlier. - -That rejection also settles the fate of the dormant `PurgeClusterFacts`, which had been left exported -and tested in case purge-on-adopt was ever wanted. It is deleted as part of this change, along with -the decision record that kept it alive. A shared route makes it unsafe rather than merely unused: -purging on one provider's deletion would drop the facts of every other provider reading that route, -including the operator's own cluster, which was never torn down. The reasoning that still binds (no -finalizer, because `helm uninstall` would strand the object in `Terminating`) now lives on -`LegacyClusterProviderFinalizer` in -[`clusterprovider_controller.go`](../../internal/controller/clusterprovider_controller.go). - -## The naming of the field - -`auditRoute` is chosen, and the rest of the system is renamed to speak the same word. That second -half is the point: the concept is currently spelled three different ways, and adding a fourth name -for it would be the actual mistake. - -| Where | Today | Becomes | -|---|---|---| -| the `ClusterProvider` field | does not exist | `spec.attribution.auditRoute` | -| the fact key infix | `cluster:` | `route:` | -| the shared-stream flag | `--author-attribution-cluster-annotation-key` | `--author-attribution-audit-route-annotation-key` | - -The word earns it on two grounds. It is already the internal name: -[`audit_handler.go`](../../internal/webhook/audit_handler.go) defines an `auditRoute` type whose job -is mapping one request's events to a source cluster, and it deliberately spans both delivery forms -(a named path sets `provider`, the shared endpoint sets `annotationKey`). And it says *audit*, which -is the half that distinguishes this from every other cluster identity in the system: a -`ClusterProvider` also has a name, a kubeconfig, and a watch context, and none of those are this. - -The objection to it is that a reader takes "route" to mean the URL, while the value also arrives as -an annotation with no URL involved -([`audit_handler.go`](../../internal/webhook/audit_handler.go) does -`name := event.Annotations[route.annotationKey]`, which is the kcp form). Renaming the flag to -`--author-attribution-audit-route-annotation-key` answers that directly: the annotation is stated, -in the flag's own name, to carry an audit route. The word then means "the label these events arrive -under", in both forms, everywhere. - -| Rejected | Why | -|---|---| -| `clusterID` | matches the current key infix and `internal/watch`'s 99 uses of `clusterID`, but says nothing about audit, and a `ClusterProvider` has several identities that are not this one. Renaming the key and the flag removes its one advantage | -| `auditWebhookBackendRoute` | uses the official upstream term, and is the closest miss. A [webhook backend](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend) is one per API server, but one backend carries many routes under annotation routing (a kcp shard posts every logical cluster through a single backend), so the name implies a 1:1 exactly where the interesting case is N:1. It also names the sender's configuration rather than our partition | -| `auditPathSegmentName`, `auditSegmentName` | describe one of the two delivery forms, and the wrong one for kcp. A `...Name` suffix also works against the point, which is separating a technical identifier from the friendly `metadata.name` | -| `auditClusterID` | `auditRoute` with a redundant noun, once the flag and the key both say route | -| `clusterIngestionID` | reads well standing alone, but "ingestion" is ambiguous: [watch-first ingestion](../finished/watch-first-ingestion-architecture.md) is the object-state path, and this is the audit path | -| `auditIngressID` | "audit ingress" is established repo vocabulary, but it names the operator's own receiving server, so it reads as the wrong end of the connection | -| `auditID` | taken: `event.AuditID` is the per-event UUID on every Kubernetes audit event | -| `auditStreamID` | "stream" means watch streams (`StreamsRunning`, `status.streams.summary`) | -| `auditSourceID` | "source" means the watched cluster (`SourceCluster()`, `sourceNamespace`) | -| `uploadID` | implies a per-batch handle, as in S3 multipart uploads | - -The name still carries less weight than the field's description, which is what a reader meets in -`kubectl explain`. That description states both delivery forms literally, so the concrete question -("what do I type here?") is answered where it is asked rather than compressed into an identifier. - -The block is `spec.attribution`, not `spec.authorAttribution`, even though every flag in the group -is spelled `--author-attribution-*`. The prefix is a -[flat-namespace device](../config-flag-conventions.md): flags need it because the product has two -attribution concepts (author attribution and -[render attribution](support-boundary/render-attribution.md)) sharing one undifferentiated list, -while a block on a source-cluster object cannot mean the rendering one. The chart already made this -call in four places, mapping `attribution.ttl` to `--author-attribution-ttl`, and the metrics agree -with `gitopsreverser_attribution_*`. Only the annotation-key flag is renamed here, and it keeps the -prefix, so the flag group stays internally consistent. - -The field sits under `spec.attribution` rather than at the top level, so later attribution settings -(per-provider grace, per-provider mode, both already contemplated in the `ClusterProvider` decision -record) have somewhere to go, and so `auditRoute` is unambiguous within its block. - -## What this costs - -One field, and one new way to be silently wrong. An `auditRoute` that names a route no apiserver -posts to produces exactly the failure in the bug report, and so does forgetting to set it on the -second provider for a cluster. That is why the warning below is part of this change rather than a -follow-up: without it, this design moves the silence one field to the left. - -The cost is bounded by being **explicit and inspectable**. `kubectl get clusterprovider -o yaml` -shows which route a provider reads, `kubectl get` can print it as a column, and two providers -sharing a cluster are visible as two objects carrying one value. None of that is true today, where -the coupling is implied by a name and discoverable only by reading a Redis key. - -### What it does not cost - -The semantics do not move, which is worth stating because the alternatives below all move something: - -- No change to the key's **structure**. The infix is relabelled and its value is sourced - differently, but no family gains or loses a dimension, so every join keeps its meaning. -- No test inversion. `TestAttributionIndex_CrossClusterIsolation` and - `TestAttributionIndex_RVOnlyHatchIsClusterScoped` both stay true as written, and only their key - fixtures move. -- No behaviour change for any existing install, because the field defaults to `metadata.name`. -- No resolve step in ingestion. The read side gains one field access on an object it already holds. - -The vocabulary rename costs a rollout window and a hard flag break, both taken deliberately and -without compatibility shims. See [migration](#no-backward-compatibility-is-implemented). - -## Could the cluster identify itself instead? - -This was the main alternative and it does not survive contact with the clone case. It is recorded -because the reasoning is what makes the declared route the right answer rather than a resigned one. - -| Candidate | Where it comes from | Distinct per cluster | Survives an etcd clone | -|---|---|---|---| -| `kube-system` namespace UID | one GET on the source client | yes | no, it is copied | -| `default/kubernetes` Service UID | one GET on the source client | yes | no, it is copied | -| kubeconfig `server` URL | the kubeconfig itself | no | not applicable | -| API server CA bundle | the kubeconfig itself | mostly | no, it is copied | -| API server `/version` | discovery | no | not applicable | -| `kcp.io/cluster` annotation | the audit event | yes, per logical cluster | not applicable | - -The `kube-system` namespace UID is the usual answer and the best of these. It is minted when the -cluster is created, it never changes, and the operator can already read it: `namespaces: get, list, -watch` is in [`config/rbac/role.yaml`](../../config/rbac/role.yaml) and applies to every source -client, so it would cost one cached GET and no new permission. - -The server URL cannot work, for two independent reasons. One cluster is reachable at several -addresses (an internal service address, an external load balancer, a bastion), so the same cluster -yields different strings. And the in-cluster config's server is `https://kubernetes.default.svc` for -every cluster viewed from inside itself, so the string that would identify the local cluster is the -one string that is identical everywhere. - -### A fingerprint cannot separate a clone from its origin - -An etcd snapshot restore copies the keyspace byte for byte, including every `metadata.uid`. A -cluster restored from a snapshot of another therefore carries the same `kube-system` UID as its -origin, exactly as it carries the same object UIDs. A fingerprint read from inside the cluster -cannot tell a clone from its origin, because the fingerprint is one of the things that got cloned. - -A restore that replays objects through the API instead (the Velero shape) has the API server mint -fresh UIDs for everything, so that topology collides on nothing and needs no protection either way. - -What separates two clones is not an identity that can be read. It is a human deciding these are two -clusters and saying so. A declared `auditRoute` is that decision, written down in the one place the -audit pipeline can act on it. - -### The options in full - -| # | Scheme | Fixes the reported bug | Separates clones | Cost | -|---|---|---|---|---| -| A | provider name in the key (today) | no | yes | none, it is what ships | -| B | object UID, dropping the cluster partition | yes | no | inverts a unit test and a documented invariant | -| C | cluster fingerprint as the key | yes | no | a resolve step on both sides, and the layer ingestion is meant not to have | -| **D** | **declared `auditRoute`, chosen** | **yes** | **yes** | **one field, and a new way to misconfigure** | -| E | B plus the fingerprint as a diagnostic | yes | no | one cached GET per provider | - -B is the tempting one, because object UIDs are globally unique and the -earlier fact-purge decision already leaned on exactly that to justify not purging. It was rejected -here for one reason: it makes the cluster boundary an emergent -property of UUID uniqueness instead of a stated fact, and that property is the one thing an etcd -clone breaks. C is B with more machinery and no more safety, since its identity is copied by the -same clone that copies the object UIDs. - -One property worth stating because it is easy to miss: under any of these, **a cluster sends its -audit events once**, however many `ClusterProvider`s name it. Under A that is also true, but only -one of those providers can use them. - -## Making the misconfiguration loud - -Part of this change, not a follow-up, because `auditRoute` introduces a value that can be wrong. - -`lastAuditEventTime` was specified in the `ClusterProvider` decision record and never implemented. -The `ClusterProviderStatus` doc comment still records it as deferred, and there is no such field in -`api/v1alpha3` today. It remains the right shape, because a quiet cluster is not an unhealthy one -and a timestamp says that where a condition cannot: - -**Built.** The resolver tracks, per route, whether attribution has ever resolved and how many events -have gone unresolved since. A route that has never resolved and produces -`attributionUnresolvedWarnThreshold` (5) unresolved events in a row logs once, naming the fix rather -than the symptom. It is one line per route per process: the condition is a configuration mistake, so -repeating it per event would bury it. - -The threshold is not 1 because a lone miss is ordinary (an audit batch can land after the grace -window under load). A run of them with nothing ever matched is the signature of a route nobody -writes to. - -**Not built.** The two status bullets originally specified here: - -- Record the time of the last fact stored per audit route. -- Surface it on `ClusterProvider.status`, alongside the resolved route. - -`lastAuditEventTime` remains what the `ClusterProviderStatus` doc comment already calls it: -deferred. The warning delivers the loudness this design owes the bug report; the timestamp is a -separate, smaller piece of observability work, listed here so its absence is a decision rather than -an oversight. - -## Migration - -No data migration, and no API version bump. `auditRoute` defaults to `metadata.name`, which is the -value every install already uses, so a provider that sets nothing resolves exactly what it resolved -before. Adopting the fix is per-provider and opt-in: set `auditRoute` on the extra providers for a -cluster, and their targets start resolving. The chart's `default` provider needs no change. - -### No backward compatibility is implemented - -Both renames are hard renames. There is **no deprecated flag alias, no dual-read of the old key -prefix, and no migration shim**, because the shared-stream audit path has no users to protect. That -is a deliberate decision recorded here so the absence does not read as an oversight, and so a -reviewer does not add compatibility code back on the assumption that it was forgotten. - -- `--author-attribution-cluster-annotation-key` becomes - `--author-attribution-audit-route-annotation-key`, and the old spelling is removed outright. An - operator that still passes it fails to start on an unknown flag, which is the loud outcome, and - the chart's `values.yaml` key is renamed in the same change. -- The fact key infix becomes `route:` with no dual-read. Facts written by an old pod under - `cluster:…` are never read again, and they expire on their own. - -The one consequence that survives is a rollout window: while old and new pods overlap, facts written -by one are not read by the other, so those events find no fact and commit as the configured -committer. That is the ordinary degradation, it self-clears at the fact TTL of minutes, and it is -the same rollout behaviour [`redis-key-schema-v3.md`](../finished/redis-key-schema-v3.md) §10 -documented for the `v2` to `v3` bump. Attribution is a freshness SLO rather than a correctness one, -so a few minutes of committer-authored commits is the accepted price for one vocabulary. - -## Tests - -Unit: - -- `ClusterProvider.AuditRoute()` returns the name when unset and the field when set. -- Two providers with different names and one shared `auditRoute` resolve one recorded fact. -- Two providers with distinct routes do not cross-credit. This is - `TestAttributionIndex_CrossClusterIsolation` unchanged in meaning, with its key fixtures moved. -- The audit handler records a fact for a route with no matching `ClusterProvider`, replacing the - test that asserts a 404. - -An e2e pins the reported behaviour so it cannot return. It belongs with the existing attribution -specs and mirrors their shape (`commit_author_attribution_e2e_test.go` already proves the OIDC claim -chain through the `default` provider): - -1. Create a dedicated in-cluster `ClusterProvider` whose name is not `default` and which sets - `spec.attribution.auditRoute: default`, plus a `GitProvider` with a zero commit window and a - `GitTarget` referencing that provider. -2. Create a `ConfigMap` while impersonating a user carrying OIDC display-name and email claims, - using the same helper the existing spec uses. -3. Assert `git log --pretty=%an <%ae>` for that object's own path is the impersonated identity, and - specifically not `unknown (attribution unresolved)`. - -Step 3 fails on `main` today and passes after this change, which is the property that makes the spec -worth having. A second case should mirror an object reached through a `rules[].sourceNamespace` -override on that provider, because that is the exact shape the bug report hit, and it also records -that the override itself was never implicated. - -A third case is worth its cost: a provider that sets no `auditRoute`, and whose name is not what the -apiserver posts under, still commits `unknown (attribution unresolved)` and fires the warning. That -pins the loud path, which is the half of this design that stops the next occurrence being -invisible. - -## Open questions - -1. **Should an in-cluster provider default differently?** Two distinct defaults are in play, and - only the second is open. - - The **baseline default is settled**: an empty `auditRoute` resolves to `metadata.name`, for every - provider. That is what keeps existing installs byte-identical, since the object name is already - what partitions their facts. - - The open one is a **conditional override on top of it**: a provider with no `kubeConfig` could - resolve an empty `auditRoute` to the literal `default` instead of to its own name, which would - fix the reported case with no configuration at all and save the srcns e2e providers a line each. - - The recommendation is no, and the reason is stronger than "it infers something". It would make - `default` a reserved name for the local cluster again, and - [that exact rule was proposed, enforced with CEL, and reversed](../finished/multi-cluster-author-attribution.md) - before shipping: local-versus-remote follows from `spec.kubeConfig` alone, for any name, and - pinning a name to a physical cluster is *"the same silent-retarget hazard that `kubeConfig` - immutability exists to prevent"*. A conditional default would smuggle that coupling back in - through the audit path, and it would also assume a provider named `default` is the local audit - route, which is exactly what an operator who renamed their route has not got. One explicit line - per extra provider, plus a warning that names the fix, is the cheaper trade. -2. **Value format validation.** RESOLVED, as a DNS-1123 **subdomain** (max 253) rather than the - label this section first proposed. A label would have been more restrictive than the default: - `metadata.name` on a cluster-scoped object is a subdomain and may contain dots, so an explicit - `auditRoute` could not have expressed the value it defaults to. Subdomain characters are all safe - in a URL path segment and in a Redis key, where `escapeKeyField` already handles `:` and `%`. -3. **Does the hardening doc's option A become identifier-bound mTLS?** Binding a certificate to the - `auditRoute` is strictly more useful than binding it to a provider object now that the identifier - is what partitions facts, but that is that document's call. diff --git a/docs/design/attribution-metrics-proposal.md b/docs/design/attribution-metrics-proposal.md deleted file mode 100644 index 1d66cf43..00000000 --- a/docs/design/attribution-metrics-proposal.md +++ /dev/null @@ -1,314 +0,0 @@ -# Proposal: the attribution metric surface - -> **design**: the attribution half is **built**. `watch_event_queue_seconds` is not: it moved to -> Phase 2 as a pipeline-wide metric. The Phase 1 surface below is now Phase 1 of -> [`metrics-observability-plan.md`](metrics-observability-plan.md), the canonical plan; this document -> is the reasoning trail behind it. Index: [`../INDEX.md`](../INDEX.md) - -Revised after review. An earlier draft proposed thirteen new metric families at once and four of them -would have produced misleading diagnoses. The corrections are spelled out in -[what the first draft got wrong](#what-the-first-draft-got-wrong), because two of the mistakes are -the kind that stay invisible: a series that is always zero, and a gauge that moves in the right -direction while counting the wrong thing. - -What survives is a smaller first release that covers normal-operation health and the loss paths -nothing can currently see, plus a deferred set with the preconditions each one needs. - -## Why change anything now - -**This release has already broken the `result` label.** `exact_deletecollection_item` is gone with the -`deletecollection` rework, replaced by `collection_uid` and `collection_scope`, and `name` is new. -Anything reading the old value stops matching whatever else happens. - -The usual reason to leave a label alone is that changing it breaks consumers, and that cost is paid -once per break. Since this release breaks `result` already, finishing the job costs the same single -migration; deferring it costs a second one later, on a label that will have been wrong twice. Nothing -consumes these metrics yet, which will not stay true. - -That argument covers the renames. It does not cover new metric families, which is why they are phased. - -## Phase 1: the set built first - -| Change | Kind | Why it is in the first release | -|---|---|---| -| `result` becomes `tier` plus `actor_kind` | label rework | the break is already happening | -| `weak` splits into `latest` and `resource_version` | label rework | same break, and `latest` is the tier the removal path turns on | -| `event_kind` on the wait histogram | new label | the wait behavior differs entirely between writes and removals | -| follower errors and last-success timestamp | new family | a wedged follower is silent today | -| `no_attribution_fact` outcome on `audit_events_total` | new value, existing counter | the population that produces no fact, counted where the decision is made | -| a stream-entry decode-error counter | new family | undecodable entries are dropped and skipped past with no log and no metric | -| `attribution_transport_info` | new family | changes how every other metric here is read | -| the four low-risk renames | rename | free while the surface is moving | - -Everything else waits. The renames are in [the table below](#the-renames). - -### `result` becomes `tier` plus `actor_kind` - -Today `result` has seven values and two of them are one tier seen twice: - -```text -exact_user exact_serviceaccount weak collection_uid collection_scope name absent -``` - -`exact` is the only tier that also encodes who the actor was, so counting exact resolutions means -summing two series, and the actor kind cannot be asked of any other tier. There is no way to learn -how many `name` or `collection_uid` resolutions named a service account. - -`gitopsreverser_commits_total` already carries `author_kind` with `user`, `serviceaccount`, -`committer` and `unresolved`, so two metrics currently disagree about the shape of one distinction. - -| Label | Values | -|---|---| -| `tier` | `exact`, `latest`, `resource_version`, `name`, `collection_uid`, `collection_scope`, `absent` | -| `actor_kind` | `user`, `serviceaccount`, `none` | - -**`weak` splits at the same time.** It currently covers both a `latest` (uid) match and the rv-only -hatch, which are different evidence: the object's own last write against a fact that had a -resourceVersion and no uid. The removal path turns on `latest` specifically, and the measurement that -found the window race had to infer "these were `latest` matches held as fallbacks" from a wait -distribution because the label could not say it. - -### `event_kind` on the wait histogram - -`ExactCapable` splits every query into a write or a removal, and the wait design differs completely -between them: a removal holds a fallback and keeps waiting, a write does not. Today the histogram -cannot distinguish an absent write from an absent removal. Adding `event_kind` = `write` / `removal` -makes the removal wait directly queryable, which is the number anyone tuning the grace needs. - -### `watch_event_queue_seconds`, moved to Phase 2 - -This was proposed here and is **not** part of the attribution release. It measures head-of-line -blocking on a watch shard, which is a property of the whole pipeline rather than of attribution, so -the canonical plan took it as the processing-delay stage of §4.2 and scheduled it with the watch -families. The argument for it stands and is kept here: the failure that broke an e2e spec was not a -slow resolution but the delay a slow resolution imposed on the events queued behind it on the same -single-threaded shard, which the wait histogram cannot see because it times each resolution in -isolation. It is also the pressure signal that makes a separate "resolvers waiting" gauge -unnecessary for now. - -### Follower health - -`attribution_fact_follower_errors_total` plus -`attribution_fact_follower_last_success_timestamp_seconds`. - -When the follower fails, `Run` logs and retries with a backoff, and nothing counts it. A follower -that is flapping, or wedged and retrying forever, degrades attribution to committer-authored across -the board, with a rising unresolved rate as the only symptom and nothing pointing at the cause. - -The timestamp matters more than the counter. A counter says errors are happening; only the timestamp -distinguishes "erroring occasionally while making progress" from "has not read anything in ten -minutes", and only the second is an outage. - -### `no_attribution_fact` on `audit_events_total` - -[`internal/audit/outcome`](../../internal/audit/outcome/outcome.go) is already the single bounded -vocabulary for what ingestion did with one event, with a derived `Category` and an e2e invariant that -gates on `category="error"` being zero. An event that is accepted but yields no attribution fact has -no terminal value there today. - -Adding one, in the `Dropped` category rather than `Error`, counts that population at the point where -the decision is made and where the event's type and verb are still on the label set. That is where -the aggregated-API create shows up: it is rejected before publication, so no fact-side counter can -ever see it. - -### A decode-error counter for stream entries - -This is the gap the "every silent drop gets a counter" principle should have caught first and did -not. Both transports do the same thing with an entry they cannot decode: - -```go -facts, err := factsFromMessage(messages[j]) -if err != nil { - continue -} -``` - -and then advance the cursor past it. No log, no metric, no retry. A malformed or future-schema entry -is discarded and the follower moves on as though it had read it. - -`attribution_fact_stream_decode_errors_total` is the whole fix. It belongs in the first release -because it is the one loss path with no symptom at all: unlike a trim gap it is not detectable after -the fact, and unlike a publish failure the API server does not retry it. - -### `attribution_transport_info{transport}` - -An info gauge, value always 1, with `transport="redis"` or `transport="memory"`. - -It is in the first release despite being a new family because it is interpretive metadata rather than -a signal: the two transports have different failure modes, and the same symptom means different -things under each. A burst of unresolved commits after a restart is expected under the in-memory -transport, which loses every fact on restart by design, and is a bug under Redis. Reading any of the -other metrics without knowing which is in force is reading them without knowing the contract. - -If the first release needs to be smaller still, this is the one to cut. - -## What the first draft got wrong - -Each of these was checked against the code. They are recorded rather than deleted, because the reason -each was wrong is more useful than the corrected proposal. - -### `published - filed` is not delivery loss - -The draft proposed a lifecycle counter whose stages could be subtracted: `published` minus `filed` -for delivery loss, `filed` minus `matched` for facts that went unused. - -The subtraction is invalid. `published` counts every fact appended by the audit receiver, for every -type. `filed` would count only facts arriving on streams **this process follows**, which is a subset -chosen by which watches are running. Replay compounds it: a restart re-reads the retention window and -files the same facts again, so the second number can exceed the first without anything being wrong. - -Two counters over different populations do not subtract. Delivery loss has to be measured where -delivery happens, which is what the follower health signals above do. - -### `unfilable` would be a permanently zero series - -The draft claimed a stage for facts the index can file under no key, and that it would show "every -aggregated-API create". - -It would show neither. The publish gate rejects an event with no resolvable name unless it is a -collection verb, so a fact reaching the index always has a name, a uid, an rv, or is a collection -fact. With the name tier in place every one of those files somewhere. The `default` branch in `file` -is unreachable, and a counter on it would be a flat zero that reads as health. - -The aggregated create is the population the draft was reaching for, and it never becomes a fact at -all: it is rejected before publication. Counting it is what the `no_attribution_fact` outcome above -does, on the ingestion side where the event still exists. - -### The named failure metric does not exist - -The draft said publish failures are already visible as `audit_eventlist_*{outcome="write_error"}`. - -`write_error` is a value on `gitopsreverser_audit_events_total`, which is per event. The -`audit_eventlist_*` families are request-level and carry a different outcome set. An alert written -against the name in the draft would report zero forever, which is the worst failure mode a monitoring -change can have. - -### `facts_filed_total{tier}` conflates two models - -The draft proposed counting facts by the tier they were filed under, to show the publish-side -distribution. - -Tiers are resolution outcomes, and they do not partition facts. A fact with a uid and a -resourceVersion is filed under **both** `exact` and `latest`, which the publish-side documentation in -this repository states explicitly. Counting by tier would double-count the most common fact shape and -skew the distribution toward the tier that matters least. - -The question underneath it stays interesting: what shape are facts arriving in, and how many carry -only a name. That needs a fact-shape taxonomy (`uid_rv`, `uid_only`, `rv_only`, `name_only`, -`collection`), which is a different label with different values. Deferred rather than renamed, -because it needs designing rather than editing. - -### `resolvers_waiting` would count registrations - -The draft proposed a gauge incremented when a resolver registers its waiter keys. - -`Await` registers **before** its first lookup, deliberately, so that a fact arriving in the gap wakes -a waiter already listening. Most resolutions then return from that first lookup without ever -blocking. A gauge incremented at registration therefore counts resolutions in flight rather than -resolvers blocked, and it would read as pressure on a healthy system. - -An earlier revision of this document also claimed the existing `factWaiterRegistry.len()` could be -exported directly. It cannot: `len()` returns the number of candidate KEYS holding a waiter, and one -resolver registers under several, so it over-counts by roughly the tier fan-out. - -If it is built later it has to be incremented around the blocking `select` alone. -`watch_event_queue_seconds` measures the same pressure and is scheduled for Phase 2, so this may -never be needed. - -### `streams_behind` does not mean what the name says - -The draft treated it as a backlog depth and an early-loss indicator. - -`behind` is set when **the last read filled its entry budget**, meaning more was waiting when the -read returned. It is the precondition for trim-gap detection rather than a measure of how far behind -the follower is. A stream one entry behind and a stream a thousand entries behind carry the same -value. - -Named and interpreted as drafted it would invite an alert on a condition that occurs during any -ordinary burst. It needs redefining, or replacing with a real lag measure, before it can carry that -meaning. - -### `fact_index_replay_seconds` cannot show what it was for - -The draft proposed measuring replay to show that a restart warms the index before serving. - -There is no replay-complete boundary to measure. The follower runs continuously, streams are added to -the subscription set as watches start, and no readiness barrier gates serving on the index being -warm. A duration recorded today would measure an arbitrary window rather than the property the metric -was proposed to prove. - -The boundary has to exist first. That is a design change with its own value, and it is the same -question the fact-stream record leaves open about HA handover: whether a replica must warm its index -before starting a watch it has taken over. Build the barrier, then measure it. - -## Deferred, and what has to be true first - -| Deferred | Precondition | -|---|---| -| `fact_index_replay_seconds` | a replay-complete boundary and a readiness barrier exist | -| the four stream-scaling metrics | the followed-stream count is large enough to be in question, and `behind` is redefined as real lag | -| fact-shape distribution | a shape taxonomy distinct from the tier taxonomy | -| `resolvers_waiting` | queue delay proves insufficient, and it is measured around the blocking select | -| `fact_index_expired_total` | wanted when tuning the TTL or the caps; low risk, low urgency | - -The stream-scaling set was designed in an earlier revision of this document and that design stands on -its own merits. What changed is the ordering: it is an investigation suite for a question nobody has -observed a problem with, and building it before the health signals above inverts the priority. The -part worth keeping in view is that a count alone cannot answer whether the stream count is reasonable, -because the same number is fine or fatal depending on what it costs. - -## The renames - -| Now | Proposed | Why | -|---|---|---| -| `attribution_resolutions_total{result}` | `{tier, actor_kind}` | one label, two dimensions | -| `attribution_resolution_wait_seconds{result}` | `{tier, event_kind}` | same, plus the write and removal split | -| `attribution_fact_events_total{op}` | `attribution_facts_total{op}` | "events" already means audit events and watch events | -| `attribution_fact_index_size` | `attribution_fact_index_entries` | a gauge should name what it counts | -| `attribution_collection_degraded_total{reason}` | `attribution_collection_without_uidset_total{reason}` | nothing broke; the precise join was unavailable | - -`attribution_fact_index_evictions_total{reason}` and `attribution_fact_stream_gaps_total{stream}` are -unchanged and stay as they are. - -## Reconciling with the canonical plan - -**Done.** [`metrics-observability-plan.md`](metrics-observability-plan.md) declares itself the single -canonical metrics plan, and its attribution taxonomy and this proposal could not both be right. Its -version had drifted from the code when the fact-stream work landed: - -| It specified | The code has | -|---|---| -| `result` includes `conflict` and `expired` | neither value exists | -| `attribution_fact_events_total{op}` includes `expired_unmatched` and `late` | neither op exists | -| `attribution_fact_index_size` is "facts parked in **Redis**" | the index has been in process memory since the fact-stream work | - -So this was never a choice between two live designs. The canonical plan has now absorbed the Phase 1 -surface below as its attribution stage (§4.4 and §5 there), records the drift above as a correction, -and links back here as the reasoning trail. A taxonomy that lives in two places diverges again, and -the canonical plan is the one people are told to read; this document keeps the argument, not the -inventory. - -Two things the canonical plan took from here and generalized, because they are not attribution's -alone: - -- **`watch_event_queue_seconds`** became the processing-delay stage of the whole pipeline rather than - an attribution metric. It measures head-of-line blocking on a watch shard, which attribution - happens to be the loudest current cause of. -- **"every silent drop gets a counter"** became a numbered principle there. It is the rule this - document broke first and then repaired: the decode-error gap below is what a stated principle would - have caught. - -## What to write down - -An [`UPGRADING.md`](../UPGRADING.md) entry with a table of old label and metric names against new -ones, so a query can be rewritten mechanically. It should state that `result` is gone rather than -only describing what replaces it. - -The break itself needs no defense beyond being written down. **Nothing consumes these metrics yet**: -no dashboard ships, no alert rules ship, and no user has been told to build against these names. The -migration costs one entry today and a consumer migration after the first published dashboard, which -is the whole reason the surface is corrected in the same release that broke `result` anyway. - -Worth fixing while in that file: its current attribution entry is headed -`## Unreleased — … (next minor; …)`, which [`AGENTS.md`](../../AGENTS.md) forbids, because by the -time an upgrade guide is read both halves of that heading are false. diff --git a/docs/design/attribution-publish-and-join.md b/docs/design/attribution-publish-and-join.md deleted file mode 100644 index 493dbf57..00000000 --- a/docs/design/attribution-publish-and-join.md +++ /dev/null @@ -1,436 +0,0 @@ -# How attribution works: the publish side and the join side - -Attribution has two halves that never call each other. One turns an audit event into a FACT; the -other turns a watch event into an AUTHOR by finding a fact. They meet only in the index, through the -keys a fact was filed under. - -This is the reference for what each half does, exactly. For the measurements that produced these -rules, see [`attribution-branch-findings.md`](attribution-branch-findings.md). - -The one thing to carry into both diagrams: **neither half branches on the type.** Every decision is -made on the VERB of the request and on which fields the event happens to carry. Why that is a -requirement rather than a coincidence is the last section. - -## Part 1: the publish side, audit event to fact - -One audit event in. Zero or one fact out, filed under one to three keys. - -```mermaid -flowchart TD - A[Audit event, ResponseComplete] --> B{objectRef present
with a resource?} - B -->|no| X[No fact] - B -->|yes| C{user resolvable?} - C -->|no| X - C -->|yes| D[identity = objectRef namespace, name, uid
then backfill what is missing from the body] - D --> E{verb is
deletecollection?} - - E -->|yes| F[COLLECTION fact
drop uid, name, rv
keep selector from requestURI
keep uid set from the body, if any] - E -->|no| G{identity has
a name?} - G -->|no| X - G -->|yes| H[OBJECT fact
namespace, name, uid, rv, verb, author] - - F --> K[file under: collection namespace] - H --> L{strongest key it has
FIRST MATCH WINS} - L -->|uid and rv| M[file under: exact uid+rv
AND latest uid] - L -->|uid, no rv| N[file under: latest uid] - L -->|rv, no uid| O[file under: rv] - L -->|name, no uid, no rv| P[file under: namespace+name] - L -->|none of these| X - - style X fill:#7f1d1d,color:#fff - style K fill:#14532d,color:#fff - style M fill:#14532d,color:#fff - style N fill:#14532d,color:#fff - style O fill:#14532d,color:#fff - style P fill:#14532d,color:#fff -``` - -Two gates drop an event entirely: no resource, and no resolvable name on a non-collection verb. A -collection request is exempt from the name gate because it names no object by nature, and that is -the one place the verb changes which gate applies. - -A third gate is the fact's whole reason to exist: **no resolvable user, no fact**. It is the one -field the wire contract requires, and the read side enforces it too. `AuthorFact.UnmarshalJSON` -refuses an entry naming nobody, which lands on -`gitopsreverser_attribution_fact_stream_decode_errors_total` rather than being half-absorbed. So a -fact in the index always names an actor, and the metrics can read attribution coverage off the tier -alone. The event that produced no fact is not lost either: it is counted `no_attribution_fact` on -`audit_events_total`, where its type and verb are still in hand. - -The body backfill is why the name gate is survivable. `objectRef` alone often lacks the name or the -uid; `IdentityFromAuditEvent` fills what is missing from the request or response object, preferring -the request object for a delete and the response object otherwise. What the event carries in its body -therefore decides which keys the fact ends up with, and the type has nothing to do with it. - -### Filing picks one branch - -The fact keeps every field it recovered, but it is FILED under one branch only. `file` is a switch on -the strongest key present, and the first matching case wins: a fact with a uid is not also filed -under its name or its resourceVersion, even though it has them. - -The reason is memory. A watch event always knows its object's uid, so it always asks a uid tier -first, and a uid-keyed fact always answers there. A second copy of that same fact under its name -would never be the one read. Storing it anyway costs the entry on every replica following the type, -for the whole TTL, and again on every restart replay, and buys nothing. - -The one branch that files more than once is the uid case: `exact` (when the fact also has a -resourceVersion) serves creates and updates, `latest` serves removals, and the two answer different -questions about the same object. A fact whose own verb is a **removal** takes a third structure, the -sticky removal pointer, which answers a question no later fact can: who asked for the deletion. - -So the rule is: keep every field, file under exactly the keys a query could reach you by. - -## Part 2: the join side, watch event to author - -One watch event in. It asks the index for the strongest fact about this object, waiting up to the -grace for one to arrive. - -```mermaid -flowchart TD - A[Watch event] --> B[scope = audit route + group/resource] - B --> C{scope known?} - C -->|no| Z[absent: committer-authored] - C -->|yes| AA{a removal, and a sticky
removal pointer for its uid?} - AA -->|yes| AB[delete_sticky] - AA -->|no| D{a fact under
this uid and rv?} - - D -->|match| E[exact] - D -->|no match| F{is this a removal?} - - F -->|yes| G{uid in a collection's
uid set?} - G -->|yes| H[deletecollection_body_uid] - G -->|no| I{latest uid
is a DELETE fact?} - I -->|yes| J[latest: the object's own delete] - I -->|no, it is a write| K[hold it as a fallback] - K --> L{name tier holds
a DELETE fact?} - L -->|yes| M[name] - L -->|no| N{fallback held?} - N -->|yes| O[latest: last writer] - N -->|no| P{collection covers
this scope + selector?} - P -->|yes| Q[deletecollection_scope] - P -->|no| R - - F -->|no| R{rv-only hatch?} - R -->|match| S[resource_version] - R -->|no match| T{name tier?} - T -->|match| U[name] - T -->|no match| Z - - style Z fill:#7f1d1d,color:#fff - style AB fill:#14532d,color:#fff - style E fill:#14532d,color:#fff - style H fill:#14532d,color:#fff - style J fill:#14532d,color:#fff - style M fill:#14532d,color:#fff - style O fill:#166534,color:#fff - style Q fill:#166534,color:#fff - style S fill:#166534,color:#fff - style U fill:#14532d,color:#fff -``` - -### The tiers, strongest first - -| Tier | Key | `tier` label | What it asserts | -|---|---|---|---| -| delete, sticky | uid (a delete fact, sticky) | `delete_sticky` | who asked for this object's deletion | -| exact | uid + rv | `exact` | this actor produced this exact version | -| collection uid | uid in a collection's set | `deletecollection_body_uid` | the API server said this request deleted this object | -| latest, delete | uid | `latest` | this object's own delete fact | -| name, delete | namespace + name | `name` | this object's own delete fact, when it has no uid | -| latest, write | uid | `latest` | who last wrote it; a fallback for a removal | -| collection scope | namespace + selector + window | `deletecollection_scope` | a collection request covering it was made | -| rv-only | rv | `resource_version` | a fact with an rv but no uid | -| name | namespace + name | `name` | the only key an aggregated write has | -| absent | none | `absent` | committer-authored | - -Who the evidence named is the separate `actor_kind` label (`user` / `serviceaccount` / `none`), so -every row above can be asked about either kind of actor. - -### Three rules that are easy to miss - -**A fact about a DELETION may not be replaced by a fact about a WRITE.** Every ordinary structure -here is last-writer-wins, and a finalizer patch's fact carries the resourceVersion the DELETION -stamped, so it lands under the deleter's exact key and the deleter's uid key alike. A removal fact -therefore also takes a **sticky removal pointer**, keyed by uid, that only another removal fact may -fill, and a removal consults that pointer before the exact tier. It is the one structure the TTL does -not bound: a uid is unique across space and time, so the statement can never be superseded, and its -horizon is the index's caps instead. It is strictly uid-keyed, because the same stickiness on the -name tier would be a defect: a name is reused after a delete and recreate. See -[attribution-deletion-intent-actor.md](attribution-deletion-intent-actor.md). - -**A removal never returns on a write fact without looking further.** The per-object tiers are -last-writer-wins, so for a removal they hold whoever last EDITED the object, which is not who deleted -it. Such a match is held as a fallback while the search continues, and the caller keeps waiting for -delete evidence until the grace expires. A fact about the deletion, filed under any key, ends the -wait immediately. - -**An exact-capable event may not fall through to the removal tiers.** A create or update presents the -resourceVersion its own write produced. If the exact tier misses, the `latest` pointer may name an -older, different author, so the lookup skips straight to the rv hatch and the name tier. - -The gate is one-directional, and only one of the two tiers above is gated. The exact tier is tried -for *any* query carrying a uid and a resourceVersion, a removal included. A removal that misses -the sticky pointer can therefore still resolve at `exact`, which is exactly what it did before the -pointer existed. What an exact-capable event may not do is the reverse: reach the tiers below. - -## The wait, and what changed about it - -The two halves are racing, and the watch side reliably wins. The API server batches audit deliveries -(`--audit-webhook-batch-max-wait`), while the watch event is streamed, so by the time a watch event -needs an author its fact is usually still inside the batch window. The first lookup is a -near-guaranteed miss. That is the whole reason a grace window exists. - -So the resolver does not ask repeatedly. It arms a signal, looks once, and then sleeps until either a -fact that could match it arrives or the grace runs out. - -```mermaid -sequenceDiagram - participant W as watch shard - participant R as waiter registry - participant I as index - participant F as fact follower - - W->>R: register(waiterKeys): one entry per tier this query could match - Note over W,R: registered BEFORE the read, so a fact
landing in the gap still wakes it - W->>I: Lookup - I-->>W: absent - W->>W: select { waiter.ch | ctx.Done | timer.C } - - F->>I: apply fact, file under its keys - I->>R: wake(keys the fact filled) - R-->>W: signal - W->>I: Lookup again - I-->>W: resolved - Note over W: defer unregister, whatever the outcome -``` - -Registering first is what closes the race the old 150ms poll loop papered over by looking again: a -fact delivered between the register and the read wakes a waiter that is already listening. There is -no Redis call on this path. The fast case is a map read; the waiting case is a channel receive. - -### The Go mechanics, because they carry the guarantees - -The registry is `map[factWaiterKey]map[*factWaiter]struct{}`: candidate key to the set of resolvers -blocked on it. That shape is the fan-out. One resolver registers under SEVERAL keys, one per tier its -event could resolve through, and one applied fact wakes every resolver registered under any of the -keys that fact filled. It is a many-to-many join done through an index rather than a broadcast, so a -fact never touches a resolver it could not have answered. - -Four details do real work: - -- **`chan struct{}` with buffer 1.** The signal carries no payload, because the payload is the index - itself: the woken resolver re-reads it. Buffering one means a signal sent while the resolver is - mid-recheck is still there when it comes back around, so it is not lost. -- **Non-blocking send.** `wake` does `select { case ch <- struct{}{}: default: }`, so the goroutine - applying facts is never slowed by a resolver that has not looked yet, and a second signal on an - already-signaled waiter is dropped. One pending wake-up is enough, because the resolver - re-reads everything rather than consuming a queue of events. It also makes the send safe to do - while holding the registry lock, since it cannot block. -- **`select` on three cases.** The resolver waits on the waiter, `ctx.Done()`, and the grace timer - together, so shutdown and the deadline are not special paths. -- **`defer unregister`.** Registration is undone on every exit, including the ones that return early. - The registry exposes a `len()` purely so a test can assert a resolver left nothing behind. - -The loop around the `select` matters too, because a wake-up is only a hint. The resolver re-runs the -whole lookup, and if what arrived was not good enough (a write fact when it needs delete evidence) -it keeps waiting rather than treating the signal as a result. - -### A removal waits for evidence about the deletion - -A match does not always end the wait. The per-object tiers are last-writer-wins, so the fact present -earliest for a removal is usually the object's last WRITE, which says who edited it and nothing about -who deleted it. Returning on that answered "who deleted this" with "who last edited it" whenever -anyone had touched the object first. - -Such a match is now held as a fallback and the wait continues for evidence about the deletion itself. -Waiting never costs an attribution: the worst case returns exactly what returning early would have -returned, one grace later. - -### What that cost, and the fix - -The wait is not free, and this is the part worth knowing before tuning anything. `attachAuthor` runs -on the watch shard's own goroutine, and a shard processes its events serially, so a removal that -waits out its grace is **head-of-line blocking** for every later event of that type. The commit -window for a subsequent write cannot open until its event is processed. - -That turned into a real failure. Measured on the e2e cluster, which runs a 10s grace, three removals -in one run spent 20.18s between them; a later Deployment create queued behind them, its window opened -about ten seconds late, and a CommitRequest created 105ms after the write reported `NoWindowInGrace` -about a window that had not been allowed to exist yet. - -The cause was that the delete evidence was in the index and unreachable. A delete fact only has a uid -if the API server answered the request with the object; when it answers with a `Status` the fact's -only key is its name. `Lookup` returned as soon as the removal ladder yielded anything, and the uid -tier yielded the last write fact, so the name-keyed delete fact below was never consulted. - -`lookupRemoval` now applies one rule: **a fact about the DELETION outranks a fact about a write, -whichever key each is filed under.** The object's own delete fact answers from the uid tier, then -from the name tier, and only then does the held write fact answer. - -| | before | after | -|---|---|---| -| `weak` | 3 resolutions, 20.18s, mean 6.73s | 2 resolutions, 0.28s, mean 0.14s | -| `name` | never reached | 2 resolutions, 0.60s, mean 0.30s | -| total resolver wait | **21.24s** | **1.63s** | - -Still open: the head-of-line block itself. A removal with no delete fact coming at all (a type the -audit policy excludes) still stalls its shard for a whole grace. This change removes the -largest population that was hitting it; it does not change that a blocking resolve on a serial -goroutine can do this at all. See -[`attribution-removal-wait-options.md`](attribution-removal-wait-options.md). - -## What is observable - -Every metric on this path, and the question each answers. - -| Metric | Labels | Answers | -|---|---|---| -| `gitopsreverser_attribution_resolutions_total` | `tier`, `actor_kind`, `group`, `version`, `resource` | which evidence named the author and who it named, per type | -| `gitopsreverser_attribution_resolution_wait_seconds` | `tier`, `event_kind`, `group`, `version`, `resource` | how long the join waited, by tier and by write/removal | -| `gitopsreverser_attribution_facts_total` | `op` = `written` / `matched` | how much of what is published is ever used | -| `gitopsreverser_attribution_fact_index_entries` | none | entries held across every scope | -| `gitopsreverser_attribution_fact_index_evictions_total` | `reason` = `per_type` / `total` | whether the caps are binding | -| `gitopsreverser_attribution_collection_without_uidset_total` | `reason` = `uid_cap` / `no_uids` | how often the precise collection join was unavailable | -| `gitopsreverser_attribution_fact_stream_gaps_total` | `stream` | facts lost for good to a trim | -| `gitopsreverser_attribution_fact_stream_decode_errors_total` | `transport` | entries skipped because they could not be decoded | -| `gitopsreverser_attribution_fact_follower_errors_total` | `transport` | follower reads that failed and were retried | -| `gitopsreverser_attribution_fact_follower_last_success_timestamp_seconds` | none | whether the follower is reading at all | -| `gitopsreverser_attribution_transport_info` | `transport` | which contract the metrics above are read under | -| `gitopsreverser_commits_total` | …, `author_kind` | what reached Git | - -The wait histogram is the one that earns its keep. Splitting wait time BY TIER is what turned the -window race from a mystery into a measurement: the uid-latest tier at a 6.7s mean against the exact -tier at 0.18s said immediately that removals were sitting out their grace, which no aggregate mean -would have shown. `event_kind` now makes that reading direct rather than inferred: the measurement -that found the race had to deduce "these were latest matches held as fallbacks" from the shape of a -distribution, because no label could say it. - -### What is not visible, and one gap that mattered - -**`written` minus `matched` is not delivery loss, and never was.** `written` counts every fact -appended for every type; `matched` counts only facts joined on streams THIS process follows, and a -restart re-files the whole retention window. Two counters over different populations do not subtract. - -The population that motivated the subtraction (name-only delete facts published and silently -discarded) no longer exists: the name tier files them, so `file` returning no keys is now -unreachable behind the publish gate, and a counter on that branch would be a flat zero that reads as -health. What the loss paths needed was measuring where delivery happens: the stream -decode-error counter and the follower's last-success timestamp, both of which now ship. - -**Head-of-line blocking is not measured.** The wait histogram times each resolution in isolation. It -does not measure the delay a slow resolution imposes on the events queued behind it on the same -shard, which is the thing that broke a spec. Time-in-queue per shard, or the age of an event -when it reaches the branch worker, would name it directly. - -**The publish-side tier distribution is not counted.** How many facts land under a name versus a uid -is only discoverable by reading the index. Given that this ratio is the aggregated-API story, it is -worth a counter. - -## The `exact_user` / `exact_serviceaccount` split was a modeling wart, now **fixed** - -`result` is gone; `tier` and `actor_kind` replace it, `weak` split into `latest` and -`resource_version`, and the wait histogram gained `event_kind`. The reasoning is kept below because -it is the argument for the shape, and the migration is in -[`UPGRADING.md`](../UPGRADING.md#0410--the-attribution-metrics-are-relabelled-and-partly-renamed-breaking-for-queries). - -The inconsistency was real rather than cosmetic. `result` should have named the TIER: `weak`, -`name`, `deletecollection_body_uid`, `deletecollection_scope`, `absent` all did. `exact` was the only one that also -encoded WHO the actor was, which crammed two orthogonal dimensions into one label. - -Two consequences followed directly: - -- counting exact resolutions meant summing two series, and any new actor kind would have multiplied - them again; -- the actor kind could only be asked of the exact tier. There was no way to ask how many `name`-tier - or `deletecollection_body_uid` resolutions named a service account, because that dimension did not exist - there. - -The decisive argument was that the codebase already modeled it correctly one metric over: -`gitopsreverser_commits_total` carries `author_kind` as its own label, with `user`, `serviceaccount`, -`committer` and `unresolved` as values. So the two metrics disagreed about the shape of the same -distinction. - -The shipped shape is that one: `tier` names the evidence, `actor_kind` matches `commits_total` and is -available on every tier. It was cheap in code, as predicted: the actor kind is derived from the -author string at read time, so nothing new is stored or plumbed. It was still a **breaking metric -change**, taken deliberately in one release with the other label work rather than as a side effect of -an attribution fix. - -## Why it is split into two halves at all - -The split is not an accident of layering. Three requirements each rule out the obvious alternative of -resolving an author inside the audit receiver, or handing it to the watcher over a channel. - -**The audit endpoint must answer fast, and keep answering during a deploy.** The receiver decodes a -batch, appends one entry per type, and returns. It does no lookup, waits for no watcher, and holds no -per-object state. A retried POST may append the same batch twice and that is safe without any -deduplication work on the hot path, because a fact is keyed data rather than a position in a -sequence: the duplicate carries the same author under the same `(uid, rv)`, `latest` is -last-writer-wins over identical content, and a waiter woken twice resolves to the same name. - -**It has to survive more than one replica.** The API server's audit webhook posts through a Service -to whichever replica answers, while a given object's watch shard lives on whichever replica owns that -`GitTarget`. Those are unrelated choices, so the fact and the watcher that needs it routinely land in -different processes. A per-type stream with independent per-reader cursors is exactly the primitive -for that: the receiving replica appends, every replica watching the type reads, and neither needs to -know about the other. An in-process channel works perfectly on one replica and has to be thrown away -on the second. The alternatives are worse in a more expensive way: sticky audit routing would make -the API server's load balancing this operator's problem. - -**Rollouts are the normal state, not the exception.** A replicated deployment is almost always -mid-rollout, reconnecting, or restarting a pod, and those are precisely the cases where plain publish -and subscribe drops facts silently. A resumable stream replays the retention window instead, so a -process that restarts rebuilds its index rather than starting blind. - -**And the delay is not ours to remove.** The batching parameters belong to the API server. Since the -fact is late by construction, the resolver must wait for something rather than ask repeatedly, which -is why the wait is a signal on an in-process index and not a poll against Redis. The old loop ran to -completion on essentially every attributable event, because the first lookup was a near-guaranteed -miss. - -What the split does NOT solve is worth stating too: it stops attribution being an HA blocker, but the -real HA problem is ownership: which replica owns a `GitTarget`, and keeping commits to one -`(GitProvider, branch)` serialized through a single writer. That lives in -[`ha-gittarget-distribution-plan.md`](../future/ha-gittarget-distribution-plan.md). - -## No branch anywhere depends on the type - -Verified across the whole path: `internal/queue`, `internal/auditutil`, and the resolver contain no -comparison against a group, a resource, a kind or an API version. The only `Resource ==` in the path -is `Resource == ""`, a presence check. - -Everything dispatches on one of three things: - -- **the verb**: `deletecollection` publishes a collection fact, and `delete` plus `deletecollection` - are what `isRemovalVerb` recognizes as evidence about a deletion; -- **the operation kind**: `ExactCapable` is false for a removal, which is what unlocks the weaker - tiers; -- **which fields are present**: uid, resourceVersion, name, request body, response body. - -The type appears exactly once, in `factScope{route, groupResource}`, and there it is a PARTITION -rather than a decision: it keeps one cluster's and one type's facts from being handed to another's. -No code reads it to choose a behavior. - -### This is a requirement, not an accident - -A type-based rule would be wrong, and the corpus already proves it. Two ConfigMap deletes, same -cluster, same type, same verb, different shapes: - -- `configmap/finalizer-delete/audit.delete.yaml`: the response object is the **ConfigMap**, so the - uid is recoverable and the fact lands on the uid tiers. -- `configmap/owner-ref-cascade/audit.delete.cm-parent.yaml`: the response object is a **`Status`**, - so there is no uid anywhere and the fact's only key is its name. - -A rule of the form "ConfigMaps behave like this" cannot express that, because the difference is not a -property of ConfigMaps. It is a property of the individual request, decided by propagation policy and -by what the API server chose to return. - -The same principle is what makes aggregated APIs work without ever being mentioned in the code. A -flunder is not special-cased anywhere; it produces events with no uid, and the shape-driven rules -route it to the name tier on their own. When the wardle API was added to the lab, no -attribution code changed to accommodate it. - -So the answer to "is a type check needed?" is no, and adding one would be a regression: it would -replace a rule that reads what the event contains with a guess about what a type usually contains. -The exception that would justify one has not appeared, and the two ConfigMap rows above are the -standing argument that it would be unsound if it did. diff --git a/docs/design/attribution-removal-wait-options.md b/docs/design/attribution-removal-wait-options.md index da23bc2d..87b063ea 100644 --- a/docs/design/attribution-removal-wait-options.md +++ b/docs/design/attribution-removal-wait-options.md @@ -191,8 +191,9 @@ a route has produced a long run of unresolved events. Extend that to `(route, gr use it to decide the wait: a type that has never once produced a fact on this route is not going to start, so a removal on it should take its fallback immediately. -This is the [circuit breaker](attribution-wait-poll-vs-push.md#option-c-circuit-break-a-route-that-has-never-resolved-anything) -the earlier record already proposed, applied to the case that turns out to need it. +This is the circuit breaker an earlier record already proposed for a route that has never resolved +anything, applied to the case that turns out to need it. The loud-route warning it extends is in +[the attribution spec](../spec/attribution.md#the-scope-is-an-audit-route-and-a-type). - **For**: it targets situation 5 exactly, and for the dominant cause — a type the audit policy drops — it is not a heuristic but a fact about the configuration: the type is excluded, so it diff --git a/docs/design/attribution-wait-poll-vs-push.md b/docs/design/attribution-wait-poll-vs-push.md deleted file mode 100644 index 8a8809ce..00000000 --- a/docs/design/attribution-wait-poll-vs-push.md +++ /dev/null @@ -1,545 +0,0 @@ -# Waiting for an audit fact without polling for it - -> **design**: superseded. Index: [`../INDEX.md`](../INDEX.md) -> -> **The decision was taken in [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md)**, which -> replaces the fact keyspace with a per-type Redis stream and an in-memory index. This record is -> kept as the reasoning trail: the six options, what each costs, and the measurements that ruled -> most of them out. Read it for why, then read the other one for what. -> -> Nothing here is shipped. The current behavior is the poll loop described under -> [what the wait costs today](#what-the-wait-costs-today); everything from -> [the options](#the-options) onward is intent. -> -> Prompted by a review question: the attribution lookup waits for a fact to appear in Redis, and -> the way it waits is a fixed-interval poll. This record separates the part of that design that is -> forced (the wait) from the part that is a choice (the poll), and lists what could replace the -> choice. -> -> The finding that drives the recommendation is in -> [which one fires first](#which-one-fires-first): audit delivery is batched by the apiserver, so -> the watch event arrives first by roughly the batch window and the first lookup is a -> near-guaranteed miss. Two measurements already in the repository support it. - -## The decision to make - -A live watch event needs a commit author before it can be routed, and the audit fact that names -that author may not have arrived yet. Waiting is unavoidable. Polling Redis every 150ms for up to -three seconds, on the watch shard's own goroutine, is one way to wait, and it is the expensive one. - -The question is whether to keep it, tune it, or replace it with a notification. - -## Why the shape of this architecture is worth keeping - -Before criticizing the wait, it is worth being explicit about what the surrounding design gets -right, because every option below is constrained by wanting to preserve it. - -**Attribution is a layer on top of the watch, not a rewrite of it.** The mirroring path is complete -without it. A watch event carries the object, the operation, and the resource identity, and it -produces a correct commit whether or not anyone ever names an author. -[`RedisStore`](../../internal/queue/redis_store.go#L45) holds the resume cursors and is a hard -dependency in every mode; -`AttributionIndex` is built on the same connection -only when the operator asks for author attribution, and it -[knows nothing about cursors](../../internal/queue/redis_store.go#L79). Turning attribution off is -expressed by leaving `Manager.AuthorResolver` nil, at which point -[`attachAuthor`](../../internal/watch/target_watch.go#L748) returns immediately and the commit is -authored by the configured committer. There is no second code path for the unattributed case, no -feature flag threaded through the writer, and no degraded mode to test separately. The enhancement -is optional in the strong sense, and the cost of not using it is zero. - -**One fact serves every watcher that needs it.** A fact key is -`route:::object::` -(`factKeyExact`). Notice what is absent from it: -the `GitTarget`, the `WatchRule`, the branch, the folder. The key names the *write that happened in -Kubernetes*, and it says nothing about which consumer is interested. So when five `GitTarget`s -mirror the same `Deployment` into five repositories, the API server posts one audit event, -ingestion stores one fact, and all five watch shards join that same key independently and each -stamps the same author on its own commit. Adding a sixth `GitTarget` adds no work on the write side -at all. - -That property is what makes the audit route the correct partition rather than the provider name, -which is the whole argument of -[`attribution-fact-identity.md`](attribution-fact-identity.md): several `ClusterProvider`s naming -one physical cluster deliberately share one route so they share its facts. Fan-in on a shared key -is what the schema is for. - -Both properties survive every option below, and the push option in -[option D](#option-d-publish-from-the-audit-receiver) actively exploits the second -one: a single notification on a shared key wakes every waiter on it, for the same reason a single -`GET` on that key serves every reader. - -## Why a wait is unavoidable - -The author fact and the watch event are two independent deliveries out of the same kube-apiserver. -One travels the audit webhook backend as an HTTP POST to -[`audit_handler.go`](../../internal/webhook/audit_handler.go#L312); the other travels the watch -stream into [`routeLiveTargetWatchEvent`](../../internal/watch/target_watch.go#L692). Nothing -orders them against each other, and the watch event frequently wins. - -The wait also cannot be deferred until after routing, which is the obvious alternative. The branch -worker groups events into a commit window keyed by author: -[`canAppend`](../../internal/git/branch_worker.go#L730) force-finalizes the open window as soon as -the incoming author differs from the window's. An event routed with the author still unknown would -either split the window immediately or need its author patched in afterward, and patching it means -rewriting a commit that may already be pushed. Waiting briefly before shipping is what makes "a late -audit arrival must not rewrite a shipped commit" enforceable, which is exactly what the constant -says it is for -([`DefaultAttributionGraceWindow`](../../internal/watch/author_resolver.go#L21)). - -So the three-second grace stays. Only the mechanism inside it is open. - -Ordering between events is a separate question and it is already settled in -[`watch-event-ordering-and-attribution-grace.md`](../facts/watch-event-ordering-and-attribution-grace.md): -the wait cannot reorder the events of one object or one type, because each -`(GitTarget, GVR, scope)` watch is a single goroutine that finishes one event before reading the -next. Read that record before touching option F, which is the non-blocking variant it sketches. - -## Which one fires first - -The watch event, nearly always, and the margin is not small. This is structural rather than -incidental, and it is the single most important input to the options below. - -The watch is a streamed push: the apiserver writes the event to an open connection as soon as the -write commits. The audit event is **batched**. `--audit-webhook-mode=batch` is what the -[attribution setup guide](../attribution-setup-guide.md) tells operators to configure, and batch -mode holds events until either `--audit-webhook-batch-max-size` events have accumulated or -`--audit-webhook-batch-max-wait` elapses. The batching parameters belong to the apiserver, not to -this operator, so the delay is not ours to remove. - -The consequence for the resolver: **the first `GET` in `ResolveAuthor` is a near-guaranteed miss**. -The poll loop is not an exceptional path taken under load, it is the normal path taken by every -attributable event. That inverts the usual instinct about a retry loop, where the first attempt -usually succeeds and the retries are the rare case. - -[`configuration.md`](../configuration.md) already records the operational half of this as a warning -on `--author-attribution-grace`: the apiserver's `--audit-webhook-batch-max-wait` delays every fact -by up to that much, so a grace at or below it loses actors systematically. Worth checking against -the upstream default for that flag, which is considerably larger than the one-second value the e2e -cluster uses, and larger than this operator's three-second default grace. There is no Kubernetes -checkout under `external-sources/` to source-verify it here, so treat it as a number to confirm -before it is written down as a fact. - -## What has already been measured - -Two measurements exist in the repository today, and they agree. - -**The e2e suite reports the wait distribution on every run.** -[`reportAttributionStats`](../../test/e2e/e2e_suite_test.go) queries -`gitopsreverser_attribution_resolutions_total` by `tier` and prints the -`gitopsreverser_attribution_resolution_wait_seconds` histogram, split into resolved and absent -because the two populations answer different questions. It also prints how many resolutions -succeeded only because e2e widens the grace past the three-second default, which is a direct -measure of how much headroom the flag buys. - -The finding is written into the comment above it: the e2e cluster runs -`--audit-webhook-batch-max-wait=1s` ([`start-cluster.sh`](../../test/e2e/cluster/start-cluster.sh)), -and a healthy run puts most waits in the 0.5 to 2 second range. Fact delivery is bounded below by -the batch window, exactly as the section above predicts. - -**The corpus records which events have no audit event at all.** Several rows of the mutation-capture -lab corpus are documented silences rather than gaps: a status subresource update produces two watch -events and **no** audit event, and a graceful pod delete produces watch `MODIFIED` plus `DELETED` -and **no** audit event ([`test/mutationlab/README.md`](../../test/mutationlab/README.md), rows 5 and -7). Those events can never resolve no matter how long the resolver waits. They are a structural -population that burns the entire grace window and then ships unresolved, which is a stronger version -of the problem [option C](#option-c-circuit-break-a-route-that-has-never-resolved-anything) -addresses per route. - -## What the mutation-capture lab can add - -The lab is the right instrument for the part the Prometheus histogram cannot answer, and it needs -only a small addition. - -It already has what matters. Every recorder stamps `ObservedAt: time.Now()` on its record -([`record.go`](../../internal/mutationlab/record.go), and the four recorders under -[`internal/mutationlab/recorder/`](../../internal/mutationlab/recorder/)), all in one process on one -clock, and every record carries an `ObjectKey` with the uid and resourceVersion that correlate the -watch event with the audit event for the same write. The skew this whole design is about is -therefore already captured on every lab run. It is discarded on the way to the corpus, because the -normalizer replaces timestamps with `` so the golden files stay deterministic. - -So the addition is a **separate, non-golden timing report**: per scenario, the delta between the -audit record's `ObservedAt` and the watch record's `ObservedAt` for the same `(uid, rv)`. It must -not enter `corpus/`, which exists precisely to be byte-stable across runs. - -What that buys over the aggregate histogram: - -- **Per-scenario skew instead of one blended distribution.** A create, a server-side apply, and a - finalizer delete may sit in different places relative to the batch window, and the Prometheus - histogram blends them. -- **The sign of the skew, confirmed per scenario** rather than inferred. If any scenario ever shows - the audit record arriving first, that is worth knowing, and no current measurement would show it. -- **The structural silences priced.** Rows 5 and 7 have no audit record to subtract, so they appear - as "no fact, ever" rather than as a large delta. That distinction is exactly the one - `AttributionAbsent` cannot make today, since an aged-out fact and a fact that never existed are - indistinguishable by design. -- **A regression signal on a Kubernetes upgrade.** The corpus is already the behavioral changelog - for a version bump. A timing report alongside it would show a batching or delivery change in the - same review. - -The honest limits: the lab and the product deliver differently enough that the absolute numbers do -not transfer. The lab records at handler entry rather than after ingestion, it runs on the e2e -cluster's deliberately aggressive batch profile (which -[`audit-webhook-api-server-connectivity.md`](../facts/audit-webhook-api-server-connectivity.md) -states is a test feedback optimization and not production advice), and it does not write to Redis at -all. It answers "which one fires first, and by roughly how much, per scenario". It does not predict -a production install's numbers, and the flag values below should still come from the e2e histogram -and from real installs. - -## What the wait costs today - -[`ResolveAuthor`](../../internal/watch/author_resolver.go#L160) loops: look up, and if the result is -`AttributionAbsent`, sleep `attributionPollInterval` (150ms) and look up again until the grace -expires. - -Each iteration is not one Redis round trip. -`LookupAuthorResolution` tries up to three keys: -the immutable exact key, the `:last` pointer for removals, and the type-scoped rv-only hatch. So a -single event that never resolves costs roughly 20 wakeups and 40 to 60 `GET`s, and an event whose -fact lands mid-grace pays an average of 75ms of pure poll-interval latency on top of however long -the fact took to arrive. - -The larger cost is where the loop runs. -[`attachAuthor`](../../internal/watch/target_watch.go#L729) is called inline in the shard's event -loop, before `RouteToGitTargetEventStream` and before -[`recordTargetWatchCursor`](../../internal/watch/target_watch.go#L689). The grace is not a -background wait. It head-of-line blocks that entire `(GitTarget, GVR, namespace)` shard, so a burst -of 50 unattributed events on one shard serializes into 150 seconds during which the shard reads -nothing from its watch channel. - -```mermaid -flowchart TB - subgraph SRC["Source cluster · one kube-apiserver"] - API["Kubernetes API
write to ConfigMap web
uid U · rv 101"] - end - - subgraph OP["Operator"] - AH["audit_handler.go
RecordFact"] - WS["target_watch.go
watch shard goroutine"] - AR["author_resolver.go
ResolveAuthor"] - BW["branch_worker.go
commit window keyed by author"] - end - - FACT[("route:prod-eu-1:configmaps
:object:U:101
author alice")] - - API -->|"audit POST · path 1 · BATCHED
held up to --audit-webhook-batch-max-wait"| AH - AH -->|SET with TTL| FACT - API -->|"watch event · path 2 · STREAMED
arrives first, nearly always"| WS - WS -->|"blocks the shard here"| AR - AR -.->|"GET, absent"| FACT - AR -.->|"sleep 150ms, GET again, up to 20 times"| FACT - AR -->|"author, or unresolved after 3s"| WS - WS --> BW - - OTHER["four more GitTargets
watching the same object"] -.->|"the same key, no extra write"| FACT - - classDef cost fill:#ffe6e6,stroke:#cc3333,color:#000; - classDef good fill:#e6f7e6,stroke:#33aa33,color:#000; - class AR cost; - class OTHER,FACT good; -``` - -The red box is the only thing under discussion. The green path is the fan-in property from -[the section above](#why-the-shape-of-this-architecture-is-worth-keeping), and it is not a problem -to solve. - -Two distinct problems hide in the red box, and they need separating because most options address -only one: - -1. **Wasted round trips and added latency** while waiting for a fact that will arrive. -2. **Head-of-line blocking** for the full three seconds when the fact will never arrive. - -## The options - -### Option A: measure, then leave it alone - -The resolver already emits everything needed to price this problem: -[`recordAttributionResolution`](../../internal/watch/author_resolver.go#L213) records -`AttributionResolutionsTotal` broken down by `result` and `AttributionResolutionWaitSeconds` as a -histogram, and the e2e suite already reads both. - -This was the natural first option before the batching floor was written down. It is much weaker now. -"The poll loop rarely runs a second iteration" was the outcome that would have justified stopping -here, and the measured 0.5 to 2 second waits rule it out: with audit delivery batched, roughly every -attributable event runs the loop to completion. - -What is still worth measuring is the *split*, which decides between the options rather than against -them: how much of the population resolves late (option D territory) against how much never resolves -at all (option C and the structural silences). - -### Option B: shift the first re-check to the delivery floor - -The original form of this option was a decaying backoff starting fast, on the theory that a fact -that is going to arrive arrives almost immediately. The batching floor says otherwise, so a -sequence like 25ms, 50ms, 100ms spends its cheapest attempts in the window where a fact provably -cannot have arrived yet. - -The corrected shape is the opposite: **wait before looking, then look often**. Skip the first -re-check until roughly the observed delivery floor, then poll on a short interval through the -window where facts actually land. - -The floor is not configured in this operator and cannot be read from the apiserver, so it has to be -learned. `routeAttributionHealth` already holds per-route state and is the natural place to keep a -cheap running estimate of that route's resolved-wait percentile, which the resolver then uses as -its first-check delay. A route whose facts land at 1.2 seconds stops paying for eight useless -lookups per event. - -Cost: still small, and contained inside the resolver. It is no longer the obvious first move it -looked like, because a learned first-check delay is most of the way to admitting that a -notification would be better. - -### Option C: circuit-break a route that has never resolved anything - -[`routeAttributionHealth`](../../internal/watch/author_resolver.go#L108) already tracks, per audit -route, whether attribution has ever resolved and how many events have gone unresolved since. Today -it is used for exactly one thing: logging a warning once, in -[`warnIfRouteNeverResolves`](../../internal/watch/author_resolver.go#L237). - -The same signal can gate the wait. Once a route crosses the unresolved streak threshold having never -resolved a single event, collapse its grace to a single lookup. A route nobody posts to currently -costs three seconds of shard blocking and dozens of `GET`s per event indefinitely; this turns it -into one `GET`, while the existing warning keeps telling the operator what to fix. - -The recovery rule matters: the streak counter is already cleared on any resolution -([`observe`](../../internal/watch/author_resolver.go#L119)), so a route that starts working restores -its full grace on the first fact it resolves. The breaker must therefore keep taking one real lookup -per event rather than skipping the lookup entirely, which it does. - -This is the only cheap option that fixes head-of-line blocking, and it fixes it exactly in the case -where the blocking is unbounded rather than incidental. - -The corpus silences argue for a second breaker beside it. A status subresource update and a graceful -pod delete produce no audit event at all, so those events pay the full grace on a route that is -otherwise healthy, which the per-route breaker will never trip on. A per-`(route, resource, -operation)` variant of the same counter would catch them. That is a larger change and it needs the -lab's per-scenario report to size it, which is one concrete reason to build the report first. - -### Option D: publish from the audit receiver - -Turn the wait from a poll into a push, driven by the code that receives the audit events. - -1. `writeFactKeys` pipelines a `PUBLISH` of each - written key onto a per-route channel alongside its `SET`. Pipelined, so the write side pays no - extra round trip. See [where to publish from](#where-to-publish-from) for the better placement. -2. The resolver process holds one long-lived subscription per audit route. Routes are bounded by - the number of `ClusterProvider`s, so this is a handful of connections for the whole process. The - subscriber fans out into a `key -> waiters` registry. -3. `ResolveAuthor` registers its waiters **before** performing the existing lookup, then selects on - the notification, `ctx.Done()`, and the grace deadline. -4. A coarse re-check stays as a safety net, for example one at the deadline. - -Point 3 is the entire correctness argument. Registering after the lookup loses any fact published -in the gap between the two, which is precisely the window that matters. Point 4 is the other half: -Redis pub/sub is at-most-once and drops messages on reconnect or failover, so the fallback ensures a -lost notification degrades to today's behavior rather than to a hang. - -The transport is available without changing anything else. -[`NewRedisStore`](../../internal/queue/redis_store.go#L56) builds a plain `redis.NewClient`, not a -cluster client, so ordinary `PUBLISH` and `SUBSCRIBE` apply with no shard-routing caveat. Writer and -reader are frequently different pods (the audit POST arrives through a Service while the watch shard -lives wherever it lives), which is why an in-process channel cannot substitute. - -This option composes with the fan-in property: one publish on a shared key wakes all five waiting -`GitTarget`s at once, for the same reason one `GET` served all five before. - -Cost: a subscriber lifecycle to own and reconnect, a waiter registry, and a new dependency on a -Redis feature the deployment has not used so far. Benefit: round trips per unresolved event drop -from dozens to roughly zero, and resolution latency for a late fact drops from 75ms of poll jitter -to about one round trip. - -The batching floor is what promotes this option. When the loop was assumed to exit on its first or -second lookup, a subscriber lifecycle was a lot of machinery for a rare path. With delivery batched, -every attributable event runs the loop for most of a second or more, so the dozens of saved round -trips are the steady state rather than the exception. This is where the wasted work sits. - -It does **not** fix head-of-line blocking. An event whose fact never arrives still holds its shard -for the full grace, and it still holds it for the delivery latency when the fact does arrive. - -#### Why the batching floor also makes the push reliable - -Publish and subscribe has one structural weakness: it is at-most-once, and a message only reaches -subscribers that are subscribed **at the moment it is published**. A late subscriber gets nothing. -That is normally the reason to be careful with it. - -Here that weakness is mostly canceled by the ordering measured in -[which one fires first](#which-one-fires-first). The watch event arrives first, by roughly the batch -window, and the resolver blocks inline the moment it arrives. So by the time the audit POST is -received and the fact is published, **the waiter is already waiting**. The one arrival order that -would defeat the push is the one the apiserver's batching makes rare. - -This is the argument that promotes the push from "a caching trick" to the natural shape of the -system. The push is aimed at exactly the window the measurements say the events land in. - -The residual cases are real and are what Redis stays for: an audit event that overtakes its watch -event anyway, a watch shard that starts after the fact was published, a replica whose subscription -was reconnecting, and a `GitTarget` that begins watching a type mid-grace. Each of those is a -late-join, and a late-join is precisely what a TTL'd key in Redis serves. - -#### Carry the fact, not a pointer to it - -The step above publishes the *key* and has the woken resolver `GET` it. Publishing the -`AuthorFact` itself is better, and it is barely -more work. - -- **The woken resolver needs no Redis read at all.** The message carries the author, display name, - and email, which is everything [`userInfoForResolution`](../../internal/watch/author_resolver.go#L198) - reads. In the common case the join costs zero round trips rather than one. -- **It removes an ordering constraint on the write side.** A notify-only publish must happen - strictly *after* the `SET`, or the woken reader races back to a key that is not there yet. A - payload-carrying publish has no read-after-write dependency, so it can be issued as early as the - receiver likes. -- **A fact is small.** The struct is a dozen short string fields, and only accepted, mutating, - author-bearing events produce one. - -The cost is that a message is broadcast to every subscriber on the route, so every replica receives -every fact whether or not it is waiting for that object, where today each replica reads only the -keys it wants. The comparison is broadcast volume (audit event rate times replica count) against -saved lookups (watch event rate times polls per event). With polls per event in the high single -digits and replica counts small, the push still wins comfortably. It stops winning at a high replica -count, which is the number to watch if this is built. - -#### Where to publish from - -Two placements, and the receiver is the better one. - -`writeFactKeys` knows exactly which keys it wrote, -which makes it the obvious home for a notify-only publish. But it sees one fact at a time, so a -`deletecollection` expanding into N facts -(`storeDeleteCollectionFacts`) becomes N publishes. - -The receiver sees the whole batch. One audit POST carries an `EventList` of up to -`--audit-webhook-batch-max-size` events, decoded once in -[`serveEventListRequest`](../../internal/webhook/audit_handler.go#L186) and looped through -[`processEvent`](../../internal/webhook/audit_handler.go#L289). Publishing there means **one message -per POST carrying every fact from that batch**, which collapses a batch of 400 events into a single -publish instead of 400. The batching that causes the delay is the same batching that makes the -notification cheap. - -The constraint on that placement is that the receiver must publish only what -`RecordFact` stored. That function drops -events with no `objectRef`, no resolvable name, or no user, and expands a `deletecollection` into -per-item facts. Publishing the raw `EventList` would notify waiters about facts that do not exist -and cannot name anyone. So `RecordFact` needs to return what it wrote, and the receiver accumulates -those across the batch and publishes once. That return value is the only real change to existing -code. - -### Option E: keyspace notifications instead of an explicit publish - -Redis can emit `__keyevent@__:set` notifications by itself, giving option D's push without -touching the write path. - -Rejected as the primary mechanism, for three reasons. It requires `notify-keyspace-events` to be -enabled server side, which a managed Redis or Valkey may not permit and which the chart does not -control. It is unfiltered: a process-wide subscription receives every `SET` on the connection, -including watch cursors and command-author records, to be discarded by prefix on the client. And it -can only ever announce a key, never a fact, so it forfeits the zero-lookup join that -[carrying the payload](#carry-the-fact-not-a-pointer-to-it) buys. - -Publishing from the receiver is the same idea with none of those three costs, because the receiver -already holds the decoded events and knows which facts it stored. Keyspace notifications are worth -keeping in mind only if the write side ever needs to stay on an older version than the read side. - -### Option F: move the wait off the shard's critical path - -Buffer the event, let the shard continue reading its watch channel, and resolve attribution in a -separate stage. - -This is not a new idea and it should not be redesigned from scratch. The final section of -[`watch-event-ordering-and-attribution-grace.md`](../facts/watch-event-ordering-and-attribution-grace.md) -already specifies it: a per-`(GVR, scope)` ordered pipeline that starts the lookup immediately and -holds an event until its grace expires **or** every earlier event on that watch has shipped, using a -sequence-numbered reassembly buffer. That record also carries the warning that makes the design -non-optional rather than a nicety: parallelizing attribution per event *without* the reassembly -buffer lets a fast-matched later event overtake a slow earlier one on the same object, which breaks -same-object ordering and can let an older mutation win. - -That record deferred it explicitly, "until measurements say otherwise". The batching floor is the -measurement. The head-of-line stall is not an occasional cost paid by unattributable events, it is -paid by every attributable event, for the length of the apiserver's batch window, on a -single-threaded shard. Under the upstream default `--audit-webhook-batch-max-wait` rather than the -e2e cluster's one second, that is a serious throughput ceiling on a busy type. - -The cursor is recorded after routing -([`processLiveTargetWatchEvent`](../../internal/watch/target_watch.go#L671)), so the reassembly -buffer has to preserve that too: a cursor may only advance for an event that has shipped, -otherwise a restart skips the events still held in the buffer. - -Still the largest change here, and still not the first thing to build. It is now the structural fix -rather than a curiosity, so it should be repriced once the split from option A is known. - -## What the push does to commit order - -Nothing, and that is the point. It then improves commit fidelity as a side effect. - -Ordering is not a property of the attribution mechanism. It is a property of the watch shard, which -is one goroutine per `(GitTarget, GVR, scope)` that finishes one event, wait included, before it -reads the next one from the channel, and of the single FIFO `eventQueue` the branch worker drains. -That chain is what -[`watch-event-ordering-and-attribution-grace.md`](../facts/watch-event-ordering-and-attribution-grace.md) -establishes, and none of options B through E touches any part of it. Attribution decides *what -author is stamped* on an event. It never decides *when the event is routed relative to its -siblings*. A push shortens a wait that happens inside an already-serialized loop. - -The side effect is a real improvement, and it works in favor of commits that read like the sequence -of changes that happened. - -An open commit window accepts one `(author, GitTarget)` pair at a time, so -[`canAppend`](../../internal/git/branch_worker.go#L730) finalizes the window whenever the incoming -author differs. Today an event whose fact arrives a moment after the grace expires ships as -attribution-unresolved while the event on either side of it resolves to a real actor. That single -timeout splits one window into three commits, and the split carries no meaning: the three events -were one person's change, and the middle commit is attributed to nobody because a batch was a few -hundred milliseconds late. Resolving from a push instead of from a race against a deadline removes -most of that class. Same order, fewer commits, and each one attributed to the actor who caused it. - -**The one change that would break order** is worth stating plainly next to a design that makes it -tempting. Once waits become short, resolving several events concurrently to reclaim the shard looks -free. It is not: a fast-matched later event would overtake a slow earlier one on the same object, -and the older mutation would win in Git. That is the explicit warning in the ordering record, and it -is why [option F](#option-f-move-the-wait-off-the-shards-critical-path) is specified with a -sequence-numbered reassembly buffer rather than as plain concurrency. Push the notification, keep -the shard serial. - -## Comparison - -| Option | Fixes round trips | Fixes latency | Fixes shard blocking | Size | -|---|---|---|---|---| -| A: measure only | no | no | no | none | -| B: first check at the delivery floor | mostly | no | no | one function | -| C: route circuit breaker | for a dead route | for a dead route | for a dead route | small | -| D: publish from the receiver | yes, to zero | yes | no | moderate, new subsystem | -| E: keyspace notifications | to one lookup | yes | no | moderate, plus a config dependency | -| F: reassembly buffer off the hot path | no | no | yes | large | - -Note what the second column now says about B. Shifting the first check to the delivery floor saves -lookups but cannot make a fact arrive sooner, so it does not improve latency. Only a notification -does, and only F removes the stall. - -## Recommended sequence - -1. **Build the lab timing report.** It is the smallest piece of work here, it produces the - per-scenario number that prices everything else, and it turns a Kubernetes upgrade's timing - change into something the corpus review would catch. -2. **Read the e2e histogram for the split**, resolved-late against never-resolved. Those two - populations point at different options and the aggregate total does not separate them. -3. **C, on its own.** Small, uses state that already exists, and it is the only cheap fix for the - unbounded case. Ship it regardless of what the numbers say. -4. **Then D or F, decided by the split.** A large resolved-late population makes D the answer: - most of that wait is real delivery latency and a notification collects it at the earliest - possible moment. A large never-resolved population, especially concentrated in the structural - silences, makes F the answer: no notification is ever coming for those events, so the only - remaining fix is to stop blocking the shard while waiting for them. -5. **B as a fallback**, if neither D nor F is affordable soon. It reduces the waste without - addressing either underlying problem. - -D and F are complementary rather than alternatives, and building D first is the better order. D -shrinks the wait for every event that has a fact, which shrinks the population F has to hold in its -buffer, and it leaves the shard serial while it does so. F then handles what is left, which is the -events that have no fact at all. - -The revision that matters relative to the first draft of this record: the batching floor moves the -work from "tune the loop" toward "stop looping", and it makes F structural rather than optional. -Both conclusions came from measurements that already existed in the repository. The receiver-side -publish then falls out of the same finding, because the arrival order that makes the poll wasteful -is the arrival order that makes the push land on a waiter that is already there. diff --git a/docs/design/docs-linting.md b/docs/design/docs-linting.md index e8866701..a3c0b073 100644 --- a/docs/design/docs-linting.md +++ b/docs/design/docs-linting.md @@ -128,7 +128,7 @@ The committed configuration brings that to **723**. Running `markdownlint-cli2 - them, leaving **257**: 119 `MD013` plus **138 that need a human**. That is the entire manual backlog for structure, and it is concentrated in a handful of files, led by `docs/architecture.md`, `docs/spec/gitpath-foreign-content-stringency.md`, and -`docs/spec/deletecollection-attribution-expander.md`. +`docs/spec/attribution.md`. Three configuration choices needed a decision rather than an inference. diff --git a/docs/design/gittarget-api-wave.md b/docs/design/gittarget-api-wave.md new file mode 100644 index 00000000..b8a331df --- /dev/null +++ b/docs/design/gittarget-api-wave.md @@ -0,0 +1,274 @@ +# One breaking wave: the folder is described on the GitTarget + +> **design**: a sequencing proposal, not a plan of record. Nothing here binds until scheduled. +> Index: [`../INDEX.md`](../INDEX.md) +> Date: 2026-07-30. +> +> Combines three pieces of work that are all `feat(api)!` on `GitTarget` and are cheaper together +> than apart: +> +> - the layout model, [`gittarget-layout-model.md`](gittarget-layout-model.md); +> - the still-open API-surface block of the maintainer review, +> [`flux-maintainer-review-status-and-config-model.md`](../future/flux-maintainer-review-status-and-config-model.md) +> §4 "Then": **F6**, **F9**, **F10**, plus F12's reference-type nit and §3's pushbacks; +> - the Tier 2 breaking items in [`open-asks-priority.md`](open-asks-priority.md): **B4**, **B1**, +> **#5**, **#6**. +> +> It does not touch Tier 1 (the removal-wait decision, #15's condition). Those are not breaking and +> should not wait for this. + +## Why one wave + +The consumer pins us three ways (image, Go module, `require` line), so each breaking release costs a +coordinated bump. That argues for batching. It is the weaker half of the argument. + +The stronger half: **four of these items are the same design decision seen from different angles**, +and building them separately means deciding it four times, inconsistently. + +> **The folder is described on the GitTarget. The connection describes only the connection.** + +- `spec.layout` says what the folder **is** (layout model). +- `spec.mode: Observe|Write` says whether we write to it at all (B1). +- `spec.suspend` says whether we write to it *now* (F6). +- `commitWindow` and `commit.message` say how writes to it are batched and phrased, and today they + live on `GitProvider`, which is the connection (B4, and §3's "GitProvider is doing three jobs"). + +Ship `spec.layout` alone and the principle is asserted by one field while `commitWindow` still +contradicts it. Ship them together and the object reads as one idea: a GitTarget is a folder plus a +policy for writing it, and a GitProvider is how you reach the repository. + +## The interactions that change the design + +These are the reasons to combine, as opposed to merely batch. Each one changes what gets built. + +### 1. `spec.mode: Observe` is how a layout is adopted safely + +The layout model's weakest point is adoption: a user pointing `kind: Kustomize` at a real repository +has to trust it before any file moves, and placement only ever affects *new* documents, so there is +nothing to preview by inspection. + +`Observe` mode plus `status.layout` is that preview. In `Observe` the operator scans, resolves the +layout, publishes `renderRoot`, `kind`, and what it *would* do, and writes nothing. Flip to `Write` +when the status says what you expected. That turns "declare a layout and hope" into a dry run, and +it costs nothing extra because both halves are already in the wave. + +This also gives `Observe` a purpose beyond "a safety switch nobody uses". It is the mode you adopt a +repository in. + +### 2. `spec.interval` is what keeps the layout observation fresh + +The layout status has two halves ([the other document](placement-visibility-and-declared-defaults.md) +records why): a **current** half derived from the last repository scan, and a **historical** half +accumulated since. The current half is the useful one, and it has a hole: a repository scan happens +on a write or a resync, so a stable target that writes nothing may not scan for a long time, and the +field a user consults would be stamped with a revision from last week. + +`spec.interval` (F6) plus `Observe`'s scan-without-writing is the mechanism that closes it: a +periodic observation pass refreshes `renderRoot` and `observedRevision` whether or not anything was +written. Neither piece was proposed for this reason, and together they answer a question neither +answers alone. + +### 3. `spec.suspend` is a precondition for a layout that creates files + +`kind: Kustomize` with `create: true` writes a `kustomization.yaml` the user did not author. That is +the right behavior and it raises the stakes on the review's central complaint (F6): *this controller +writes to a Git repository and there is no way to make it stop that is not deleting the object.* + +So `suspend` is not a rider here, it is a precondition. A layout that creates structure must ship +with the button that stops it. And `suspend` must stop bootstrap creation specifically, not only +resource writes, which is a detail worth stating before either is built. + +### 4. The Events question is already answered, and the layout is what to say + +[`open-asks-priority.md`](open-asks-priority.md) left one thing open about the inference deletion: +whether a fall-back to canonical should raise an Event on the GitTarget, and it reasoned that this +was expensive because placement runs on the branch worker with no recorder. + +That is no longer true. **F7 shipped an `EventRecorder` on every reconciler** +(review §6), and the roll-up seam projects data-plane facts into status with an enqueue on change. So +the Event is now: emit when `status.layout` changes in a way a human should know about, which is +`renderRootReason` becoming `Ambiguous`, or a type falling back for the first time. One Event per +persisted change, the pattern F7 already established for `Ready`. + +### 5. Layout is immutable, which puts it with `path` rather than with `prune` + +`providerRef`, `branch`, `path` and `clusterProviderRef` are immutable because a folder's meaning is +constituted by them (review §3 defends this well). `spec.prune` is deliberately mutable because +freezing it would destroy the one thing that cannot be rebuilt. + +An earlier draft of this document put `spec.layout` with `prune`. That was wrong, and the reason is a +fact worth checking before designing around it: **`GitTarget` has no finalizer**, so deleting one +leaves the folder in Git untouched, and re-creating it at the same path re-adopts every document by +identity. Changing a layout by recreating the object costs status and a moment of mirroring, not data, +which is a different bargain from `prune` entirely. Meanwhile a mutable layout leaves a folder +permanently half one structure and half another, because existing files never move and nothing records +which file came from which. + +So: immutable, with a CEL exception for a **widening** transition (`Flat` to `Tree`) that cannot lose +the identity-completeness the folder already had. And `Auto` resolves once and is pinned in status, +because immutability of a field that says "look at the folder" pins nothing. The reasoning is in +[`gittarget-layout-model.md`](gittarget-layout-model.md). + +This settles a question **#6** would otherwise have to reopen: if `path` ever becomes movable +(`status.observedDestination`), the layout moves with it, because a new folder may have a different +structure, and an immutable layout means that transition is one deliberate act rather than two +independent edits. + +### 6. Namespace scope makes `allowedSourceNamespaces` and the layout answer to each other + +The layout's `scope: SingleNamespace` is a structural claim, and `spec.allowedSourceNamespaces` is an +authorization bound. They are different questions about the same folder, and after this wave they are +checked against each other at admission: a matcher admitting more than one namespace beside a +single-namespace layout is refused. + +This is the connection the wave makes available, and it is not one either item asks for alone. +Authorization already exists and already has a two-party delegation the review praises; the layout is +what turns "who may write here" into "and therefore what this folder looks like", including whether +`metadata.namespace` is written into the files at all. The namespace-in-file question is inference +today, and it is the one piece of inference an empty folder cannot perform, which is why bootstrapping +needs it declared. + +## The GitTarget after the wave + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: prod + annotations: + reconcile.configbutler.ai/requestedAt: "2026-07-30T09:14:22Z" # F6 +spec: + # --- the connection: unchanged, and now only the connection --- + providerRef: + name: platform + branch: main + path: clusters/prod + + # --- what the folder is --- + layout: # immutable, except a widening transition + kind: Kustomize # Auto | Kustomize | Tree | Flat | Template + scope: SingleNamespace # must agree with allowedSourceNamespaces + writeNamespace: Never # the created kustomization carries namespace: + kustomize: + create: true + byType: + v1/secrets: "secrets/{name}{sensitiveSuffix}" + + # --- whether and when we write it --- + mode: Write # B1: Observe | Write + suspend: false # F6 + interval: 5m # F6, and what keeps status.layout fresh + prune: + mode: OnEvent + + # --- how writes are batched and phrased: moved off the connection --- + commitWindow: 5s # B4, was GitProvider.spec.push.commitWindow + commit: + message: + template: "chore(mirror): {{ .Summary }}" # B4, was GitProvider.spec.commit.message +status: + layout: + declaredKind: Kustomize + kind: Kustomize + renderRoot: . + renderRootReason: SingleKustomization + observedRevision: 9f3c1ab + observedTime: "2026-07-30T09:14:22Z" + placedResources: 14 + refusedResources: 0 + lastHandledReconcileAt: "2026-07-30T09:14:22Z" # F6 + observedDestination: # 6 + branch: main + path: clusters/prod +``` + +Read top to bottom it is one story: reach this repository, this is what the folder is, this is whether +and when we write it, this is how the writes look. + +## What rides along without a claim of synergy + +Honesty matters more than a tidy narrative. These are in the wave because they are breaking and the +consumer should pay once, not because they interact with the layout: + +- **#5, `CommitRequest.spec.author`, SAR-guarded.** Independent, and it stands on the argument that + attribution needs an audit webhook a hosted control plane will not give you. +- **F10, CommitRequest lifecycle** (`ttlSecondsAfterFinished` or an `ownerReference`, plus the + `delete` verb). Unrelated to placement; it is the other object in the API with a lifecycle hole. +- **F12's reference types**: embedding `meta.LocalObjectReference` for the name half of our six + near-identical reference shapes. If GitTarget is breaking anyway, this is the moment. +- **§3's `TooManyStreams` cap** for `sourceNamespace: "*"` fan-out. A `Stalled` reason plus a bound, + rather than discovering the cliff as apiserver watch pressure. +- **§3's `ClusterProvider` "default" message.** One error string, and the most likely first-run + support ticket. + +**F9** is not in the wave at all: it is one envtest against the minimum supported Kubernetes version, +and its outcome (widen the enum, or keep it) should be known *before* anyone plans an API change +around it. + +## Version strategy: stay `v1alpha3` + +A wave this size invites `v1alpha4`, and I would not take it. + +- A new version means a **conversion path**, and the honest options are a conversion webhook (a + serving dependency for the CRD, plus a cert lifecycle) or `None` conversion with a stored-version + migration. Both cost more than the problem. +- We are `v1alpha3` and pre-1.0 with **one consumer**. The convention the repo already uses is a + **loud rejection**: keep the removed field in the schema, refuse it with a message naming the + replacement, for one release. `ClusterWatchRule.spec.rules[].scope` set that precedent, and the + reasoning holds better here: refusing a stored field the user can see beats translating it behind + their back. +- `spec.placement` therefore becomes a rejection that says "use `spec.layout`; `byType` moves + verbatim, `default` becomes `layout.kind: Template`". The mapping is mechanical, which is what makes + the rejection kind rather than merely strict. + +If a second consumer appears before this ships, revisit: the calculus that makes loud rejection cheap +is one coordinated bump. + +## Order inside the wave + +Dependencies first, then the things that only need the object to be breaking. + +1. **F9's envtest.** Not an API change; its answer constrains the enum work. Do it before planning. +2. **`spec.suspend`** (F6). Precondition for anything that creates files, and independently the + review's highest-value gap. +3. **`spec.layout`** with `byType`, plus rule 1 (every written file is registered with the + kustomization that governs it) and rule 2 (a structural kind excludes a blanket `default`). + `spec.placement` becomes a loud rejection. +4. **`status.layout`**, current half from the scan, historical half from the roll-up. Needed before + `Observe` is useful, because `Observe` with nothing to read is a mode that does nothing. +5. **`spec.mode: Observe|Write`** (B1). Now an adoption path rather than a switch. +6. **`spec.interval` + `requestedAt` + `lastHandledReconcileAt`** (F6, rest). Closes the freshness + hole in step 4 and gives the object the reflexes a Flux user already has. +7. **Events on layout change**, over F7's existing recorder. +8. **B4**: `commitWindow` and `commit.message` move from `GitProvider` to `GitTarget`. Last of the + principle items, and the one that makes the object coherent. +9. **The riders**: #5, F10, F12 references, `TooManyStreams`, the `default` ClusterProvider message. + +Steps 2 to 8 are one release. Step 1 gates the planning. Step 9 can be trimmed if the wave gets too +big to review, since nothing else depends on it. + +## What this costs, stated plainly + +- **One coordinated consumer bump**, with a mechanical migration for every field: `placement.byType` + moves verbatim, `placement.default` becomes `kind: Template`, `commitWindow` and `commit.message` + move object, everything else is additive. +- **One `docs/UPGRADING.md` entry** covering the field moves, the layout mapping, and the one real + behavior change: a declared template stops silently disabling the render root and starts + registering its files. +- **A larger review surface than any change this project has taken.** That is the argument for + trimming step 9 first and for keeping step 1 outside the wave. + +## Open questions + +- **Does `mode: Observe` write status only, or also refuse admission of new WatchRules?** Observe + should be silent about everything except what it observed, but a user in Observe mode with rules + piling up may expect to be told nothing will happen. +- **Should `suspend` and `mode: Observe` be one field?** They are close: both stop writes. They differ + in intent (temporary versus declared) and in what they do to status, and Flux keeps `suspend` + separate from everything else. Two fields, but the field docs must each say what the other is for. +- **Where does `interval` live?** `GitProvider` needs it for `ls-remote` cadence (review F6); + `GitTarget` needs it for the observation pass. Two fields with one name on two objects is a smell, + and one field on the provider cannot express a per-folder observation cadence. +- **Is `Auto` still the right layout default once `Observe` exists?** With a dry-run mode available, + requiring an explicit `kind` costs the user much less than it would have, and it would make every + target's layout self-evident. diff --git a/docs/design/gittarget-layout-model.md b/docs/design/gittarget-layout-model.md new file mode 100644 index 00000000..7b475eed --- /dev/null +++ b/docs/design/gittarget-layout-model.md @@ -0,0 +1,525 @@ +# A layout is the declared thing, not a path + +> **design**: a proposal, not a plan of record. Nothing here binds until scheduled. +> Index: [`../INDEX.md`](../INDEX.md) +> Date: 2026-07-30. Supersedes Question 2 of +> [`placement-visibility-and-declared-defaults.md`](placement-visibility-and-declared-defaults.md), +> which argued about whether to default a path template. The answer is that a path template is the +> wrong primitive to be defaulting. +> +> This is a `feat(api)!` change to `GitTarget`. It does not fit in PR #291 and is not proposed for it. +> How it sequences with the other breaking work on the same object (`spec.suspend`, `spec.mode`, the +> `commitWindow` move, CommitRequest lifecycle) is in +> [`gittarget-api-wave.md`](gittarget-api-wave.md), which also records the two places where those +> items change this design rather than merely accompanying it. + +## Why the current shape keeps producing dead ends + +Placement today is a ladder of four rungs, three of which are path templates and one of which is not: + +```text +byType -> default -> the folder's one kustomize root -> canonical +``` + +Every unresolved question in the review of #291 traces to that one mismatch: + +- a CRD default for `placement.default` cannot be added, because a non-empty template consumes the + slot in front of the rung that is **not** a template (F9 in the other document); +- `byType: {v1/configmaps: "configmaps/{name}.yaml"}` in a kustomize folder produces a file no + kustomization lists, so it is committed and never rendered (F10). One line of user config; +- `placement.default` set on a kustomize folder does the same thing to every type at once, which is + the same bug with a bigger blast radius; +- "where do my files go" is answered by simulating a four-rung ladder against a folder, which is why + it needed a metric and a status field to be legible at all; +- nothing in the model can **create** structure, so a repository that needs a `kustomization.yaml` + before it can render cannot be bootstrapped by the thing that writes into it. + +The primitive is wrong. A path template cannot express "beside this folder's one kustomization", +cannot be read at a glance, and cannot bring a folder into existence. What a user wants to declare is +what the folder **is**. + +## The model + +One field, a discriminated union on `kind`, plus per-type overrides that are valid under every kind. + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +spec: + providerRef: + name: platform + branch: main + path: clusters/prod + layout: + kind: Kustomize # Auto | Kustomize | Tree | Flat | Template + scope: SingleNamespace # SingleNamespace | MultiNamespace + writeNamespace: Never # FromContext | Always | Never + kustomize: + root: . # relative to spec.path; where the kustomization.yaml lives + create: true # write it if absent, so an empty repo becomes a buildable folder + fileName: "{kindLower}-{name}" # optional; the default + byType: # optional overrides, valid under every kind + v1/secrets: "secrets/{name}{sensitiveSuffix}" +``` + +| `kind` | Where a new document goes | Exists because | +|---|---|---| +| `Auto` (default) | one supported kustomization in the subtree: `Kustomize`; else exactly one namespace in scope: `Flat`; else `Tree`. Resolved once and pinned (see below) | it is what most folders want, and it is honest about reading the repository | +| `Kustomize` | beside the declared root, registered in its `resources:` list | a file that root cannot reach is never applied | +| `Tree` | the built-in `{namespaceOrCluster}/{groupPath}/{resource}/{name}` path | identity-complete by construction; right for a fleet folder | +| `Flat` | `{kindLower}-{name}.yaml` at the root of `spec.path` | the legible single-namespace folder people hand-author | +| `Template` | the declared `default` template | the escape hatch. Everything expressible today stays expressible | + +Two rules complete it, and they are where the value is: + +1. **Whatever chose the path, the file must be reachable.** If any kustomization governs the + destination, the write registers the file in it, in the same commit. This is an invariant of the + layout rather than a special case of one rung, which is what turns F10 from a bug into something + the model cannot express. +2. **A structural kind and a blanket template are mutually exclusive.** `default` is valid only under + `kind: Template`. You cannot ask for a kustomize folder and a nested canonical tree at the same + time. + +## What the model makes unstatable + +This is the headline, and it is worth more than the readability. + +| Today | Under the model | +|---|---| +| `placement.default` on a kustomize folder silently disables the render root and produces unrendered files | unstatable: `default` requires `kind: Template`, and `Template` asserts there is no structural rule | +| `byType` into a subdirectory produces a file no kustomization lists | unstatable: registration is an invariant, not a rung | +| Defaulting the fallback would shadow the structural rung | dissolved: the default is `kind: Auto`, which **names** the structural rule instead of standing in front of it | +| "Which of four rungs answered?" needs a metric to be legible | one word in the spec, and status says what `Auto` resolved to | +| An empty repository cannot be given the structure it needs to render | `kind: Kustomize` with `create: true` | + +The defaulting problem is the interesting one. It was never that defaults are bad; it was that we +tried to default a **path**, and a path is the one thing that cannot say "look at the folder". +`kind: Auto` is a safe CRD default precisely because it is a mode: it declares that the folder will be +read, which is the difference between this and the sibling inference we deleted. That inference was +undeclared. This one has a name, appears in the spec, and reports what it resolved to. + +## Namespace scope belongs to the layout + +A folder that omits the namespace from its paths is a folder for **one** namespace. That is what makes +`Flat` legible, and it is an assumption the layout has to carry, because the thing that would otherwise +carry it lives somewhere else. + +```yaml +spec: + allowedSourceNamespaces: # AUTHORIZATION: who may be mirrored here + names: [team-a] + layout: + kind: Flat + scope: SingleNamespace # STRUCTURE: what shape this folder has +``` + +**These are two different questions and they must agree.** `allowedSourceNamespaces` is a permission +bound owned by the destination +([`gittarget_types.go`](../../api/v1alpha3/gittarget_types.go)); `layout.scope` is a structural claim. +Validation checks the direction it can, at admission: a matcher that admits more than one namespace +alongside `scope: SingleNamespace` is refused. + +**Why the layout cannot derive it.** Two reasons, and the second is the one that matters: + +- `allowedSourceNamespaces` is an upper bound and may be **absent**, which + [`NamespaceMatcher`](../../api/v1alpha3/namespace_matcher.go) defines as "no policy declared" rather + than as "one namespace". There is often nothing to derive from. +- the namespaces that do arrive come from **N WatchRule objects that do not own the folder**. A + derived single-namespace assumption could be invalidated later by a rule created in another object, + which would turn a layout guarantee into a path collision. Declaring the scope makes that + invalidation a **refusal** instead: a document from a second namespace is declined with a message + naming both namespaces and counted as a placement refusal, rather than landing on a path another + object already occupies. + +That is the same distinction the whole redesign rests on. Reading the world is fine when the reading +is declared and its failure is loud; it is not fine when it silently re-decides. + +## Whether the namespace is written into the file + +Today this is **inferred**, and it cannot be inferred in the case bootstrapping cares about most. +`namespaceIsInheritedFromContext` omits `metadata.namespace` exactly when the governing kustomization's +`namespace:` equals the resource's own. An empty folder has no kustomization to read, so a folder we +are about to create cannot inherit a convention that does not exist yet. + +```yaml +layout: + writeNamespace: FromContext # FromContext (default, today's behavior) | Always | Never +``` + +| Value | Meaning | When it is valid | +|---|---|---| +| `FromContext` | omit when the governing kustomization sets this resource's namespace | always; today's behavior | +| `Always` | always write `metadata.namespace` | always; the only safe choice when nothing downstream supplies it | +| `Never` | never write it | only when something guarantees the namespace: a kustomization we control with `namespace:` set, or a declared downstream supplier such as a Flux `Kustomization.spec.targetNamespace` | + +`Never` needs that guard because omitting the namespace hands the object to whatever namespace the +applier happens to be pointed at, which is a different object with the same name. + +**And this is where bootstrapping closes its own loop.** `kind: Kustomize` with `create: true` and +`scope: SingleNamespace` lets the operator write `namespace: team-a` into the `kustomization.yaml` it +creates, and then legitimately omit `metadata.namespace` from every file it places. The convention is +**established** rather than guessed, which is the thing inference structurally cannot do on an empty +folder. + +## The layout is immutable, with one widening exception + +An earlier draft of the wave document put `layout` with `prune` as a mutable field. That was wrong. + +**Existing files never move**, so a mutable `kind` leaves a folder that is permanently half one layout +and half another, with nothing in the folder recording which file came from which. The structure of a +folder should be a property of the folder, not of the last edit to an object. + +**The cost of immutability is lower than it looks, and this is the fact that decides it:** `GitTarget` +has **no finalizer**, so deleting one leaves the folder in Git untouched, and re-creating it at the same +path re-adopts every document by identity (match-first). Changing a layout by recreating the object +costs the object's status and a moment of mirroring. It does not cost data. That is a materially +different bargain from `spec.prune`, where the review's argument for mutability was that a +delete-and-recreate would destroy the one thing that cannot be rebuilt. + +**The exception is widening.** `Flat` to `Tree` cannot break the folder: old flat files stay and remain +match-first, new files get identity-complete paths, and nothing collides. Narrowing (`Tree` to `Flat`) +is what can put two namespaces' objects on one path. So the CEL rule is "immutable except a transition +that cannot lose the identity-completeness the folder already had", which in practice means you may +widen and may not narrow. + +### `Auto` resolves once and is then pinned + +Immutability of the *field* does not pin the *resolution*, because `Auto` says "look at the folder". If +someone deletes the `kustomization.yaml`, `Auto` would silently become `Tree` and the folder would grow +a second layout without any object changing. That is the defect this project spent a release deleting, +re-entering through the default value. + +So `Auto` resolves on first observation, `status.layout.kind` records what it became, and a later +folder state that would resolve differently raises a condition rather than re-laying-out the folder. +Declared inference is fine. Silent re-decision is not. + +That also settles the earlier open question about whether `Auto` should be the default at all: it can +be, because pinning removes the harm, and it keeps the quickstart to four fields. + +## Examples + +### 1. `Auto` on a brownfield kustomize repo + +No layout declared, so the CRD default applies. The subtree has one supported `kustomization.yaml`. + +```yaml +spec: + path: clusters/prod + # layout: {kind: Auto} <- defaulted, so the field is visible in the object +``` + +A new ConfigMap `cache` in `team-a` lands at `clusters/prod/configmap-cache.yaml` and is added to +`resources:`. `metadata.namespace` is omitted if and only if the kustomization's `namespace:` is +`team-a`. Status reports what `Auto` became: + +```yaml +status: + layout: + declaredKind: Auto + kind: Kustomize + renderRoot: . + renderRootReason: SingleKustomization + observedRevision: 9f3c1ab +``` + +### 2. `Kustomize` with `create: true` on an empty repository + +The bootstrapping case. The folder does not exist yet, so nothing can be inferred from it. + +```yaml +spec: + path: clusters/prod + layout: + kind: Kustomize + kustomize: + create: true +``` + +The first write commits a folder that builds, rather than a file that happens to be YAML: + +```text +clusters/prod/ + kustomization.yaml # created, listing what was written + configmap-cache.yaml +``` + +```yaml +# clusters/prod/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - configmap-cache.yaml +``` + +`kubectl apply -k clusters/prod` works on the first commit. This is the part the current model cannot +do at all, and it is the difference between mirroring into a directory and producing a GitOps folder. + +### 3. `Tree`, declared + +A fleet folder holding many namespaces, where a flat layout would collide names. + +```yaml +spec: + layout: + kind: Tree +``` + +`clusters/prod/team-a/apps/deployments/api.yaml`. Identity-complete by construction, so two +namespaces holding `api` never share a file. Naming it in the spec is the point: it is a choice now +rather than what is left when nothing matched. + +### 4. `Flat`, the layout you asked for by name + +```yaml +spec: + allowedSourceNamespaces: + names: [team-a] + layout: + kind: Flat + scope: SingleNamespace + writeNamespace: Always # nothing here supplies the namespace, so the file must carry it +``` + +`clusters/prod/deployment-simon.yaml`. Legible, and the reason it cannot be the built-in is that it is +**not identity-complete**: two namespaces with a Deployment named `simon` render one path. + +**`Flat` is also the kind that may not omit the namespace**, and this is the table above applied rather +than an exception to it. `Never` is legal only when something guarantees the namespace, and under +`kind: Flat` there is no kustomization for us to write `namespace:` into: a flat directory has no build +step at all. So the only guarantor left would be a downstream applier pointed at the right namespace, +which is a promise made outside this object and invisible to it. `Always` is the honest setting here. +An earlier draft of this example wrote `Never` with the comment "the build supplies it, or the applier +does", which is exactly the hand-wave the `Never` guard exists to refuse. + +As a *kind* rather than a template, that is checkable instead of a caveat in prose. `Flat` requires +`scope: SingleNamespace`, admission refuses it beside an `allowedSourceNamespaces` matcher that admits +more than one namespace, and a document arriving from a second namespace is refused at the write +boundary with a message naming both. A template can only document the hazard; a kind can decline it. + +### 5. `Template`, the escape hatch, with the invariant still in force + +```yaml +spec: + layout: + kind: Template + default: "{namespace}/{resource}.yaml" # a per-namespace bundle per type + byType: + v1/secrets: "secrets/{namespace}/{name}{sensitiveSuffix}" +``` + +Everything today's `placement` can express, this can express. The difference is rule 1: if a +kustomization governs `team-a/configmaps.yaml`, the file is registered in it. Today the same +declaration produces an unrendered file (F10) whenever the governing kustomization is an ancestor +rather than a sibling. + +### 6. `byType` under a structural kind + +Overrides are not exclusive to `Template`, because "kustomize folder, but Secrets in their own +directory" is an ordinary thing to want. + +```yaml +spec: + layout: + kind: Kustomize + byType: + v1/secrets: "secrets/{name}{sensitiveSuffix}" +``` + +A Secret goes to `clusters/prod/secrets/db.sops.yaml`, and rule 1 registers it in the root's +`resources:`. What is refused is a blanket `default` here, because that would be an assertion that the +folder has no structural rule alongside an assertion that it has one. + +### 7. Two roots, and the answer ambiguity has been missing + +```yaml +spec: + path: clusters # holds overlays/staging AND overlays/production + layout: + kind: Kustomize +``` + +The user asserted a single-root folder. The folder disagrees, so this is a **misconfiguration of the +GitTarget**, not a placement puzzle: `Validated=False`, naming both roots, with the fix being one +GitTarget per overlay. Under `kind: Auto` the same folder resolves to `Tree`, and status says +`renderRootReason: Ambiguous` so nobody has to guess why files stopped landing in the overlays. + +This is why the refuse-or-write question was so hard to settle in the current model: without a +declaration there was nothing to contradict. With one, refusing is not a policy preference, it is +honoring what the user said. + +## Bootstrapping, and where it stops + +`create: true` is deliberately narrow: **the layout may create only what its own invariant requires.** +For `Kustomize` that is exactly one file, the `kustomization.yaml` the layout claims exists, plus the +`resources:` entries for what we write. + +Everything else people mean by "bootstrap a GitOps repo" is a different concern: per-environment +directories, an app-of-apps root, a Flux `Kustomization`, a README, a `.sops.yaml`. The last two we +already write ([`bootstrapped_repo_template.go`](../../internal/git/bootstrapped_repo_template.go)), +which is a useful precedent and also a warning: that mechanism is per-path bootstrap staging, and it +is where a "repository template" belongs if we ever grow one. Folding a folder skeleton into +`spec.layout` would make the layout responsible for the shape of a repository it does not own. + +So the boundary is: the layout creates what it needs to be true. A repository template, if it ever +exists, is a separate object with a separate lifecycle. + +## Should the layout be its own CRD? + +The reuse instinct is real: a platform team with thirty GitTargets should not paste the same four +lines thirty times. Three shapes, and I would not build the second one yet. + +### The argument for a `LayoutProfile` + +- **Reuse.** One house layout, referenced by every target. +- **Guardrails.** A cluster-scoped profile owned by the platform team, referenced by namespaced + GitTargets, is an RBAC boundary: tenants pick a layout, they do not invent one. +- **One place to change it.** A fleet-wide layout change becomes one edit. + +### The argument against, which I find decisive today + +**A shared object that changes where N folders write, with nothing on the GitTarget recording it, is +structurally the same defect this project spent a release deleting.** Sibling inference was +removed because a human's edit to a repository changed the operator's behavior with no Kubernetes +object changing and nothing in status recording the move. A profile edited in another namespace is +that same shape with a different actor: the GitTarget that owns the folder is unchanged, unreviewed, +and yet its next new file lands somewhere else. Being an API object rather than a folder makes it +auditable, which is better, but it does not make it *local*, and locality is what made the deletion +worth doing. + +The rest is ordinary cost, and it is not small: + +- **A third place to look.** The redesign exists to make "where do my files go" answerable from the + object. A `layoutRef` re-splits the answer across two objects, plus status to reconcile them. +- **Another readiness chain.** `GitProvider` already teaches this: a missing or invalid reference is a + new `Ready=False` mode, and a profile deleted while targets reference it needs an answer (freeze the + last resolved layout, or stop writing). +- **Cross-namespace authorization, again.** A namespaced profile referenced across namespaces needs + what `ClusterProvider` needed: an `allowedNamespaces` selector and a fail-closed SAR. That was + expensive to build and is expensive to keep correct. +- **The thing being shared is four lines.** `kind: Kustomize` plus two options does not carry enough + weight to justify an object. The reuse pressure is concentrated entirely in one place: a large + `byType` map. That is worth remembering, because it means if we ever do share something, we should + share **the type map**, not the kind. +- **Reuse is already solved one layer up.** Whatever creates thirty GitTargets (Helm, kustomize, a + Flux `ResourceSet`) repeats four lines for free, and it does so in a place the user already reviews. + A CRD that exists to avoid repetition in generated YAML is solving a problem the generator does not + have. + +### The middle shape, if evidence demands it + +Keep the layout inline and authoritative, add an optional `layoutRef` whose **resolved content is +projected into `GitTarget.status`**, stamped with the profile's `generation`. Reuse without +invisibility: the target still shows what it is doing, and a fleet-wide change is observable per +target rather than only at the profile. + +**Recommendation: inline now.** Revisit when someone has a fleet-wide layout they want to +change centrally, and revisit for the `byType` map first, since that is the only part that grows. The +trigger is written down so this is a decision rather than an omission. + +## Status under the model + +The two-halves rule from the other document still applies: a **current** half derived from the last +repository scan and stamped with the revision it came from, and a **historical** half accumulated +since. Placement is sparse, so the current half must never depend on a placement having happened. + +```yaml +status: + layout: + # current, from the last scan + declaredKind: Auto # what the spec says + kind: Kustomize # what it resolved to + renderRoot: . + renderRootReason: SingleKustomization # SingleKustomization | Ambiguous | None + byTypeEntries: 1 + observedRevision: 9f3c1ab + observedTime: "2026-07-30T09:14:22Z" + # historical, since observedRevision + placedResources: 14 # appends included + overriddenTypes: 1 # types a byType entry routed + refusedResources: 0 # resources NOT mirrored + examples: + - type: v1/secrets + path: clusters/prod/secrets/db.sops.yaml + source: ByType +``` + +`declaredKind` beside `kind` is the pair that makes `Auto` honest: it says both what was asked for and +what the folder produced, so declared inference never looks like a decision the user made. + +## Metrics under the model + +`placements_total` keeps `source` (which mechanism produced the path) and gains `layout` (the resolved +kind), because they answer different questions and both are one label: + +```promql +# is any target's declared layout not the one its folder produces? +sum by (gittarget_name, layout) (increase(gitopsreverser_placements_total[24h])) +``` + +`source` values follow the model: `byType`, `layout` (the kind's own rule), and nothing else. The +current `kustomize_root` and `canonical` values become `layout` with the kind in the other label, and +`declared`/`default` collapse into `byType` plus `Template`. That is a metric-label break, and it is +free while nothing consumes them. + +## Migration + +Mechanical, and every current configuration has an exact image: + +| Today | Under the model | +|---|---| +| no `spec.placement` | `layout: {kind: Auto}` (defaulted). Same behavior | +| `placement.byType` only | `layout: {kind: Auto, byType: {...}}`. Same behavior, plus rule 1 fixing F10 | +| `placement.default` set | `layout: {kind: Template, default: "..."}`. The declared default no longer shadows the render root, because the kind now says there is no structural rule | +| `placement.byType` + `default` | `kind: Template` with both | + +The one behavior change is the good one: a declared template stops silently disabling the render root, +and starts registering its files instead. + +`spec.placement` becomes a loud rejection for one release rather than a silent alias, following the +pattern `ClusterWatchRule.spec.rules[].scope` set: refusing a stored field the user can see beats +translating it behind their back. It rides the Tier 2 breaking wave in +[`open-asks-priority.md`](open-asks-priority.md), so the consumer pays one coordinated bump. + +## What this changes about the work already queued + +Nothing already planned is wasted, and one item should wait: + +- **F10's ancestor walk** is rule 1's implementation. Build it now; the model makes it an invariant + rather than a fix. +- **`{kindLower}`** is `Flat`'s file name and a `Template` variable. Build it now. +- **The `{version}` identity fix** is needed by `Flat`'s validation and by any versionless template. + Build it now. +- **Canonical as a template constant** is `Tree`'s implementation. Build it now. +- **`status.layout`** is the same field, one release early, and `declaredKind`/`kind` slot into it. + Build it now. +- **The ambiguity policy** should wait. In this model it follows from whether the user asserted a + root, and deciding it before the model exists would bake in an answer to a question the model asks + differently. + +## Open questions + +- ~~Is `Auto` the right default?~~ **Yes**, given that it resolves once and pins the result. Pinning + is what removes the harm, and the quickstart stays four fields. +- ~~Should `Flat` be refused for a multi-namespace target?~~ **Refuse the writes, not the target.** A + scope-widening edit in another object must not break a folder; the second namespace's documents are + declined with a counted refusal naming both namespaces, and the fix is a widening layout change. +- Should `scope` be **derived and materialized** at creation (write `SingleNamespace` into the spec + when exactly one namespace is admitted) rather than declared? It would make the common case + zero-config, at the price of a mutating webhook writing spec, which this project has deliberately + avoided outside identity capture. +- Does `writeNamespace: Never` need to name its supplier (`Kustomize`, `FluxTargetNamespace`, + `Asserted`) so validation can check the guarantee rather than trust it? +- **Where does the namespace VALUE come from?** `scope: SingleNamespace` constrains cardinality; it + does not say *which* namespace, and the bootstrap case is exactly the one that cannot read it off + existing resources. Writing `namespace: team-a` into a `kustomization.yaml` we create presupposes + knowing `team-a`. The obvious source is `allowedSourceNamespaces.names` when it admits exactly one, + which every example here implicitly assumes, and that is worth making explicit rather than implied: + it would mean `create: true` plus `SingleNamespace` is refused when the admitted set is a selector + or is empty, because there is no name to write. The alternative — take it from the first document + that arrives — makes the folder's convention depend on arrival order, which is the failure mode this + whole model exists to remove. +- Does `kind: Kustomize` imply `create: true`? Asserting a folder is a kustomize folder arguably + asserts the file exists, and requiring both feels like ceremony. The argument for keeping them + separate is that creating a file in someone's repository should always be something they asked for. +- Should `Tree` remain identity-complete-by-definition, or become configurable (with or without the + version segment)? The versionless decision is deliberate and this document does not reopen it. diff --git a/docs/design/metrics-observability-plan.md b/docs/design/metrics-observability-plan.md index abe4a0ee..84ef64af 100644 --- a/docs/design/metrics-observability-plan.md +++ b/docs/design/metrics-observability-plan.md @@ -7,9 +7,9 @@ > below maps to a stage in [Common Flows](../architecture.md#common-flows). The live baseline and the > documentation bar come from [interpreting-metrics.md](../interpreting-metrics.md). This doc is the > single canonical metrics plan — it supersedes the per-feature metric notes now in `finished/`, and -> it now **absorbs** the attribution surface designed in -> [attribution-metrics-proposal.md](attribution-metrics-proposal.md), which stays as the reasoning -> trail for why that surface has the shape it does. +> it now **absorbs** the attribution surface that was designed in a separate proposal, since +> consolidated into [the attribution spec](../spec/attribution.md#what-is-observable). This document +> owns the plan; that one owns the shipped surface. ## 1. Why now @@ -30,7 +30,7 @@ uniformly dark. What is left is a sharper, smaller list: - **The delay between an event arriving and being processed** is unmeasured, and it is a proven failure mode rather than a theoretical one: a slow resolution head-of-line blocks its shard, which is what broke a CommitRequest e2e spec (see - [attribution-publish-and-join.md → what that cost](attribution-publish-and-join.md#what-that-cost-and-the-fix)). + [the attribution spec → the wait](../spec/attribution.md#the-wait)). - **Attribution was instrumented but mislabelled**, and Phase 1 has now fixed it: `result` crammed a tier and an actor kind into one label, `weak` covered two different kinds of evidence, and the wait histogram could not tell a write from a removal — exactly the distinction the removal-wait design @@ -101,6 +101,7 @@ to it is [interpreting-metrics.md](../interpreting-metrics.md). | Audit ingress | `audit_eventlists_total`, `_eventlist_events_total`, `_eventlist_duration_seconds`, `audit_events_total{outcome,category,group,version,resource,verb}` | ✅ good | | Attribution publish & join | `attribution_resolutions_total{tier,actor_kind,…}`, `attribution_resolution_wait_seconds{tier,event_kind,…}`, `attribution_facts_total{op}`, `attribution_fact_index_entries`, `_index_evictions_total{reason}`, `_stream_gaps_total{stream}`, `_stream_decode_errors_total{transport}`, `_fact_follower_errors_total{transport}`, `_fact_follower_last_success_timestamp_seconds`, `_collection_without_uidset_total{reason}`, `attribution_transport_info{transport}` | ✅ good (Phase 1 shipped) | | Git write | `commits_total{provider_*,branch,author_kind}`, `git_operations_total`, `objects_written_total`, `prune_retained_documents_total`, `branch_worker_queue_depth`, `resync_sweep_deletes_total` | 🟡 no push latency / conflict | +| New-file placement | `placements_total{source,disposition,gittarget_*,group,version,resource}`, `placement_refusals_total{reason,…}`, `placement_kustomization_entries_total{outcome,gittarget_*}` | ✅ good — shipped with the Option C deletion | | Control plane / reconcile | `target_reconcile_completed_total`, `resync_background_failures_total`, `watched_types` | ✅ good | | Secret encryption | `secret_encryption_{attempts,success,failures,cache_hits,marker_skips}_total` | ✅ good | @@ -185,6 +186,15 @@ records the metric at the smallest honest boundary. |---|---|---|---| | `git_push_duration_seconds` | histogram | `provider_*`, `branch` | push latency (re-added with a recording site and doc row) | | `git_push_conflicts_total` | counter | `provider_*`, `branch` | non-fast-forward → fetch/reset/replay retries ([PushAtomic](../../internal/git/git_atomic_push.go) detects a moved remote; [BranchWorker](../../internal/git/branch_worker.go) fetches, rebuilds, and retries) | +| `placements_total` | counter | `source`, `disposition`, `gittarget_*`, `group`, `version`, `resource` | ✅ shipped — "why did this file land here?", and which (target, type) needs a `placement.byType` line (`source="canonical"`) | +| `placement_refusals_total` | counter | `reason`, `gittarget_*`, `group`, `version`, `resource` | ✅ shipped — which resources are **not** in the mirror, and why. Replaces a log line plus `ResyncStats.PlacementSkipped` | +| `placement_kustomization_entries_total` | counter | `outcome`, `gittarget_*` | ✅ shipped — `failed` is a file committed outside every render: in Git, looks mirrored, applied by nothing | + +**Note on `placements_total`, since this plan argued the other way.** An earlier revision of the +priority queue said to argue *against* leading with a `placement_fell_back_total`, because "it happened +somewhere" is not actionable. That objection was to the **labels**, not to the counter: with the +GitTarget and the type key on it, one series reads directly as the `byType` line that is missing. The +per-resource detail (path, name) deliberately stays in the log line. ### 4.6 Catalog, reconcile, secrets @@ -196,7 +206,7 @@ rules/CRDs change (pairs with `target_reconcile_completed_total{trigger=rule_cha This is the subsystem to keep glass-box, and it is the one that changed most. The model is now two halves that never call each other, meeting only through the keys a fact was filed under -([attribution-publish-and-join.md](attribution-publish-and-join.md)): +([the attribution spec](../spec/attribution.md)): ```text kube-apiserver --POST--> /audit-webhook --gate--> append one entry per type to the fact stream @@ -257,7 +267,7 @@ that found the window race had to *infer* "these were `latest` matches held as f distribution, because the label could not say it. The tier ladder itself, strongest first, is documented in -[attribution-publish-and-join.md → the tiers](attribution-publish-and-join.md#the-tiers-strongest-first); +[the attribution spec → the tiers](../spec/attribution.md#the-tiers-strongest-first); the operator-facing reading of each value is in [interpreting-metrics.md](../interpreting-metrics.md#audit-attribution-optional). @@ -323,8 +333,10 @@ before it can carry that meaning. | `resolvers_waiting` | queue delay (§4.2) proves insufficient, **and** it is incremented around the blocking `select` alone — `Await` registers *before* its first lookup on purpose, so a gauge at registration counts resolutions in flight rather than resolvers blocked | | `fact_index_expired_total` | wanted when tuning the TTL or the caps; low risk, low urgency | -The full record of what an earlier draft of this surface got wrong, and why each mistake was -invisible, is in [attribution-metrics-proposal.md](attribution-metrics-proposal.md#what-the-first-draft-got-wrong). +What an earlier draft of this surface got wrong, and why each mistake was invisible, is recorded in +`git log` on the proposal that has since been folded into +[the attribution spec](../spec/attribution.md#what-is-observable); the three things that surface +deliberately cannot answer are listed there. ## 6. The reference dashboard @@ -511,10 +523,8 @@ those rules are written from, not shipped rules. - [architecture.md](../architecture.md) — leading source of truth (esp. *Common Flows*, *Optional Attribution*, *State Ingestion*, *Observability*). - [interpreting-metrics.md](../interpreting-metrics.md) — the live baseline + the per-metric doc bar. -- [attribution-metrics-proposal.md](attribution-metrics-proposal.md) — the reasoning trail for §4.4 - and §5, including what an earlier draft got wrong. -- [attribution-publish-and-join.md](attribution-publish-and-join.md) — how the two halves work, and - the tier ladder the `tier` label names. +- [spec/attribution.md](../spec/attribution.md) — the shipped attribution surface behind §4.4 and + §5, how the two halves work, and the tier ladder the `tier` label names. - [attribution-fact-stream.md](../finished/attribution-fact-stream.md) — the shipped transport, the in-process index, and the follower these metrics watch. - [watch-first-ingestion-architecture.md](../finished/watch-first-ingestion-architecture.md) — the diff --git a/docs/design/open-asks-priority.md b/docs/design/open-asks-priority.md index 4d76a204..ae319604 100644 --- a/docs/design/open-asks-priority.md +++ b/docs/design/open-asks-priority.md @@ -2,8 +2,17 @@ > **design**: a priority call, not a plan of record. Nothing here binds until scheduled. > Index: [`../INDEX.md`](../INDEX.md) -> Date: 2026-07-29. Written against `v0.40.1` / `main` plus the attribution branch -> (`feat/attribution-sticky-removal-pointer`), which is where the largest item on this page went. +> Date: 2026-07-29, swept 2026-07-30 against the 0.41.0 release. +> +> **Where this stands as of the sweep.** `0.41.0` is the attribution release: the fact stream, the +> sticky removal pointer, the metric relabel, the name tier, and the analyzer/encoder corrections, +> plus PR #291's sibling-inference deletion and placement counters. That is a large breaking release +> on its own, and **the GitTarget work is deliberately not in it**. The layout model and the API wave +> ([`gittarget-layout-model.md`](gittarget-layout-model.md), +> [`gittarget-api-wave.md`](gittarget-api-wave.md)) are postponed to a later deployment and tracked as +> GitHub issues, so the queue below is read with one standing caveat: **every Tier 2 entry that +> changes a `GitTarget` field is now part of that postponed wave, not independently schedulable.** +> The Tier 1 entries are not, and should not wait for it. > > **The queue was built bottom-up rather than top-down.** Tier 0 and Tier 1 are still unbuilt, and > the Tier 2 item nobody scheduled — the attribution fact stream — shipped anyway, together with a @@ -54,7 +63,13 @@ do not own. --- -## The one real design call: delete sibling inference, do not switch it off +## The one real design call: delete sibling inference, do not switch it off — SHIPPED + +> **Built.** `resolveInferred` through `allSameDir` are gone, the kustomize-root fallback stayed, and no +> enum was added. What building it added to the argument below is recorded in +> [what the deletion taught](#what-the-deletion-taught). The spec's Option C sections are retained as +> history in [`gittarget-new-file-placement-rules.md`](../spec/gittarget-new-file-placement-rules.md), +> and the behaviour change has a [`docs/UPGRADING.md`](../UPGRADING.md) entry. The config-surface proposal's **B3** offers `spec.placement.mode: Infer|Declared|Strict`: an enum that lets a user turn inference off. This document argues the opposite: **remove Option C's @@ -170,39 +185,94 @@ and the word *Event* meant a real `corev1.Event` through **The shape that follows is a split, not one of the three.** An **Event on the GitTarget** for timeliness, over the existing refusal seam plus an enqueue; and **`status.layout` (B2)** for durability, because "what the operator understood about this folder" is where someone looks a day -later. The log line stays. A Prometheus counter is the one to argue *against* leading with: -`placement_fell_back_total` says it happened somewhere, not which type in which target, which is the -only actionable part. +later. The log line stays. + +**What shipped, and where this paragraph was wrong.** A Prometheus counter was the one this document +argued *against* leading with, on the grounds that `placement_fell_back_total` says it happened +somewhere and not which type in which target. That objection was to the **labels**, and it does not +survive them being fixed: `placements_total{source, disposition, gittarget_namespace, gittarget_name, +group, version, resource}` names the target and the exact `byType` key, so one series **is** the line +that is missing. It shipped with two companions the argument had not asked for and should have — +`placement_refusals_total{reason}`, because a resource the writer *declined* to place had no +countable trace at all, and `placement_kustomization_entries_total{outcome}`, whose `failed` value is +a file committed outside every render. The Event and `status.layout` are still the right split for +timeliness and durability, and neither is built; what is no longer true is that there was nothing +actionable in a metric. + +### What the deletion taught + +Three things came out of building it that the argument above did not contain. + +- **Deleting the inference exposed a second implementation of a rule, not just the rule.** "Omit + `metadata.namespace`, the build context supplies it" was read off a *sibling's bytes*, so it only + ever fired for an inferred placement. A **declared** path into the same kustomize directory wrote a + `namespace:` line every other document in that folder omits. The obligation belongs to the + kustomization that governs the destination, not to whatever document happened to be next door, and + moving it there fixed the declared path for free. +- **And that rule was missing its safety half.** The old kustomize-root fallback asked only whether a + `namespace:` transformer was *set*, never whether it named the resource's own namespace. Omitting + the namespace hands it to kustomize, so a transformer naming a different namespace rendered the + document as a different object: the mirror claimed to hold a resource it did not. It now writes the + namespace explicitly in that case and lets the render oracle report a folder that cannot express + the object. This was reachable before the deletion and is one of the things the deletion's own + test table found. +- **The write path had no identity to label with, and that is why it had no metrics.** `LocateNew` + runs on the branch worker with no reconcile context, which this document already noted about the + Event. The same fact is why the placement counters did not exist: there was no GitTarget on the + batch to name. It is one field, taken from the events on the live path and from the resolved + metadata on the resync path — deliberately both, because which of the two created a file is not + something the operator chose, and a fall-back visible for one and invisible for the other would be + worse than neither. --- ## The queue -| # | Ask | Source | Tier | -|---|---|---|---| -| 15 | A declared `auditRoute` with zero facts must say so, and a route losing them with it | gitops-api | **1** | -| n/a | Stop paying a full grace for a delete fact that will never arrive (F, then C) | [`attribution-removal-wait-options.md`](attribution-removal-wait-options.md) | **1** | -| n/a | Delete sibling inference (answers #10) | this doc | **1** | -| F6 | `spec.suspend`, `spec.interval`, `requestedAt` | maintainer review | **2** | -| 5 | `CommitRequest.spec.author`, SAR-guarded | gitops-api (#220) | **2** | -| B4 | `commitWindow` / `commit.message` move to GitTarget | config surface | **2** | -| B1 | `GitTarget.spec.mode: Observe\|Write` | config surface | **2** | -| 6 | Movable destination via `status.observedDestination` | gitops-api (#220) | **2** | -| F10 | CommitRequest TTL / ownerRef + the `delete` verb | maintainer review | **2** | -| n/a | The blocking resolve is head-of-line on the shard goroutine | [`attribution-branch-findings.md`](attribution-branch-findings.md) | **2** | -| B2 | `GitTarget.status.layout` | config surface | **3** | -| F9 | The `scope: Namespaced` status-write envtest | maintainer review | **3** | -| B6 | The `default` ClusterProvider not-found message | config surface | **3** | -| n/a | An aggregated create carries no name and no body: accept it, or stop waiting for it | [`attribution-branch-findings.md`](attribution-branch-findings.md) | **3** | -| n/a | Entry-size ceiling and per-type stream count under a few hundred watched types | [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md) | **3** | -| 10 | Namespace-aware sibling inference *as asked* | gitops-api | **declined** | -| B3 | `spec.placement.mode` enum | config surface | **declined** | +**Filed** means there is a GitHub issue, so the item is legible without this page. **Wave** means it is +part of the postponed breaking GitTarget sequence ([#294](https://github.com/ConfigButler/gitops-reverser/issues/294)) +and is not independently schedulable. + +| # | Ask | Source | Tier | Tracked | +|---|---|---|---|---| +| 15 | A declared `auditRoute` with zero facts must say so, and a route losing them with it | gitops-api | **1** | — | +| n/a | Stop paying a full grace for a delete fact that will never arrive (F, then C) | [`attribution-removal-wait-options.md`](attribution-removal-wait-options.md) | **1** | — | +| n/a | A declared path in a kustomize subdirectory is never rendered; the identity gate rejects the versionless canonical path | [`placement-visibility-and-declared-defaults.md`](placement-visibility-and-declared-defaults.md) | **1** | [#295](https://github.com/ConfigButler/gitops-reverser/issues/295) | +| n/a | `spec.layout`: declare what the folder is | [`gittarget-layout-model.md`](gittarget-layout-model.md) | **2** | [#293](https://github.com/ConfigButler/gitops-reverser/issues/293), wave | +| F6 | `spec.suspend`, `spec.interval`, `requestedAt` | maintainer review | **2** | wave | +| 5 | `CommitRequest.spec.author`, SAR-guarded | gitops-api (#220) | **2** | wave | +| B4 | `commitWindow` / `commit.message` move to GitTarget | config surface | **2** | wave | +| B1 | `GitTarget.spec.mode: Observe\|Write` | config surface | **2** | wave | +| 6 | Movable destination via `status.observedDestination` | gitops-api (#220) | **2** | wave | +| F10 | CommitRequest TTL / ownerRef + the `delete` verb | maintainer review | **2** | wave | +| n/a | The blocking resolve is head-of-line on the shard goroutine | [`../spec/attribution.md`](../spec/attribution.md#the-wait) | **2** | — | +| B2 | `GitTarget.status.layout` | config surface | **3** | [#296](https://github.com/ConfigButler/gitops-reverser/issues/296) | +| n/a | The ambiguous render root, the `declared` metric split, `{kindLower}`, canonical-as-template | [`placement-visibility-and-declared-defaults.md`](placement-visibility-and-declared-defaults.md) | **3** | [#296](https://github.com/ConfigButler/gitops-reverser/issues/296) | +| F9 | The `scope: Namespaced` status-write envtest | maintainer review | **3** | outside the wave, deliberately | +| B6 | The `default` ClusterProvider not-found message | config surface | **3** | — | +| n/a | An aggregated create carries no name and no body: accept it, or stop waiting for it | [`../spec/attribution.md`](../spec/attribution.md#what-the-shape-driven-rules-reach-and-what-they-do-not) | **3** | — | +| n/a | Entry-size ceiling and per-type stream count under a few hundred watched types | [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md) | **3** | — | +| 10 | Namespace-aware sibling inference *as asked* | gitops-api | **declined — answered by the deletion, SHIPPED** | — | +| B3 | `spec.placement.mode` enum | config surface | **declined** | — | + +**One entry moved up in this sweep.** The declared-path-in-a-subdirectory bug is Tier 1, not Tier 3, +under this page's own first test: one line of ordinary user configuration silently produces a file that +is in Git and rendered by nothing, and nothing in status or in the counters says so. That is the +product being silently wrong, which is what Tier 1 is for. It was written down as a finding rather than +ranked, because it was found while arguing about metric names. ### Already shipped, and struck from the queue -Four things left this page between 2026-07-28 and 2026-07-29, all on -`feat/attribution-sticky-removal-pointer`. They are listed rather than deleted because two of them -change what the *remaining* entries should be. +Everything in this section is in **0.41.0**. Four items left this page between 2026-07-28 and +2026-07-29 on `feat/attribution-sticky-removal-pointer`, two more arrived with #290, and the +placement break arrived with #291. They are listed rather than deleted because several of them change +what the *remaining* entries should be. + +The release is worth naming as one thing, because it is what makes postponing the GitTarget wave the +right call rather than a delay: **0.41.0 replaces the whole attribution model** (a fact keyspace +becomes a per-type stream, the resolver stops polling Redis, the metric surface is relabelled, and +three populations that used to ship committer-authored now resolve) **and breaks placement** +(sibling inference is gone). Two breaking dimensions in one release is already a lot to ask a consumer +to absorb. A third, on the shape of `GitTarget` itself, is a separate conversation. - **The attribution fact stream** (was the largest Tier 2 entry). Built as #283, #284, #286 and #287: the audit receiver appends one batched entry per type to a per-`(route, group/resource)` stream, @@ -211,8 +281,9 @@ change what the *remaining* entries should be. `exact_deletecollection_item` is replaced by `deletecollection_body_uid` and `deletecollection_scope`, and `--author-attribution-transport=memory` runs attribution with no Redis on one replica. Record: [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md). - §5, §6 and §8 of [`deletecollection-attribution-expander.md`](../spec/deletecollection-attribution-expander.md) - have been rewritten to say what took their place, which discharges commitment 6 below. + the expander spec has been folded into [`../spec/attribution.md`](../spec/attribution.md), which keeps + its deletion-as-intent rule as §1 and drops the sections about the deleted machinery. That discharges + commitment 6 below. - **#23 — deletion-as-intent picked the cleanup controller, not the deleter.** Filed in revision 11 and fixed before it was ranked, because the reproduction fell out of the switchover's own corpus: the human's `delete` and the controller's finalizer `patch` both return a body carrying the @@ -222,16 +293,16 @@ change what the *remaining* entries should be. about a write, keyed strictly by uid, consulted ahead of the exact tier for a removal, bounded by the index's caps rather than the join TTL. Ships `delete_sticky` on `attribution_resolutions_total{tier}`. Record: - [`attribution-deletion-intent-actor.md`](attribution-deletion-intent-actor.md). + [`../spec/attribution.md`](../spec/attribution.md#three-rules-that-are-easy-to-miss). - **A name tier**, which was not asked for by anyone. An aggregated-API write or single delete is audited with a name but no uid and no resourceVersion, so every stronger tier misses it and it used to ship committer-authored. Facts carrying neither identifier are now filed under `(namespace, name)` and consulted last. Record: - [`attribution-branch-findings.md`](attribution-branch-findings.md) §4. + [`../spec/attribution.md`](../spec/attribution.md#the-tiers-strongest-first). - **Phase 1 of the attribution metric surface.** `result` split into `tier` and `actor_kind`, the `no_attribution_fact` audit outcome, and the loss-path counters — including the stream decode error, which had no symptom at all. Record: - [`attribution-metrics-proposal.md`](attribution-metrics-proposal.md), migration in + [`../spec/attribution.md`](../spec/attribution.md#what-is-observable), migration in [`UPGRADING.md`](../UPGRADING.md). - **#22 — the analyzer contract's three false sentences**, all three fixed: @@ -324,10 +395,10 @@ shipped with it already say when a follower is losing facts. Two consequences: losing facts and a route that never had any are the same user-visible failure: commits authored `unknown (attribution unresolved)`. One condition, two messages. -Where `auditRoute` came from is [`attribution-fact-identity.md`](attribution-fact-identity.md). +Where `auditRoute` came from is [`../spec/attribution.md`](../spec/attribution.md#the-scope-is-an-audit-route-and-a-type). The related question about *how the watch waits* is answered for the transport and reopened one level down: the six options in -[`attribution-wait-poll-vs-push.md`](attribution-wait-poll-vs-push.md) are superseded by +an earlier option analysis are superseded by [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md), and what remains is *when a removal should stop waiting*, immediately below. @@ -351,9 +422,11 @@ What is explicitly *not* measured yet, and should be before either lands: how co never-resolved population is outside the e2e suite, and whether a quiet route's watermark advances often enough to be worth having. The one number we do have came from a single run. -**The inference deletion** sits in this tier for the reason argued above: repo state changing -operator behavior invisibly is the same class of defect as an audit route that silently resolves -nothing. +**The inference deletion — SHIPPED.** It sat in this tier for the reason argued above: repo state +changing operator behavior invisibly is the same class of defect as an audit route that silently +resolves nothing. What replaced it is a declaration plus +`placements_total{source="canonical"}` per (GitTarget, type), so the same class of defect now has a +query. `#10` and `B3` are answered by it and stay declined. ### Tier 2: the breaking wave, all at once, while `v1alpha3` @@ -373,7 +446,7 @@ evidence the index already held, and the write behind them missed the commit win The lookup-ordering half is fixed (that is what the sticky pointer and the tier reordering did), and the structural half is untouched: *any* removal that must wait out its grace still stalls every later event on its shard. Two directions, from -[`attribution-branch-findings.md`](attribution-branch-findings.md): +[`../spec/attribution.md`](../spec/attribution.md#the-wait): 1. **Bound the removal's extra wait separately from the grace.** Once a fallback is in hand the fact stream for that scope is demonstrably live, so what is outstanding is an audit-batch interval @@ -473,7 +546,7 @@ likely first-run support ticket. **The aggregated create: decide, and the decision is small either way.** The name tier reaches an aggregated update, patch and single delete. It cannot reach a create: the `objectRef` carries no name and there is no response body to recover one from, so nothing is published for any tier to join. Two -non-exclusive options, from [`attribution-branch-findings.md`](attribution-branch-findings.md) — +non-exclusive options, from [`../spec/attribution.md`](../spec/attribution.md) — **accept it** (document that per-object attribution does not apply, and let it ship committer-authored, which makes the guarantee type-dependent in a way a user cannot predict from the API surface), or **stop paying for it** (recognize the shape at publish time and skip the grace, which @@ -497,13 +570,21 @@ defects. ## What this commits us to -1. A `feat(api)!` sequence for Tier 2, landed together, with one `docs/UPGRADING.md` entry. -2. A behavior change (inference removal) that needs its own UPGRADING entry and a decision on the - fall-back-to-canonical Event. -3. Rewriting Option C's sections in - [`gittarget-new-file-placement-rules.md`](../spec/gittarget-new-file-placement-rules.md). That - document binds the code, so the ladder cannot be deleted from one and left in the other. The - kustomize-root fallback keeps its section; P1–P10 become history rather than live risks. +1. A `feat(api)!` sequence for Tier 2, landed together, with one `docs/UPGRADING.md` entry — **and + not in this release.** It is tracked as [#294](https://github.com/ConfigButler/gitops-reverser/issues/294) + with the layout model as [#293](https://github.com/ConfigButler/gitops-reverser/issues/293). The + commitment this sweep adds is the negative one: 0.41.0 ships the attribution model and the + placement break, the wave waits, and no half of the wave is allowed to land on its own — because + the reason to batch it was never only the consumer's bump, it was that four of the items are one + decision and deciding it four times is how the object stops reading as one idea. +2. ~~A behavior change (inference removal) that needs its own UPGRADING entry and a decision on the + fall-back-to-canonical Event.~~ **Done for the removal and the entry**; the Event is still + undecided, and the metric now carries the actionable part in the meantime. +3. ~~Rewriting Option C's sections in + [`gittarget-new-file-placement-rules.md`](../spec/gittarget-new-file-placement-rules.md).~~ + **Done**: the ladder is documented as three steps, the kustomize-root fallback keeps its section + and gained the namespace-match rule, and P1–P10 are annotated one by one with which are retired by + the deletion and which (P7, P9, P10) are facts about the code that remains. 4. Telling the gitops-api team which two of their asks we are answering differently, before they build against the shapes they proposed — and that **#23 is fixed**, that the fix has a name (`delete_sticky` on `attribution_resolutions_total{tier}`) they can assert on, and that the @@ -514,8 +595,18 @@ defects. ~~Specifying it in transport-neutral terms before the stream work starts~~ — the stream landed first, which makes this cheaper rather than harder. 6. ~~Retiring §5 and §8 of the expander spec when the expander goes~~ — **done**: §5, §6 and §8 of - [`deletecollection-attribution-expander.md`](../spec/deletecollection-attribution-expander.md) - now say what replaced them, and §2's deletion-as-intent rule is kept, which the collection join - and the sticky pointer both depend on. + the expander spec are gone with the spec itself, folded into + [`../spec/attribution.md`](../spec/attribution.md). Its deletion-as-intent rule is kept as §1, which + the collection join and the sticky pointer both depend on. 7. Deciding the removal-wait question (F, then C) with a measurement rather than by argument, since the one number on the table came from a single e2e run whose population is not a workload. +8. Keeping attribution documented in **one** place. Six design records described one part each while + it was being built; they are folded into [`../spec/attribution.md`](../spec/attribution.md), which + binds, and the only reasoning trail kept in full is + [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md). The commitment is that a + change to attribution behaviour changes that spec, rather than adding a seventh record. +9. Not letting a decided-but-unbuilt list read as imminent. The placement work is filed as + [#295](https://github.com/ConfigButler/gitops-reverser/issues/295) (correctness) and + [#296](https://github.com/ConfigButler/gitops-reverser/issues/296) (visibility); + [`placement-visibility-and-declared-defaults.md`](placement-visibility-and-declared-defaults.md) + now says which of its eight items shipped, which is **none of them**. diff --git a/docs/design/placement-visibility-and-declared-defaults.md b/docs/design/placement-visibility-and-declared-defaults.md new file mode 100644 index 00000000..7ef490e3 --- /dev/null +++ b/docs/design/placement-visibility-and-declared-defaults.md @@ -0,0 +1,557 @@ +# Placement, made visible: naming, a declared default, and `status.layout` + +> **design**: decided, mostly **not built**. Index: [`../INDEX.md`](../INDEX.md) +> Date: 2026-07-30, reconciled 2026-07-30 against what PR #291 actually contains. +> +> **The decisions below stand. The build list does not.** An earlier revision of this page said +> "everything decided here lands in that same PR", and that turned out to be wrong: PR #291 shipped +> the sibling-inference deletion, the three placement counters, and the namespace-transformer safety +> fix, and **none of the eight items** this page had queued behind them. What is decided and unbuilt +> is now tracked as GitHub issues rather than asserted here as imminent; see +> [what is built and what is filed](#what-is-built-and-what-is-filed). +> +> Reading it in order matters, because the two halves have different standing. The **findings** are +> checkable facts about the tree and they are why the calls are what they are. The **calls** are still +> the calls. Question 2 (a CRD default for `placement.default`) has since been superseded outright by +> [`gittarget-layout-model.md`](gittarget-layout-model.md), which argues that a path template is the +> wrong primitive to be defaulting at all. + +Three questions came out of reviewing #291, and one of them (a CRD default for +`placement.default`) is a better idea than my first answer to it gave credit for. This document +states the findings behind each, the options, and the call. + +## The decisions, up front + +| Question | Call | Why in one line | +|---|---|---| +| Rename `source="canonical"` to `default`? | **No.** Keep `canonical`, split `declared` into `byType` and `default`, and fix the prose | `placement.default` is a *declared* template; reusing the word for the built-in path makes one name mean both a declaration and the absence of one | +| Default `placement.default` in the CRD? | **No**, on narrower grounds than the first draft claimed | A defaulted default consumes the slot ahead of the one step no template can express ("beside the folder's one kustomization"). Once F10 is fixed this is a legibility trade rather than a correctness wall | +| Publish the effective layout in status? | **Yes, now** | It gives the clarity the CRD default was reaching for, without freezing anything, and the seam it needs already exists | +| `{kindLower}` or a `toLower` function? | **`{kindLower}`** | A function syntax is a language; one variable answers the actual need | +| Is the kustomize root "still a little bit inference"? | **Yes, and it earns its keep anyway** | Its answer changes only when a `kustomization.yaml` changes, and ignoring it produces a file nothing renders. It gets the same visibility obligation as everything else | +| A declared path in a subdirectory of a kustomize folder | **Bug, fix it**: walk up to the nearest kustomization | Today it is registered only when render-root scoping happens to be in force, so a `byType` entry into a subdirectory silently produces a file nothing renders (F10) | +| Two kustomize roots (ambiguous) | **Keep writing, make it loud** | The file is currently unreachable *and* uncounted. Refusing would protect nothing that mirroring does not | + +## Findings + +Each of these is checkable against the tree, and three of them changed a decision. + +**F1. The word "default" already means two things in our own docs.** `spec.placement.default` is a +user-declared catch-all. [`architecture.md`](../architecture.md) calls the built-in path "the +**built-in default** path"; the CRD comment calls the same thing "the built-in canonical path". One +of those has to go, whatever we do with the metric. + +**F2. `source="declared"` hides which declaration answered.** A `byType` hit and a `default` hit are +one series, so a catch-all quietly swallowing a type you meant to name explicitly looks identical to +a rule working as intended. For a metric whose job is "is a rule missing?", that is the wrong place +to lose resolution. + +**F3. A CRD default for `placement.default` fails our own validation today.** Not the decisive +objection (F9 is), and not unfixable (F4 is the fix), but it is where the idea stops today. +[`validateSecretSafety`](../../internal/controller/gittarget_placement_validation.go) rejects a +`default` that is not identity-complete unless the target also declares an identity-complete +`byType["v1/secrets"]` entry. And `IdentityCompletePlacementTemplate(tmpl, false)` requires +`{groupPath}`, `{version}` **and** `{resource}`. The built-in path is deliberately **versionless** +(see [`../facts/`](../INDEX.md) and the versionless-path decision), so the template we would default +to, + +```text +{namespaceOrCluster}/{groupPath}/{resource}/{name}{sensitiveSuffix} +``` + +is judged *not* identity-complete, and every GitTarget that did not also declare a Secret route +would go `Validated=False`. The CRD's default would be refused by the CRD's own gate. + +**F4. The `{version}` requirement is wrong on its own terms.** Two versions of one group/resource +are the *same object*, which is exactly why the built-in path dropped the version segment. A +template carrying scope, group, resource and name cannot collide two distinct identities, with or +without a version. So the requirement rejects safe templates: a user who writes the +versionless canonical layout by hand is told it is a bundling path. That is a bug independent of +anything else here, and fixing it is a precondition for F3 ever being reconsidered. + +**F5. Defaulting semantics, precisely, because a first draft of this document got them wrong.** +A CRD default is applied to the request object on create, update and patch, and is therefore +**persisted**. It is also applied **in memory when reading from etcd**, using the storage version's +schema, so the controller sees the value even for an object stored before the default existed. It is +**not** persisted by our own status writes: `GitTarget` has a status subresource +([gittarget_types.go](../../api/v1alpha3/gittarget_types.go)), so a status update writes only the +status stanza and leaves the stored spec alone. In a GitOps-managed cluster the answer is +nevertheless "yes, quickly", because the user's own Flux or Argo re-applies the object continuously +and each apply persists it. + +Two consequences follow, and **both are smaller than an earlier draft of this document claimed**. +Freezing the built-in path per target is arguably a feature rather than a cost: placement is already +create-time and non-retroactive, so a long-lived target keeping its established layout for new types +is the stability we already promise, and a value in the spec is visible and editable in a way a +built-in is not. "Indistinguishable from a declaration" is close to true, and an earlier draft's +escape from it was wrong: `metadata.managedFields` is field-management bookkeeping rather than +durable provenance, so it cannot separate "the user declared this" from "the schema supplied it". +What is left is spec bloat: every GitTarget carries a template string whether +its owner cares about layout or not. That alone would not decide anything. + +**F5b. Defaulting a map is a floor that vanishes when you stand on it.** A CRD default applies to an +**absent** field; it is not an additive base and is never re-merged per key. So a defaulted +`byType: {"v1/secrets": "..."}` is gone as soon as that field is present at all, which is the moment a +user writes any `byType` entry of their own. +Their unrelated ConfigMap entry would silently drop the Secret route, leaving a defaulted bundling +`default` unguarded, and the object would go `Validated=False` on an edit that had nothing to do with +Secrets. This is why "default the Secret route too" (the obvious repair for F3) is worse than +fixing F4. + +**F6. The data-plane to status seam already exists, and it enqueues.** +[`retention_rollup.go`](../../internal/watch/retention_rollup.go) is the pattern: +`MarkTargetRetention` records a fact from the write path into an epoch-scoped per-target roll-up and +calls `enqueueGitPathChange` **on a change only**, and the controller projects it in +`gitTargetRetentionStatus`. This retires the objection recorded in +[`open-asks-priority.md`](open-asks-priority.md) that placement facts cannot reach the GitTarget +promptly because "a refusal recorded on the data plane does not enqueue the GitTarget". One does +already. `status.retention` is proof. + +**F7. `{kind}` renders capitalized and `{resource}` renders plural.** So neither gives +`deployment-simon.yaml`, and the spec's own "keep it small" rules out template functions ("no +template functions except safe path-segment sanitization"). + +**F8. Two supported kustomizations produce a file nothing renders, and nothing counts it.** +Ambiguity declines to the canonical path, which no root's `resources:` graph reaches. No entry is +attempted, so `placement_kustomization_entries_total` never sees it. It is the exact failure that +counter exists for, in the one case it cannot observe. + +**F9. A defaulted `default` would make the kustomize-root step unreachable.** This is the objection +that decides Question 2, and it is structural rather than a trade. +[`LocateNew`](../../internal/manifestanalyzer/placement.go) resolves in this order: + +```text +byType -> default -> kustomize root -> canonical +``` + +`resolveDeclared` returns on any non-empty `policy.Default`, so a default that is *always* set means +step 3 never runs. Every new file in a kustomize overlay would take the canonical path instead of +landing beside the `kustomization.yaml`, with no `resources:` entry: committed, looking mirrored, and +rendered by nothing. That is exactly the bug the kustomize-root step was added to prevent. + +The repairs are worse than the defect. Reordering so the render root beats `default` makes a user's +real declaration lose to the folder, which inverts the precedence the whole feature rests on. +And there is no sound way to distinguish a defaulted value from a declared one at read time. + +**F10. And that failure is reachable today, from a declaration, which makes it a live bug rather +than an argument about defaulting.** Review found this, and it is the most useful thing on the page. +[`governingKustomization`](../../internal/manifestanalyzer/placement.go) decides whether a new file +gets a `resources:` entry by looking in exactly two places: the kustomization in the file's **own** +directory, and (only when render-root scoping is in force) the write scope's root. So: + +| Folder shape | Declared path | Registered? | +|---|---|---| +| overlay reading a base (`writeScope != ""`) | `configmaps/app.yaml` | yes, in the overlay root, by accident of a branch added for another reason | +| self-contained folder that IS the kustomize root | `configmaps/app.yaml` | **no** | + +One line of user config reproduces it: a kustomize folder plus +`byType: {v1/configmaps: "configmaps/{name}.yaml"}`. The file lands in a directory with no +kustomization of its own, no entry is added, and kustomize never builds it. Because no entry is +attempted, `placement_kustomization_entries_total` cannot see it either. + +**The fix**: `governingKustomization` walks **up** from the resolved path to the nearest kustomization +within the write jail, replacing the own-directory check and the write-scope special case with one +rule. An ancestor walk cannot escape the jail by construction, the relative entry it produces is what +`appendKustomizationResource` already computes, and the already-listed check is path-based, so it +stays idempotent. A nearest ancestor that is *unsupported* still cannot be edited; that becomes a +counted `failed` instead of silence. + +**What this does to F9**: it weakens it, and the page should say so rather than keep an argument that +has been undercut. With the ancestor walk a defaulted `default` no longer produces unrendered files, +because the nested tree would be registered in the root and would build. What survives is narrower. +The built-in path is the wrong *layout* for a kustomize folder, and **no template can express "beside +the folder's one root"**, which is why that step is a step rather than a template. Defaulting +`default` consumes the slot ahead of a step that exists precisely because a template cannot say what +it says. That is a legibility and structure argument, not a correctness one, so the CRD default moves +from "no" to "a trade available to us". The recommendation does not change; its grounds do. + +## Question 1: `canonical` or `default`? + +The pull toward `default` is real: it is the word for "what happens when you say nothing", and +`placement.default` is the field a user reaches for. But that is precisely the collision (F1). If +the metric said `source="default"`, a reader could not tell whether their `spec.placement.default` +had matched or whether nothing had matched at all, and those are opposite situations with opposite +fixes. + +So: **`canonical` keeps the built-in path**, the prose stops calling it "the built-in default", and +the resolution gains the distinction that was missing (F2): + +| `source` | Means | +|---|---| +| `byType` | an exact type entry matched | +| `default` | the declared catch-all matched. Same word, same meaning as the CRD field | +| `kustomize_root` | the folder has exactly one supported kustomization, so the file went beside it | +| `canonical` | nothing above applied; the built-in path was used | + +`default` then means one thing everywhere: the user's catch-all. + +## Question 2: a CRD default for `placement.default` + +The idea: if the built-in path were the CRD's default for `placement.default`, every GitTarget would +show its own layout, "canonical" would stop being a hidden fourth mechanism, and `source` would +collapse to `byType` and `default`. That reasoning is sound, and for a reader of one object it **is** +clearer. This section was rewritten after review, because the first version of it led with the wrong +objection. + +### What is not the reason + +**Not "it fails validation" on its own (F3).** It does today, but the rule doing the rejecting is +itself wrong (F4): the versionless canonical template cannot collide two identities, and demanding +`{version}` of it contradicts the decision that removed the version segment. One line fixes it. + +**Not "the value gets persisted" (F5).** It does get persisted, on every spec-writing apply, and in a +GitOps-managed cluster that is continuous. But freezing the built-in path per target is arguably +*desirable*: placement is already create-time and non-retroactive, so a long-lived target keeping its +established layout for new types is the stability we already promise, and a string in the spec is +visible and editable where a built-in is neither. `metadata.managedFields` is not a way around +it either: field management is bookkeeping, not provenance. What is left is spec bloat, which decides +nothing. + +### Why not also default the Secret route? + +The obvious repair for F3 is to default `byType["v1/secrets"]` as well: narrowed to one type, it only +needs `{name}` plus scope to be identity-complete, `secretRouteComplete` goes true, and the +bundling-default check stops objecting. It does work, and it is still the wrong move, because +**defaulting a map is a floor that vanishes when you stand on it** (F5b). A user who writes any +`byType` entry of their own replaces the whole map, silently dropping the Secret route and leaving a +defaulted bundling `default` unguarded, so an edit about ConfigMaps flips the object to +`Validated=False`. Fixing F4 removes the need for the workaround entirely. + +### The objection that decides it + +**A defaulted `default` is never empty, so it shadows the kustomize-root step (F9).** `resolveDeclared` +returns on any non-empty declared template, and the render-root step runs after it, so defaulting the +field makes step 3 unreachable. + +The first version of this section claimed the consequence was unrendered files. **That claim does not +survive F10**: the same gap is reachable from a plain declaration today, it is a bug we are fixing, +and once `governingKustomization` walks up to the nearest kustomization a nested path is registered +and builds. So the honest form of the objection is smaller and structural: + +- a nested canonical tree registered inside an overlay **renders**, and no one would choose it over a + file beside the `kustomization.yaml`. Defaulting would make the worse layout the layout; +- **no template can express "beside the folder's one supported kustomization"**, which is why that + step exists as a step. A defaulted template consumes the slot in front of it, so the one part of + the ladder that cannot be written down as a path would become unreachable by construction. + +The repairs still invert something load-bearing. Putting the render root ahead of `default` makes a +user's real declaration lose to the folder. And there is no sound way to tell a defaulted value from a +declared one at read time, so "treat a declared default differently" is not implementable. + +### What we do instead + +Publish the **effective** placement in status. It answers the same question ("what will happen to a +new Deployment in this folder?") from a derived field, so it cannot fork from the code and improves +when the code improves. Crucially it can show the ladder **without collapsing it**: the declared half, +the render root the operator found, and what happened, all at once. A spec default can only +express the ladder by flattening it, which is what F9 is. + +**If we want the spec default later**, the order is written down: fix F4, pin the canonical template +against `ToGitPath()` byte-for-byte, and solve F9 first, because nothing else on this page matters +until a defaulted value can coexist with the render-root step. Status is worth having either way, so +nothing built now is wasted. + +## `status.layout` + +An **observation, not a condition**, in the sense +[`GitTargetStatus.Retention`](../../api/v1alpha3/gittarget_types.go) already establishes: nothing +here may fail a reconciliation or move a condition, because a folder with no declared placement is a +supported configuration and not a fault. It follows `status.streams`'s bounding rule too: counts and +a capped example list, never a per-type list, however many types a target watches. + +```yaml +status: + layout: + # what the operator understood about the FOLDER + renderRoot: overlays/production # empty unless exactly one supported kustomization + renderRootReason: SingleKustomization # SingleKustomization | Ambiguous | None + # where new files are decided FROM + effectiveFallbackSource: KustomizeRoot # DeclaredDefault | KustomizeRoot | Canonical + declaredTypes: 2 # count of placement.byType entries in force + # what actually happened, bounded + placedResources: 14 # resources placed since observedRevision, appends included + canonicalTypes: 1 # distinct types that took the built-in path + refusedResources: 0 # resources NOT mirrored (see reasons below) + examples: # <= 3, most recent first + - type: apps/v1/deployments + path: team-a/apps/deployments/api.yaml + source: Canonical # ByType | Default | KustomizeRoot | Canonical + observedRevision: 9f3c1ab # the scanned commit the current half came from + observedTime: "2026-07-30T09:14:22Z" +``` + +`renderRootReason` and `effectiveFallbackSource` are the two fields that make the operator's judgement +inspectable per target rather than only aggregated in a metric, and `effectiveFallbackSource` answers +"what happens to a type I have not named?" in one word. An earlier draft called it `defaultSource` +with values `BuiltInCanonical | Declared`, which was wrong: it read `BuiltInCanonical` for a folder +whose files were going to the kustomize root, contradicting the `source` on its own example. + +**Two halves with different freshness, which must not be confused.** `renderRoot`, +`renderRootReason`, `effectiveFallbackSource` and `declaredTypes` are the **current** answer, derived +from the last repository scan and stamped with `observedRevision`. `placedResources`, +`canonicalTypes`, `refusedResources` and `examples` are **historical**, accumulated since that +revision. Placement is sparse, so a stable target may place nothing for weeks: a status built only +from write events would be empty for exactly the folder someone is asking about. The current half has +to come from the scan rather than from a placement having happened. `examples` is what makes the +historical half actionable: a type plus the path it landed on is the `byType` line to write. + +### Case 1: greenfield, nothing declared + +A target this operator bootstrapped. No `spec.placement`, no kustomization. Canonical is not a +fallback here, it is the layout, and status says so without implying a problem. + +```yaml +status: + layout: + renderRootReason: None + effectiveFallbackSource: Canonical + declaredTypes: 0 + placedResources: 23 + canonicalTypes: 4 + refusedResources: 0 + examples: + - type: v1/configmaps + path: team-a/configmaps/app-config.yaml + source: Canonical + observedTime: "2026-07-30T09:14:22Z" +``` + +Nothing to do. `canonicalTypes: 4` with `renderRootReason: None` is a tidy canonical folder. + +### Case 2: a kustomize overlay, nothing declared + +The folder has one `kustomization.yaml`. Every new file lands beside it and joins its `resources:` +list, so nothing falls back and no declaration is needed. + +```yaml +status: + layout: + renderRoot: overlays/production + renderRootReason: SingleKustomization + effectiveFallbackSource: KustomizeRoot + declaredTypes: 0 + placedResources: 6 + canonicalTypes: 0 + refusedResources: 0 + examples: + - type: v1/configmaps + path: overlays/production/debug-toolbox.yaml + source: KustomizeRoot + observedTime: "2026-07-30T09:15:02Z" +``` + +This is the case a metric alone reads ambiguously. `canonicalTypes: 0` plus a named `renderRoot` says +the folder decided, and that the operator agrees with it. + +### Case 3: brownfield, one type still missing a rule + +The user declared their ConfigMap bundle. Deployments were not covered, and no kustomization governs +the folder, so they went to the built-in path. + +```yaml +status: + layout: + renderRootReason: None + effectiveFallbackSource: Canonical + declaredTypes: 1 + placedResources: 9 + canonicalTypes: 1 + refusedResources: 0 + examples: + - type: apps/v1/deployments + path: team-a/apps/deployments/api.yaml + source: Canonical + - type: v1/configmaps + path: all.yaml + source: ByType + observedTime: "2026-07-30T09:16:41Z" +``` + +The two examples side by side are the whole point: one type is doing what was asked, one is not +covered, and the fix reads straight off the field. + +```yaml +placement: + byType: + v1/configmaps: "all.yaml" + apps/v1/deployments: "{namespace}/deployments/{name}.yaml" # the missing line +``` + +### Case 4: two overlays under one GitTarget (F8, silent today) + +`spec.path` covers `overlays/staging` and `overlays/production`. Neither root can be assumed, so new +files take the canonical path, where **no kustomization reaches them**. They are committed, they +look mirrored, and nothing applies them. + +```yaml +status: + layout: + renderRootReason: Ambiguous + effectiveFallbackSource: Canonical + declaredTypes: 0 + placedResources: 3 + canonicalTypes: 2 + refusedResources: 0 + examples: + - type: v1/configmaps + path: team-a/configmaps/cache.yaml + source: Canonical + observedTime: "2026-07-30T09:18:03Z" +``` + +`Ambiguous` is the word that does not exist today. The fix is the user's to choose: split the +GitTarget per overlay, or declare a `byType`/`default` that points inside one of them. + +### Case 5: a refusal, from a type the static gate cannot see + +`placement.default` is a bundle, and an operator-configured sensitive type (via +`--additional-sensitive-resources`) resolves onto it. The `Validated` gate cannot catch this, because +it only knows about core Secrets, so the write path refuses it fail-safe. Today that is a log line +and a resync counter; here it is on the object. + +```yaml +status: + layout: + renderRootReason: None + effectiveFallbackSource: DeclaredDefault + declaredTypes: 0 + placedResources: 4 + canonicalTypes: 0 + refusedResources: 1 + refusedReason: SensitiveAppend # most recent, from the counter's closed set + examples: + - type: v1/configmaps + path: all.yaml + source: Default + observedTime: "2026-07-30T09:20:11Z" +``` + +`refusedResources: 1` is a resource **absent from the mirror**, which is the one number on this +field that is unambiguously a problem to fix. + +## Question 3: `{kindLower}` + +A variable, not a function. Once `|lower` exists, `|upper`, `|trim` and `|replace` are each one PR +away, every one of them is new validation surface, and every one is a new way to render an empty path +segment. The spec's "keep it small" section already forbids it, and one variable answers the need: + +```yaml +placement: + default: "{kindLower}-{name}{sensitiveSuffix}" # deployment-simon.yaml +``` + +`{kindLower}` is `strings.ToLower` of a value we already hold, so it needs no new inputs and cannot +be empty when `{kind}` is not. The alternative, `{resourceSingular}` from discovery's +`SingularName`, is what `kubectl` uses and is more authoritative in principle, but it has to be +threaded from the type registry, needs an empty-value fallback, and I could not construct a real type +where it differs from the lowercased Kind. Not worth the plumbing unless "identical to kubectl" is a +guarantee we want to state. + +**Say plainly in the docs** that this layout is not identity-complete: two namespaces holding a +Deployment named `simon` render the same path, and they would land as two documents in one file. It +is the right layout for a single-namespace folder and the wrong one for a fleet folder, and the +operator cannot tell the two apart without being told. + +## Is the kustomize root inference? + +Yes, in the sense that it reads the repository and its answer can change without any Kubernetes +object changing. It differs from Option C in two ways that decide whether it stays: + +| | Option C | kustomize root | +|---|---|---| +| Reads | where similar documents live, an aggregate over content | one structural fact: is there exactly one supported root | +| Changes when | any document is added or removed | a `kustomization.yaml` is added or removed | +| Cost of ignoring it | a file in an unusual place | a file **nothing renders** | + +The third row is why it stays; the first two are why it is safe enough to keep. But the visibility +obligation is identical, and that is what `renderRoot` plus `renderRootReason` discharge: the +judgement becomes a field an operator can read, instead of a behavior they infer from where files +appeared. + +For the ambiguous case (F8), whether to refuse the write or write it loudly is **open**, and an +earlier draft calling it settled was a mistake. "An unrendered file is a better record than no file" +is a policy claim, and the opposite is at least as strong: a committed manifest nothing applies +manufactures a false appearance of convergence, which is the failure class this project ranks first. +The layout model probably decides it, because refusing is principled once a user has *asserted* a +single-root folder and the folder disagrees. + +What is not open: the signal does not belong on `placement_kustomization_entries_total`. That counter +counts **attempts** to register an entry +([placement_metrics.go](../../internal/git/placement_metrics.go)), and this is the case where no +attempt is made. It needs its own value on the refusal counter (if we refuse) or its own series (if we +write), never a third outcome on a counter about attempts. + +## What is built, and what is filed + +PR #291 shipped three things, and they are the ones that did not depend on any decision on this page: +the sibling-inference deletion, the `placements_total` / `placement_refusals_total` / +`placement_kustomization_entries_total` counters, and the namespace-transformer safety fix (a +kustomization naming a namespace other than the resource's own no longer renders the document as a +different object). `source="canonical"` kept its name, which is this page's Question 1 answered by +doing nothing. + +Everything else below is **decided and unbuilt**. It is filed as GitHub issues so it is legible +without reading this page, and the two correctness items are separated from the legibility ones +because they are worth different urgency: + +| Item | State | Why it is where it is | +|---|---|---| +| F10: register a declared path with the kustomization that governs it | **filed, correctness** | One `byType` line into a subdirectory silently produces a file nothing renders. Reachable today | +| Drop the `{version}` requirement from `IdentityCompletePlacementTemplate` | **filed, correctness** | It contradicts the versionless-path decision, and it is what makes any future spec default fail our own gate | +| `status.layout` | **filed** | The durable half of "what did the operator understand about this folder". Wants the layout model's vocabulary, so it follows it | +| Split `declared` into `byType` and `default`; unify the prose on "canonical" | **filed, legibility** | A catch-all quietly swallowing a type you meant to name looks identical to a rule working | +| `{kindLower}` | **filed, legibility** | Small, self-contained | +| Canonical path as a template constant | **filed, cleanup** | Removes the hand-written duplication; what a future default would reuse | +| `renderRootReason: Ambiguous` | **filed** | Belongs with `status.layout`, and the layout model decides the refuse-or-write policy | +| A CRD default for `placement.default` | **superseded** | [`gittarget-layout-model.md`](gittarget-layout-model.md): the primitive is wrong. `layout.kind` is the defaultable thing, because it names the structural rule instead of standing in front of it | + +The original build list follows, because each entry says *how* to build the thing and that is the part +an issue should not have to restate. + +Ordered by risk, smallest first. + +1. **Register a declared path with the kustomization that governs it (F10).** Replace + `governingKustomization`'s own-directory check plus write-scope special case with a walk up to the + nearest kustomization inside the write jail, so a `byType` or `default` path into a subdirectory of + a kustomize folder joins the root's `resources:` list instead of being committed outside every + render. A correctness fix, so it goes first, and the two cases stop differing by whether + render-root scoping happens to be in force. +2. **Metric values.** Split `declared` into `byType` and `default`; keep `canonical`. Unify the prose + on "canonical" and drop "built-in default" from [`architecture.md`](../architecture.md) and the + CRD comments (F1, F2). +3. **`{kindLower}`**, plus the single-namespace recipe and its identity caveat in + [`configuration.md`](../configuration.md) (F7). +4. **Drop the `{version}` requirement** from `IdentityCompletePlacementTemplate` for a non-narrowed + template, with a test for the versionless canonical shape being accepted. This is a bug fix on its + own terms and it unblocks any future spec default (F3, F4). +5. **Express the canonical path as a template constant** rendered through + `RenderPlacementTemplate`, and pin it byte-for-byte against `ResourceIdentifier.ToGitPath()` + across cluster-scoped, core, grouped and sensitive identities. Removes the hand-written + `canonicalPath` duplication and is what a future default would reuse. +6. **`status.layout`**, in the shape above, over the `MarkTargetRetention` seam: an epoch-scoped + per-target roll-up marked from the write path, enqueued on change, projected by the controller, + asserted by an envtest (F6). +7. **The ambiguous render root**: publish `renderRootReason: Ambiguous` now, and leave the + refuse-or-write policy to the layout model. Whatever it decides, the signal gets its own series + rather than a third outcome on a counter that counts entry *attempts* (F8). +8. **Docs**: the spec's resolution-ladder section, `configuration.md`, `interpreting-metrics.md` for + the new label values, and the `UPGRADING.md` entry extended with the metric-value split. + +**Not now, with the trigger written down:** a CRD default for `placement.default`. This was already +"not now" when the page was written, and it has since become "not this shape at all", because +[`gittarget-layout-model.md`](gittarget-layout-model.md) replaces the template with a declared +`layout.kind` and `Auto` is a safe default where a path never was. The freezing question (F5) is a +trade we could take; the shadowing question is not, and the layout model dissolves it rather than +answering it. `spec.expect.layout` stays out too, on the config-surface doc's own rule: publish the +observation before inventing the assertion. + +## Open questions + +- Should `status.layout.examples` prefer the **most recent** placements or the **most actionable** + ones (fall-backs and refusals first)? Most recent is simpler and cheaper; actionable is what + someone reading it wants. My inclination is actionable, capped at three, with fall-backs ranked + above successes. +- Does `refusedReason` belong on the field at all, or is the count plus the metric enough? It is the + one value here that can go stale in a way a count cannot. +- `{resourceSingular}` as a stated "identical to kubectl" guarantee: worth the discovery plumbing, or + is `{kindLower}` the end of it? diff --git a/docs/finished/attribution-fact-stream.md b/docs/finished/attribution-fact-stream.md index 4a122137..e5cea3bf 100644 --- a/docs/finished/attribution-fact-stream.md +++ b/docs/finished/attribution-fact-stream.md @@ -5,9 +5,8 @@ > [what is built](#what-is-built-and-what-the-code-settled) for what each piece landed as, and for > the numbers the implementation chose. Index: [`../INDEX.md`](../INDEX.md) > -> Supersedes the option analysis in -> [`attribution-wait-poll-vs-push.md`](../design/attribution-wait-poll-vs-push.md), which priced six ways to -> stop polling Redis. This record picks one and specifies it: the audit receiver publishes facts, +> Supersedes an option analysis that priced six ways to stop polling Redis (folded away; see +> `git log`). This record picks one and specifies it: the audit receiver publishes facts, > the watch side subscribes per type and holds them in memory, and the per-key Redis lookup is > deleted rather than optimized. > @@ -181,7 +180,7 @@ gitops-reverser:author:v2:audit:route:: The route infix stays exactly as it is today and for the same reason: an apiserver posts under one route, several `ClusterProvider`s naming one cluster share that route and therefore share its facts, and a fact from cluster A must never name the author of an object watched on cluster B. See -[`attribution-fact-identity.md`](../design/attribution-fact-identity.md). +[the attribution spec](../spec/attribution.md#the-scope-is-an-audit-route-and-a-type). The group/resource suffix is the new part, and it is what makes the fan-out meaningful. A process watching only `configmaps` and `deployments` follows two streams and never receives a fact for @@ -276,7 +275,7 @@ one map and hand a watch event on cluster B an author from cluster A. The rv-onl that bites hardest, because a resourceVersion is opaque and not unique across clusters, and the collection tier is where it bites most quietly, because a namespace name says nothing about which cluster it is in. The v1 fact keys already carry the route for this reason -([`attribution-fact-identity.md`](../design/attribution-fact-identity.md)), and the same dimension has to +([the attribution spec](../spec/attribution.md#the-scope-is-an-audit-route-and-a-type)), and the same dimension has to travel through the waiter candidate keys and the collection scope match, not only the four maps above. A test that stores identical `(group/resource, uid, rv)` facts under two routes and resolves each from its own is the one that proves it, and it belongs with the index rather than the @@ -520,7 +519,7 @@ makes it safe. Namespace and selector narrow the scope. Precedence keeps anything with its own fact out. And the window is short, because of the deletion-as-intent rule in -[`deletecollection-attribution-expander.md`](../spec/deletecollection-attribution-expander.md): the +[the attribution spec](../spec/attribution.md#1-deletion-is-attributed-at-intent-time): the removal being attributed happens at **delete-request time**, when `deletionTimestamp` is set, not at whatever later moment finalization completes. Finalizers do not stretch the window. So the fact's own `stageTimestamp` plus a small allowance for skew and delivery is enough, and it can be far @@ -613,7 +612,7 @@ index instead of a shared key. **Facts that never resolve.** Some events produce no audit fact at all, so no wait and no transport can name their author. They still spend the grace window and ship unresolved. That is unchanged, and it is the population that keeps -[the circuit breaker](../design/attribution-wait-poll-vs-push.md#option-c-circuit-break-a-route-that-has-never-resolved-anything) +the circuit breaker for a route that has never resolved anything worth building separately — see [when a removal should stop waiting](../design/attribution-removal-wait-options.md). @@ -838,7 +837,7 @@ restart today, which is what keeps the delta small. | Subscribe and unsubscribe a type as watches come and go | [`target_watch.go`](../../internal/watch/target_watch.go) | done | | Delete the fact key builders, `SET`/`GET` paths, and the SCAN-based size gauge. The file goes with them: what survived is the fact shape and the result taxonomy, in [`author_fact.go`](../../internal/queue/author_fact.go), and the shared key helpers, in [`key_prefix.go`](../../internal/queue/key_prefix.go). `attribution_fact_index_size` survives too, now a field read on the sweep rather than a SCAN of the whole keyspace | `attribution_index.go`, deleted | done | | Delete the collection expander: `RecordDeleteCollectionFacts`, `storeDeleteCollectionFacts`, and their tests. `deleteCollectionItems` survives, reduced to uids only: the publish side still needs the body parsed once, to build the uid SET one fact carries. Nothing rebuilds N per-object facts from one request | `attribution_index.go` and `attribution_index_deletecollection_test.go`, deleted | done | -| Retire §5 and §8 of the expander spec; §2, the deletion-as-intent render rule, is untouched and still binds | [`deletecollection-attribution-expander.md`](../spec/deletecollection-attribution-expander.md) | done | +| Retire §5 and §8 of the expander spec; the deletion-as-intent render rule is untouched and still binds. The spec has since been folded into [`attribution.md`](../spec/attribution.md), which carries that rule as §1 | `deletecollection-attribution-expander.md`, deleted | done | | Add the transport selection flag, narrow the `--redis-addr` validation, and reject in-memory with more than one replica | [`cmd/main.go`](../../cmd/main.go), [`configuration.md`](../configuration.md) | done | | Correct the stale "hard dependency in every mode" comment | [`redis_store.go`](../../internal/queue/redis_store.go) | done | | Document the three new counters and the replaced result label | [`interpreting-metrics.md`](../interpreting-metrics.md) | done | diff --git a/docs/finished/redis-key-schema-v3.md b/docs/finished/redis-key-schema-v3.md index 5a363d7d..530c4590 100644 --- a/docs/finished/redis-key-schema-v3.md +++ b/docs/finished/redis-key-schema-v3.md @@ -25,7 +25,7 @@ > [`internal/queue/redis_store.go`](../../internal/queue/redis_store.go), > [`internal/watch/author_resolver.go`](../../internal/watch/author_resolver.go), > [CommitRequest authorship from admission](../spec/commitrequest-admission-authorship.md), -> [deletecollection attribution & deletion-as-intent](../spec/deletecollection-attribution-expander.md), +> [attribution](../spec/attribution.md), > [watch event ordering & attribution grace](../facts/watch-event-ordering-and-attribution-grace.md), > [watch-first ingestion architecture](watch-first-ingestion-architecture.md). @@ -183,9 +183,12 @@ A given `(uid, rv)` had **exactly one writer** — that RV exists *because* of t - A **delete** writes `:last` = the deleter; the `DELETED` watch event (RV ≠ any write RV) resolves the deleter. Correct. -- A **deletecollection** expands one `:last` per member = the actor; each per-object removal joins it - (see [deletecollection §3](../spec/deletecollection-attribution-expander.md)). The whole reason that design - pinned itself to the uid-only variant — body RV ≠ removal RV — is just "use `:last`" here. +- A **deletecollection** expands one `:last` per member = the actor; each per-object removal joins it. + The whole reason that design pinned itself to the uid-only variant — body RV ≠ removal RV — is just + "use `:last`" here. **Superseded:** there is no expansion any more. A collection delete is published + as ONE fact describing the collection, which every removal in its scope joins by uid membership or + by scope; see [the attribution spec](../spec/attribution.md). The uid-not-RV argument is the half + that survived, and it is why the uid tier outranks the scope tier. - A **burst** (author₁ rv₁, author₂ rv₂): `:`=author₁, `:`=author₂ (both precise), `:last`=author₂. Watch events for rv₁ and rv₂ each hit their exact key → both precise. This is the precision an earlier single-key sketch gave up; the grouped scheme keeps it. diff --git a/docs/future/flux-maintainer-review-status-and-config-model.md b/docs/future/flux-maintainer-review-status-and-config-model.md index f4c0ea5b..066b2ecb 100644 --- a/docs/future/flux-maintainer-review-status-and-config-model.md +++ b/docs/future/flux-maintainer-review-status-and-config-model.md @@ -14,6 +14,15 @@ > **F6** (`spec.suspend`, `spec.interval`, reconcile-request annotation), **F9** (the stored > `scope: Namespaced` status-write question), **F10** (CommitRequest lifecycle). Those are §4's > "Then (API surface)" block and are a separate change. +> +> **That change now has a shape.** F6, F10, F12's reference-type nit and §3's pushbacks are sequenced +> together with the new-file-placement redesign in +> [`gittarget-api-wave.md`](../design/gittarget-api-wave.md), on the principle that the folder is +> described on the GitTarget and the connection describes only the connection. Two of this review's +> findings change that design rather than accompanying it: `spec.mode: Observe` (config-surface B1) +> becomes the way a layout is adopted safely, and `spec.interval` is what keeps the layout's +> scan-derived status fresh. F9 is deliberately kept **outside** the wave, because its answer +> constrains the enum work and should be known before anything is planned around it. > Stance: reviewed as if this API were proposed for the GitOps Toolkit, with Flux's own > source (`external-sources/flux/`) and kstatus (`sigs.k8s.io/cli-utils/pkg/kstatus`) as ground > truth rather than recollection. @@ -629,7 +638,9 @@ criticism. 3. **F4** — delete `Reconciling`/`Stalled` rather than writing them False. 4. **F7** — wire an `EventRecorder`; emit on every terminal outcome and every push failure. -**Then** (API surface — do the breaking ones while still `v1alpha3`): +**Then** (API surface — do the breaking ones while still `v1alpha3`). These are the wave, and they +are now sequenced with the layout model in +[`gittarget-api-wave.md`](../design/gittarget-api-wave.md) rather than scheduled from here: 1. **F6** — `spec.suspend` on GitTarget/WatchRule/ClusterWatchRule/GitProvider; `spec.interval` on GitProvider at minimum; jitter the requeue; `reconcile.configbutler.ai/requestedAt` + @@ -637,8 +648,10 @@ criticism. 2. **F12** — ~~decide `PruneMode` casing **now**~~ (done, pre-release); trim printer columns; unify ObjectMeta tags. 3. **F10** — CommitRequest lifecycle (TTL or ownerRef) and the `delete` verb. -4. **F9** — verify the `scope: Namespaced` status-write path on the minimum supported Kubernetes - version. +4. ~~**F9**~~ — **not in this block.** F9 is deliberately kept OUTSIDE the wave, because its answer + constrains the enum work and should be known before anything is planned around it. Verify the + `scope: Namespaced` status-write path on the minimum supported Kubernetes version as its own + change, ahead of the wave. --- diff --git a/docs/installing-apps-as-krm.md b/docs/installing-apps-as-krm.md index 1d4176ef..1cfb9451 100644 --- a/docs/installing-apps-as-krm.md +++ b/docs/installing-apps-as-krm.md @@ -153,9 +153,9 @@ full `GitProvider` / `GitTarget` / `WatchRule` model. ## Where the document lands in Git -A brand-new document follows the layout your folder already uses (sibling -inference), or the `GitTarget`'s declared placement policy. Absent any existing -convention, the cold-start default path is: +A brand-new document goes where the `GitTarget`'s declared placement policy says, or +beside the folder's `kustomization.yaml` when the folder has exactly one. Otherwise the +built-in path is: ```text {GitTarget path}/{namespace}/{group}/{resource}/{name}.yaml diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index ba556407..04c8a83c 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -79,6 +79,9 @@ signals. Background: [architecture.md → Git Write Architecture](architecture.m | `git_operations_total` | counter | — | Events that produced Git work in a flush. | | `objects_written_total` | counter | — | Objects that resulted in a file write in a flush. | | `resync_sweep_deletes_total` | counter | `group`, `version`, `resource` | Managed documents deleted by mark-and-sweep resyncs. Steady-state watch deletes do not increment this. | +| `placements_total` | counter | `source`, `disposition`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new document written at a resolved path. `source` is `declared` / `kustomize_root` / `canonical`; `disposition` is `new_file` / `appended`. | +| `placement_refusals_total` | counter | `reason`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new resource the writer declined to place. Every increment is a resource **absent** from the mirror. | +| `placement_kustomization_entries_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | The `resources:` entry a newly placed file needs: `added`, `no_change`, `failed`. | | `branch_worker_queue_depth` | gauge | `provider_namespace`, `provider_name`, `branch` | Pending + in-flight + committed-but-unpushed work; reads 0 only when the worker has fully drained. | | `target_reconcile_completed_total` | counter | `gittarget_namespace`, `gittarget_name`, `trigger` | One increment per completed watch-recovery pass (streaming-snapshot resync applied, or cursor-backed resume). | | `resync_background_failures_total` | counter | `gittarget_namespace`, `gittarget_name` | Rule-change resyncs whose apply failed/timed out **after** enqueue (otherwise only logged). | @@ -151,6 +154,71 @@ the cluster and a scoped/full resync applies. This is not the steady-state delet sum by (group, version, resource) (rate(gitopsreverser_resync_sweep_deletes_total[1h])) ``` +### New-file placement + +Placement runs **only** for a resource with no document in Git yet; everything already written is +edited in place, forever. So these counters are sparse by nature — a busy target can go a day without +one — and a zero rate is the steady state, not a broken exporter. + +`source` answers "why did it land there?", which is the question a folder cannot answer: + +| `source` | Means | Needs attention? | +| --- | --- | --- | +| `declared` | a `spec.placement.byType` or `.default` template matched | no — this is what you asked for | +| `kustomize_root` | the folder is governed by exactly one supported kustomization, so the file went beside it and joined its `resources:` list | no — the folder's own structure decided | +| `canonical` | nothing else applied, so the built-in `{namespace}/{group}/{resource}/{name}.yaml` path was used | **maybe** — see below | + +**Which types are falling back, and in which target?** Each series is a candidate for one +`placement.byType` line. This is the signal that replaced sibling inference: the operator no longer +guesses a hand-authored layout from the folder, so this is how you learn a layout needs declaring: + +```promql +sum by (gittarget_namespace, gittarget_name, group, version, resource) ( + increase(gitopsreverser_placements_total{source="canonical"}[24h])) +``` + +Canonical is not an error. For a target whose repository the operator bootstrapped, it is the whole +layout and always will be. It is worth acting on when the folder has a convention the operator was not +told about — the file lands somewhere tidy but not where the rest of that type lives. + +**Is a bundling policy actually bundling?** `disposition="appended"` proves documents are joining an +existing file rather than each getting their own. It should only ever appear with +`source="declared"`; the fallbacks never append: + +```promql +sum by (source, disposition) (increase(gitopsreverser_placements_total[24h])) +``` + +**Are we failing to mirror resources?** Every refusal is a resource that is **not** in Git. The write +is retried on the next event or resync, so a sustained rate is a policy to fix rather than a blip: + +```promql +sum by (reason, gittarget_namespace, gittarget_name, resource) ( + increase(gitopsreverser_placement_refusals_total[1h])) +``` + +| `reason` | What to fix | +| --- | --- | +| `invalid_path` | a declared template that renders outside `spec.path` or without a YAML suffix | +| `sensitive_append` | a template that is not identity-complete, so two Secrets collide on one path | +| `plaintext_onto_encrypted` | a template routing a plaintext resource at a file holding SOPS data | +| `mixed_sensitivity_new_file` | a bundling `default` catching both a sensitive and a plaintext resource | +| `multi_document_target` | the resolved file holds a document the writer cannot account for, so it will not overwrite it | +| `unclassified` | a refusal shape newer than this table — report it | + +**The one that looks fine in the folder.** A new file whose `resources:` entry could not be added is +committed and never built by kustomize: it is in Git, it looks mirrored, and nothing applies it. This +should be zero: + +```promql +sum by (gittarget_namespace, gittarget_name) ( + increase(gitopsreverser_placement_kustomization_entries_total{outcome="failed"}[1h])) +``` + +Placement counters carry `gittarget_*` label keys rather than bare `namespace`/`name` for the +pod-scrape reason described above, and they deliberately carry **no path or resource-name label** — both +are unbounded, and both are in the log line at the write site. + --- ## Audit attribution (optional) diff --git a/docs/spec/README.md b/docs/spec/README.md index cef1ba89..4933e1f9 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -25,6 +25,7 @@ If you change one of these behaviours, change the document in the same commit. | Spec | What it pins | |---|---| +| [`attribution.md`](attribution.md) | **how a commit gets its author**: deletion-at-intent, the publish and join halves, the tier ladder, the audit-route partition, and what the metrics can and cannot answer | | [`current-manifest-support-review.md`](current-manifest-support-review.md) | the manifest store, plan/apply/flush, and the all-or-nothing folder claim | | [`contextual-namespace-and-kustomize-folder-editing.md`](contextual-namespace-and-kustomize-folder-editing.md) | kustomize graph-aware namespace inference; the supported subset | | [`reconcile-via-watchlist-mark-and-sweep.md`](reconcile-via-watchlist-mark-and-sweep.md) | initial reconcile; **no bookmark, no sweep** | diff --git a/docs/spec/attribution.md b/docs/spec/attribution.md new file mode 100644 index 00000000..7f66c410 --- /dev/null +++ b/docs/spec/attribution.md @@ -0,0 +1,611 @@ +# Attribution: how a commit gets its author + +> **spec**: current behavior. The code depends on this document; change one, change the other. +> Index: [`../INDEX.md`](../INDEX.md) +> +> This is the single reference for attribution. It replaces six design records that each described +> one part of it while it was being built: the publish/join reference, the fact-identity proposal, +> the deletion-intent-actor fix, the switchover findings, the metric-surface proposal, and the +> `deletecollection` expander spec whose subject no longer exists. Their reasoning is in +> `git log`; the one record kept in full is +> [`../finished/attribution-fact-stream.md`](../finished/attribution-fact-stream.md), which argues +> why facts are a per-type stream rather than a keyspace. +> +> What is still **open** is not here. It is ranked in +> [`../design/open-asks-priority.md`](../design/open-asks-priority.md), and the one decision with +> its own record is [`../design/attribution-removal-wait-options.md`](../design/attribution-removal-wait-options.md). +> +> Related: [`../facts/watch-event-ordering-and-attribution-grace.md`](../facts/watch-event-ordering-and-attribution-grace.md), +> [`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md), +> [`../attribution-setup-guide.md`](../attribution-setup-guide.md), +> [`../interpreting-metrics.md`](../interpreting-metrics.md). + +## The shape: two halves that never call each other + +Attribution has two halves. One turns an audit event into a **fact**; the other turns a watch event +into an **author** by finding a fact. They meet only in the index, through the keys a fact was filed +under. + +**Neither half branches on the type.** Every decision is made on the **verb** of the request and on +which fields the event happens to carry. Why that is a requirement rather than a coincidence is +[no branch depends on the type](#no-branch-anywhere-depends-on-the-type). + +The governing rule, which every choice below defers to: **a wrong author is worse than no author.** +When no usable fact is found the commit ships with the explicit unresolved author +(`unknown (attribution unresolved)`), never with a guess. + +## 1. Deletion is attributed at intent time + +This is a render-layer rule rather than an attribution mechanism, and it comes first because +everything about attributing a removal depends on it. + +Deletion is two distinct facts, and conflating them is what made an earlier design hard: + +- **Deletion intent.** The API server accepted a `DELETE` or `DELETECOLLECTION` and marked the object + with `deletionTimestamp`. The object's *desired* existence is now absent. +- **Final removal.** The object disappeared from the API after grace and finalization + completed. That is runtime cleanup, and it can take five seconds or three days. + +The Git manifest we commit is meant to be re-applied, and it already strips `deletionTimestamp` +and `deletionGracePeriodSeconds` ([`sanitize.go`](../../internal/sanitize/sanitize.go)) because they +are server-owned runtime metadata, not desired state. If those fields are not desired state, then an object that *only* +differs by having them is, as desired state, gone. + +> **A resource with `deletionTimestamp` set is treated as logically absent from the intent tree.** +> The first observation of `deletionTimestamp` (or a `DELETED` event) removes the resource from Git +> and attributes the removal to the actor who **requested** the deletion. Later finalizer updates and +> the eventual `DELETED` event do not create additional Git changes; they are runtime cleanup, +> observed operationally only. + +So when Alice runs `kubectl delete widget foo`, the Git change is `- widgets/foo.yaml`, authored +"Alice", now. It is not a commit that sets `deletionTimestamp` on the file, and not a commit +deferred until finalization finishes. + +**Why this is correct rather than a shortcut.** It does not bypass finalizers: removing the file is a +statement about desired state, not an act on the cluster, and the object stays in `Terminating` until +its controllers are done. `deletionTimestamp` is monotonic and terminal, so "logically absent" can +never flip-flop. And the repository's invariant stays clean: **a file present means the resource is +intended to exist.** + +**Why immediate removal is the reversible default.** Richer behavior can be added on top of that +invariant later: `.deletions/` records, commit-message trailers, status reporting. The reverse is +much harder: if `Terminating` objects stayed in the main tree, consumers would learn that "file +exists implies object still in the API", and removing them later would silently change what the +repository means. + +**In code.** [`routeLiveTargetWatchEvent`](../../internal/watch/target_watch.go) routes an event +whose object has `GetDeletionTimestamp() != nil` as a **Delete** regardless of whether the watch type +is `MODIFIED` or `DELETED`; the Delete path emits no body, so the file is removed. This is computed +*before* the operation is matched against the rule's verbs, so a WatchRule that excludes deletes +consistently does not act on the logical delete either. Later finalizer-clearing events re-issue +"delete X" against an already-absent path, and the writer diffs them to a no-op. + +**One operational caveat.** Logical absence is a Git statement, and it must not make us blind to a +stuck deletion. A long-`Terminating` object is surfaced through status, metrics and diagnostics, +never by keeping its desired-state file. Recording *who cleared the finalizer* is legitimate as a +diagnostic signal, and never as the Git author. + +## 2. The publish side: audit event to fact + +One audit event in. Zero or one fact out, filed under one to three keys. + +```mermaid +flowchart TD + A[Audit event, ResponseComplete] --> B{objectRef present
with a resource?} + B -->|no| X[No fact] + B -->|yes| C{user resolvable?} + C -->|no| X + C -->|yes| D[identity = objectRef namespace, name, uid
then backfill what is missing from the body] + D --> E{verb is
deletecollection?} + + E -->|yes| F[COLLECTION fact
drop uid, name, rv
keep selector from requestURI
keep uid set from the body, if any] + E -->|no| G{identity has
a name?} + G -->|no| X + G -->|yes| H[OBJECT fact
namespace, name, uid, rv, verb, author] + + F --> K[file under: collection namespace] + H --> L{strongest key it has
FIRST MATCH WINS} + L -->|uid and rv| M[file under: exact uid+rv
AND latest uid] + L -->|uid, no rv| N[file under: latest uid] + L -->|rv, no uid| O[file under: rv] + L -->|name, no uid, no rv| P[file under: namespace+name] + L -->|none of these| X + + style X fill:#7f1d1d,color:#fff + style K fill:#14532d,color:#fff + style M fill:#14532d,color:#fff + style N fill:#14532d,color:#fff + style O fill:#14532d,color:#fff + style P fill:#14532d,color:#fff +``` + +Two gates drop an event entirely: no resource, and no resolvable name on a non-collection verb. A +collection request is exempt from the name gate because it names no object by nature, and that is +the one place the verb changes which gate applies. + +A third gate is the fact's whole reason to exist: **no resolvable user, no fact**. It is the one +field the wire contract requires, and the read side enforces it too. `AuthorFact.UnmarshalJSON` +refuses an entry naming nobody, which lands on +`gitopsreverser_attribution_fact_stream_decode_errors_total` rather than being half-absorbed. So a +fact in the index always names an actor, and the metrics can read attribution coverage off the tier +alone. The event that produced no fact is not lost either: it is counted `no_attribution_fact` on +`audit_events_total`, where its type and verb are still in hand. + +The body backfill is why the name gate is survivable. `objectRef` alone often lacks the name or the +uid; `IdentityFromAuditEvent` fills what is missing from the request or response object, preferring +the request object for a delete and the response object otherwise. What the event carries in its body +therefore decides which keys the fact ends up with, and the type has nothing to do with it. + +### Filing picks one branch + +The fact keeps every field it recovered, but it is **filed** under one branch only. `file` is a +switch on the strongest key present, and the first matching case wins: a fact with a uid is not also +filed under its name or its resourceVersion, even though it has them. + +The reason is memory. A watch event always knows its object's uid, so it always asks a uid tier +first, and a uid-keyed fact always answers there. A second copy of that same fact under its name +would never be the one read. Storing it anyway costs the entry on every replica following the type, +for the whole TTL, and again on every restart replay, and buys nothing. + +The one branch that files more than once is the uid case: `exact` (when the fact also has a +resourceVersion) serves creates and updates, `latest` serves removals, and the two answer different +questions about the same object. A fact whose own verb is a **removal** takes a third structure, the +sticky removal pointer, which answers a question no later fact can: who asked for the deletion. + +So the rule is: keep every field, file under exactly the keys a query could reach you by. + +### A collection delete is one fact + +`kubectl delete configmaps --all -n team-a` produces **one** name-less `deletecollection` audit +event, while the watch delivers **N** independent events, one per object. State is correct with no +special code. Attribution is not free, and it is served by publishing one fact that describes the +**collection** (actor, type, namespace, the selector the request URI expressed, the stage timestamp, +and the set of uids the API server named when it sent a body), which every removal in that scope +joins. + +**A collection member must join by uid, not by resourceVersion.** The watch event that removes the +file carries the object's current RV (the RV the deletion stamped), while the response body lists +each removed object at its **pre-delete** RV, a different number for the same object. The only field +stable and identical on both sides is `metadata.uid`. That is why the uid tier sits above the scope +tier. + +**When there is no body**, the join proceeds on scope alone: namespace, selector and window. An +aggregated or external API server, a `Metadata`-level policy, or +`--audit-webhook-truncate-enabled` can all report a collection delete with no usable list. Scope +matching would be dangerous if it were unbounded, and three things bound it: + +- **Namespace and selector narrow it.** The selector is the intent the actor *stated*, read off the + request URI, and it is present even when the body is not. An empty selector matches everything of + the type in the namespace, which is exactly what `--all` means. +- **Precedence keeps anything with its own fact out.** The scope tier is the weakest evidence the + join has, so it is reached only when every more specific tier missed. An unrelated + `kubectl delete configmap x` in the same window is claimed by its own fact and never reaches it. +- **The window is short**, because of the intent rule in §1. The removal happens at delete-request + time, so the window only has to cover audit batching plus clock skew: + `--author-attribution-collection-window`, 30s by default, against a fact TTL of ten minutes. + Attribution chasing the eventual removal could not have offered that. + +The result is the reverse of a degradation: the cases most likely to send no body are the ones that +gain most. + +## 3. Where the facts live + +### The scope is an audit route and a type + +Facts are partitioned by `factScope{route, groupResource}`. Both halves are load-bearing. + +**The route** is where a cluster's audit events arrive, and it is **declared** rather than inferred +from an object's name: + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: tenant-acme-delegating # the name humans read and GitTargets reference +spec: + attribution: + auditRoute: prod-eu-1 # the route this cluster's audit events arrive on +``` + +`spec.attribution.auditRoute` is the `` segment the API server's webhook URL ends in +(`/audit-webhook/`), or (when several logical clusters share one backend) the value of the +audit-event annotation named by `--author-attribution-audit-route-annotation-key`. It defaults to +`metadata.name` through [`ClusterProvider.AuditRoute()`](../../api/v1alpha3/clusterprovider_types.go), +so no caller sees the empty case and every single-provider install keeps working unchanged. + +It exists because an API server posts audit to exactly **one** route. Several `ClusterProvider`s may +name one physical cluster, and without a declared route only one of those names is ever fed while +every other one authors `unknown (attribution unresolved)`. Carrying the same route lets them share +one cluster's facts; carrying different routes means a fact from cluster A can never name the author +of an object watched on cluster B, which is also what keeps a cloned or restored cluster separate +from its origin. It is **mutable**, unlike `spec.kubeConfig`: repointing it changes which partition +is read, not which cluster is mirrored, so correcting a typo does not require deleting the object. + +Ingestion makes **no Kubernetes API call**. The route is a partition name rather than a claim about +an object that must exist, so an audit batch that arrives while a provider is being created is +stored rather than 404'd (the API server does not retry a 404). The authentication boundary is +unchanged and lives elsewhere: the audit server requires a client certificate signed by the audit CA +(`RequireAndVerifyClientCert`). The existence check was never an authorization check; see +[`../design/multi-source-audit-ingress-hardening.md`](../design/multi-source-audit-ingress-hardening.md), +which owns the remaining ingress work. + +**A misrouted provider is loud.** The resolver tracks, per route, whether attribution has ever +resolved and how many events have gone unresolved since. A route that has never resolved and produces +five unresolved events in a row logs once, naming the fix rather than the symptom. The threshold is +not one because a lone miss is ordinary; a run of them with nothing ever matched is the signature of +a route nobody writes to. One line per route per process, because the condition is a configuration +mistake and repeating it per event would bury it. + +**The type** suffix is what makes the fan-out meaningful: a process watching only `configmaps` and +`deployments` follows two streams and never receives a fact for anything else. It is a **partition** +and never a decision: no code reads it to choose a behavior. + +### The stream, and the index + +One stream per `(audit route, group/resource)`: + +```text +gitops-reverser:author:v2:audit:route:: +``` + +The audit receiver decodes one `EventList` per request and appends **one batched entry per type**. +Every process issues a single blocking read across every stream in its subscription set, with a +per-stream cursor held in memory, and applies what it reads into a bounded in-memory index. There are +**no consumer groups**: a consumer group distributes entries between consumers, and this is a +fan-out, and every process watching a type needs every fact for that type. The subscription set changes +at runtime as WatchRules start and stop covering types, and the reader re-issues within one block +period rather than tearing the connection down. + +Retention is the TTL. Each append carries a `MAXLEN` bound so a hot type cannot grow without bound, +and a periodic trim drops entries past the retention horizon. `--author-attribution-ttl` (ten +minutes) bounds stream retention and the in-memory index together. **Expiry is decided on read**, so +an aged-out fact is never joined merely because the sweep has not run; the sweep bounds memory, never +correctness. A trim gap is reported only when the follower was behind, so ordinary ageing +of an entry a caught-up follower already read is not counted as loss. + +The index is bounded by `--author-attribution-max-facts` and +`--author-attribution-max-facts-per-type`, and evictions are counted. A restart re-warms from one +TTL of stream retention rather than starting blind, which is what makes a rolling upgrade safe. + +### The transports + +The transport sits behind a two-method seam (`FactPublisher` / `FactFollower`) with one conformance +suite over both implementations. `--author-attribution-transport=redis` is the default; +`memory` runs attribution entirely in-process with no Redis, and is **refused with more than one +replica**. Nothing in the downward API reports a Deployment's replica count, so the chart templates +`--replica-count` in from `.Values.replicaCount`, and the chart's own guard fails a multi-replica +install with the in-memory transport outright. + +`gitopsreverser_attribution_transport_info{transport}` says which contract the other metrics are +being read under. + +## 4. The join side: watch event to author + +One watch event in. It asks the index for the strongest fact about this object, waiting up to the +grace for one to arrive. + +```mermaid +flowchart TD + A[Watch event] --> B[scope = audit route + group/resource] + B --> C{scope known?} + C -->|no| Z[absent: committer-authored] + C -->|yes| AA{a removal, and a sticky
removal pointer for its uid?} + AA -->|yes| AB[delete_sticky] + AA -->|no| D{a fact under
this uid and rv?} + + D -->|match| E[exact] + D -->|no match| F{is this a removal?} + + F -->|yes| G{uid in a collection's
uid set?} + G -->|yes| H[deletecollection_body_uid] + G -->|no| I{latest uid
is a DELETE fact?} + I -->|yes| J[latest: the object's own delete] + I -->|no, it is a write| K[hold it as a fallback] + K --> L{name tier holds
a DELETE fact?} + L -->|yes| M[name] + L -->|no| N{fallback held?} + N -->|yes| O[latest: last writer] + N -->|no| P{collection covers
this scope + selector?} + P -->|yes| Q[deletecollection_scope] + P -->|no| R + + F -->|no| R{rv-only hatch?} + R -->|match| S[resource_version] + R -->|no match| T{name tier?} + T -->|match| U[name] + T -->|no match| Z + + style Z fill:#7f1d1d,color:#fff + style AB fill:#14532d,color:#fff + style E fill:#14532d,color:#fff + style H fill:#14532d,color:#fff + style J fill:#14532d,color:#fff + style M fill:#14532d,color:#fff + style O fill:#166534,color:#fff + style Q fill:#166534,color:#fff + style S fill:#166534,color:#fff + style U fill:#14532d,color:#fff +``` + +### The tiers, strongest first + +| Tier | Key | `tier` label | What it asserts | +|---|---|---|---| +| delete, sticky | uid (a delete fact, sticky) | `delete_sticky` | who asked for this object's deletion | +| exact | uid + rv | `exact` | this actor produced this exact version | +| collection uid | uid in a collection's set | `deletecollection_body_uid` | the API server said this request deleted this object | +| latest, delete | uid | `latest` | this object's own delete fact | +| name, delete | namespace + name | `name` | this object's own delete fact, when it has no uid | +| latest, write | uid | `latest` | who last wrote it; a fallback for a removal | +| collection scope | namespace + selector + window | `deletecollection_scope` | a collection request covering it was made | +| rv-only | rv | `resource_version` | a fact with an rv but no uid | +| name | namespace + name | `name` | the only key an aggregated write has | +| absent | none | `absent` | committer-authored | + +Who the evidence named is the separate `actor_kind` label (`user` / `serviceaccount` / `none`), so +every row above can be asked about either kind of actor. `tier` names the evidence and `actor_kind` +matches how `commits_total` already models the same distinction; an earlier `result` label conflated +the two and is gone (migration in [`../UPGRADING.md`](../UPGRADING.md)). + +### Three rules that are easy to miss + +**A fact about a DELETION may not be replaced by a fact about a WRITE.** Every ordinary structure +here is last-writer-wins, and a finalizer patch's fact carries the resourceVersion the **deletion** +stamped, so it lands under the deleter's exact key and the deleter's uid key alike. That used to +attribute a finalized deletion to the cleanup controller rather than to the human who asked for it. +A removal fact therefore also takes a **sticky removal pointer**, keyed by uid, that only another +removal fact may fill, and a removal consults that pointer before the exact tier. It is the one +structure the TTL does not bound: a uid is unique across space and time, so the statement can never +be superseded, and its horizon is the index's caps instead. It is strictly uid-keyed, because the +same stickiness on the name tier would be a defect: a name is reused after a delete and recreate. + +**A removal never returns on a write fact without looking further.** The per-object tiers are +last-writer-wins, so for a removal they hold whoever last **edited** the object, which is not who +deleted it. Such a match is held as a fallback while the search continues, and the caller keeps +waiting for delete evidence until the grace expires. A fact about the deletion, filed under any key, +ends the wait immediately. Waiting never costs an attribution: the worst case returns exactly what +returning early would have returned, one grace later. + +**An exact-capable event may not fall through to the removal tiers.** A create or update presents the +resourceVersion its own write produced. If the exact tier misses, the `latest` pointer may name an +older, different author, so the lookup skips straight to the rv hatch and the name tier. The gate is +one-directional, and only one of the two tiers is gated: the exact tier is tried for *any* query +carrying a uid and a resourceVersion, a removal included. What an exact-capable event may not do is +the reverse, reaching the tiers below. + +### The wait + +The two halves are racing, and the watch side reliably wins. The API server batches audit deliveries +(`--audit-webhook-batch-max-wait`), while the watch event is streamed, so by the time a watch event +needs an author its fact is usually still inside the batch window. The first lookup is a +near-guaranteed miss. That is the whole reason a grace window exists +(`--author-attribution-grace`, 3s by default). + +So the resolver does not ask repeatedly. It arms a signal, looks once, and then sleeps until either a +fact that could match it arrives or the grace runs out. + +```mermaid +sequenceDiagram + participant W as watch shard + participant R as waiter registry + participant I as index + participant F as fact follower + + W->>R: register(waiterKeys): one entry per tier this query could match + Note over W,R: registered BEFORE the read, so a fact
landing in the gap still wakes it + W->>I: Lookup + I-->>W: absent + W->>W: select { waiter.ch | ctx.Done | timer.C } + + F->>I: apply fact, file under its keys + I->>R: wake(keys the fact filled) + R-->>W: signal + W->>I: Lookup again + I-->>W: resolved + Note over W: defer unregister, whatever the outcome +``` + +Registering **before** the read is what closes the race a poll loop can only paper over by looking +again: a fact delivered between the register and the read wakes a waiter that is already listening. +There is no Redis call on this path. The fast case is a map read; the waiting case is a channel +receive. + +The registry is `map[factWaiterKey]map[*factWaiter]struct{}`: candidate key to the set of resolvers +blocked on it. That shape is the fan-out. One resolver registers under **several** keys, one per tier +its event could resolve through, and one applied fact wakes every resolver registered under any of +the keys that fact filled. It is a many-to-many join done through an index rather than a broadcast, +so a fact never touches a resolver it could not have answered. + +Four details do real work: + +- **`chan struct{}` with buffer 1.** The signal carries no payload, because the payload is the index + itself: the woken resolver re-reads it. Buffering one means a signal sent while the resolver is + mid-recheck is still there when it comes back around, so it is not lost. +- **Non-blocking send.** `wake` does `select { case ch <- struct{}{}: default: }`, so the goroutine + applying facts is never slowed by a resolver that has not looked yet, and a second signal on an + already-signaled waiter is dropped. One pending wake-up is enough, because the resolver re-reads + everything rather than consuming a queue of events. It also makes the send safe to do while holding + the registry lock, since it cannot block. +- **`select` on three cases.** The resolver waits on the waiter, `ctx.Done()`, and the grace timer + together, so shutdown and the deadline are not special paths. +- **`defer unregister`.** Registration is undone on every exit, including the ones that return early. + The registry exposes a `len()` purely so a test can assert a resolver left nothing behind. + +The loop around the `select` matters too, because a wake-up is only a hint. The resolver re-runs the +whole lookup, and if what arrived was not good enough (a write fact when it needs delete evidence) it +keeps waiting rather than treating the signal as a result. + +**The wait blocks its shard.** `attachAuthor` runs on the watch shard's own goroutine and a shard +processes its events serially, so a removal that waits out its grace is head-of-line blocking for +every later event of that type: the commit window for a subsequent write cannot open until its event +is processed. This is a known, measured limitation rather than a subtlety: three removals in one e2e +run spent 20.18s between them, and a Deployment create queued behind them opened its window about +ten seconds late. The lookup-ordering half of that is fixed (it is what the sticky pointer and the +tier ordering did); the structural half is open, and both it and the question of when a removal +should stop waiting are ranked in +[`../design/open-asks-priority.md`](../design/open-asks-priority.md). + +## No branch anywhere depends on the type + +Verified across the whole path: `internal/queue`, `internal/auditutil`, and the resolver contain no +comparison against a group, a resource, a kind or an API version. The only `Resource ==` in the path +is `Resource == ""`, a presence check. + +Everything dispatches on one of three things: + +- **the verb**: `deletecollection` publishes a collection fact, and `delete` plus `deletecollection` + are what `isRemovalVerb` recognizes as evidence about a deletion; +- **the operation kind**: `ExactCapable` is false for a removal, which is what unlocks the weaker + tiers; +- **which fields are present**: uid, resourceVersion, name, request body, response body. + +**This is a requirement, not an accident**, and the corpus proves it. Two ConfigMap deletes, same +cluster, same type, same verb, different shapes: in `configmap/finalizer-delete/` the response object +is the ConfigMap, so the uid is recoverable and the fact lands on the uid tiers; in +`configmap/owner-ref-cascade/` the response object is a `Status`, so there is no uid anywhere and the +fact's only key is its name. A rule of the form "ConfigMaps behave like this" cannot express that, +because the difference is not a property of ConfigMaps. It is a property of the individual request, +decided by propagation policy and by what the API server chose to return. + +The same principle is what makes aggregated APIs work without ever being mentioned in the code. A +flunder is not special-cased anywhere; it produces events with no uid, and the shape-driven rules +route it to the name tier on their own. When the wardle API was added to the mutation lab, no +attribution code changed to accommodate it. + +## What the shape-driven rules reach, and what they do not + +Measured against the corpus rather than reasoned, because "the name is not available yet" describes +two populations that fail differently. + +```mermaid +flowchart TD + A[Audit event on an aggregated type] --> B{objectRef has a name?} + B -->|no: create| C[No fact published
rejected at the name gate] + B -->|yes: update, patch, delete| D{body present to backfill
uid and resourceVersion?} + B -->|deletecollection: name-less by nature| E[Collection fact
selector + namespace] + + D -->|no: proxied, so no body| F{uid or resourceVersion
on the fact?} + D -->|yes: ordinary bodied type| G[uid and rv recovered] + + F -->|neither: name only| H[name tier
namespace + name] + G --> I[exact tier: uid + rv
latest tier: uid] + E --> J[deletecollection_body_uid if a uid set arrived
deletecollection_scope otherwise] + + C --> K[Committer-authored] + H --> L[Attributed to the actor] + I --> L + J --> L + + style C fill:#7f1d1d,color:#fff + style K fill:#7f1d1d,color:#fff + style H fill:#14532d,color:#fff + style L fill:#14532d,color:#fff +``` + +The discriminator is **the body, not the name**: + +- An **aggregated-API removal carries no uid** and no resourceVersion. The kube-apiserver proxies the + request to the extension server and never decodes what came back, so `objectRef` has only the name + from the URL path, and there is no response object to recover either from. Meanwhile the watch + `DELETED` for the same object carries the full body, uid included. The name tier is what reaches + it. +- An **aggregated `deletecollection` returns no response body**, so its collection fact carries no + uid set and the join can only proceed by scope. That is the case `deletecollection_scope` exists to + serve. +- A **`generateName` create on an ordinary type** looks like the same problem and is not: + `objectRef.name` is empty because the API server assigns the name, but the policy captures at + `RequestResponse`, so the response object carries the assigned name and uid and the backfill + recovers both. + +**The one population nothing reaches is an aggregated create.** Its `objectRef` carries no name and +there is no response body to recover one from, so nothing is published for any tier to join. This is +not a gap in the tiers; it is a request the API server logged without ever saying which object it was +about. It ships committer-authored. What to do about that (accept it explicitly, or recognize it at +publish time and stop paying a grace for evidence that cannot arrive) is ranked in +[`../design/open-asks-priority.md`](../design/open-asks-priority.md). + +## What is observable + +| Metric | Labels | Answers | +|---|---|---| +| `gitopsreverser_attribution_resolutions_total` | `tier`, `actor_kind`, `group`, `version`, `resource` | which evidence named the author and who it named, per type | +| `gitopsreverser_attribution_resolution_wait_seconds` | `tier`, `event_kind`, `group`, `version`, `resource` | how long the join waited, by tier and by write/removal | +| `gitopsreverser_attribution_facts_total` | `op` = `written` / `matched` | how much of what is published is ever used | +| `gitopsreverser_attribution_fact_index_entries` | none | entries held across every scope | +| `gitopsreverser_attribution_fact_index_evictions_total` | `reason` = `per_type` / `total` | whether the caps are binding | +| `gitopsreverser_attribution_collection_without_uidset_total` | `reason` = `uid_cap` / `no_uids` | how often the precise collection join was unavailable | +| `gitopsreverser_attribution_fact_stream_gaps_total` | `stream` | facts lost for good to a trim | +| `gitopsreverser_attribution_fact_stream_decode_errors_total` | `transport` | entries skipped because they could not be decoded | +| `gitopsreverser_attribution_fact_follower_errors_total` | `transport` | follower reads that failed and were retried | +| `gitopsreverser_attribution_fact_follower_last_success_timestamp_seconds` | none | whether the follower is reading at all | +| `gitopsreverser_attribution_transport_info` | `transport` | which contract the metrics above are read under | +| `gitopsreverser_audit_events_total` | …, `no_attribution_fact` | audit events that produced no fact | +| `gitopsreverser_commits_total` | …, `author_kind` | what reached Git | + +The wait histogram is the one that earns its keep. Splitting wait time **by tier** is what turned a +missed commit window from a mystery into a measurement: the uid-latest tier at a 6.7s mean against +the exact tier at 0.18s said immediately that removals were sitting out their grace, which no +aggregate mean would have shown. `event_kind` makes that reading direct rather than inferred. + +**Three things are deliberately not readable off these**, and it is worth knowing which: + +- **`written` minus `matched` is not delivery loss**, and never was. `written` counts every fact + appended for every type; `matched` counts only facts joined on streams *this* process follows, and + a restart re-files the whole retention window. Two counters over different populations do not + subtract. What the loss paths needed was measuring where delivery happens, which is the stream + decode-error counter and the follower's last-success timestamp. +- **Head-of-line blocking is not measured.** The wait histogram times each resolution in isolation. + It does not measure the delay a slow resolution imposes on the events queued behind it on the same + shard, which is the thing that broke a spec. +- **The publish-side tier distribution is not counted.** How many facts land under a name versus a + uid is only discoverable by reading the index, and that ratio is the aggregated-API story. + +## Why it is split into two halves at all + +Three requirements each rule out resolving an author inside the audit receiver, or handing it to the +watcher over a channel. + +**The audit endpoint must answer fast, and keep answering during a deploy.** The receiver decodes a +batch, appends one entry per type, and returns. It does no lookup, waits for no watcher, and holds no +per-object state. A retried POST may append the same batch twice and that is safe without any +deduplication on the hot path, because a fact is keyed data rather than a position in a sequence: the +duplicate carries the same author under the same `(uid, rv)`, `latest` is last-writer-wins over +identical content, and a waiter woken twice resolves to the same name. + +**It has to survive more than one replica.** The audit webhook posts through a Service to whichever +replica answers, while a given object's watch shard lives on whichever replica owns that `GitTarget`. +Those are unrelated choices, so the fact and the watcher that needs it routinely land in different +processes. A per-type stream with independent cursors is exactly the primitive for that. An +in-process channel works perfectly on one replica and has to be thrown away on the second, and +sticky audit routing would make the API server's load balancing this operator's problem. + +**Rollouts are the normal state, not the exception.** A replicated deployment is almost always +mid-rollout, reconnecting, or restarting a pod, and those are precisely the cases where plain publish +and subscribe drops facts silently. A resumable stream replays the retention window instead. + +**And the delay is not ours to remove.** The batching parameters belong to the API server, so the +resolver must wait for something rather than ask repeatedly. + +What the split does **not** solve: it stops attribution being an HA blocker, but the real HA problem +is ownership: which replica owns a `GitTarget`, and keeping commits to one `(GitProvider, branch)` +serialized through a single writer. That lives in +[`../future/ha-gittarget-distribution-plan.md`](../future/ha-gittarget-distribution-plan.md). + +## Tests + +The behavior above is pinned in three places, and the corpus is the one worth knowing about: + +- **The mutation-capture corpus** (`test/mutationlab/`) holds real audit, admission and watch records + per scenario, so the identity claims in this document are checkable rather than asserted. + `configmap/finalizer-delete/`, `configmap/owner-ref-cascade/`, `configmap/generate-name-create/`, + `configmap/deletion-intent-actor/`, `flunder/aggregated-api-delete/` and + `flunder/aggregated-api-deletecollection/` are the rows this document cites; a corpus-driven unit + test in `internal/queue` replays them against the index. +- **Unit tests** in `internal/queue` and `internal/watch` cover the publish gates, the filing + branches, each tier and its precedence, the collection uid cap, and the waiter registry leaving + nothing registered behind. +- **E2E** in `test/e2e/commit_author_attribution_e2e_test.go`, + `test/e2e/audit_route_attribution_e2e_test.go` and + `test/e2e/deletecollection_intent_e2e_test.go` prove the OIDC claim chain end to end, that a + declared `auditRoute` resolves where the object's name would not, and that a finalizer-guarded + object is removed and attributed at intent time while a *different* identity clears the finalizer. diff --git a/docs/spec/deletecollection-attribution-expander.md b/docs/spec/deletecollection-attribution-expander.md deleted file mode 100644 index 1b3c974e..00000000 --- a/docs/spec/deletecollection-attribution-expander.md +++ /dev/null @@ -1,321 +0,0 @@ -# DeleteCollection attribution & deletion-as-intent - -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) -> -> Status: **PARTLY SUPERSEDED.** The render rule (§2) is current behaviour and still binds — it is -> what makes the collection window short enough to be safe. The **expander (§5) and its -> `exact_deletecollection_item` reason code (§8) are DELETED**, replaced by one collection fact that -> every removal in its scope joins; §5, §6 and §8 have been rewritten to say what took their place, -> and [`attribution-fact-stream.md`](../finished/attribution-fact-stream.md) is the record that -> argues it. -> Scope: two complementary pieces — (1) a render-layer rule that treats `deletionTimestamp` as **logical -> absence** and removes the file at delete-request time, and (2) attribution for a name-less collection -> delete. State correctness for collection deletes is -> already solved by construction in watch-first (one watch event per object); this doc adds the *intent -> semantics* and the *attribution*. -> Related: -> [attribution facts as a stream](../finished/attribution-fact-stream.md), -> [watch-first ingestion architecture](../finished/watch-first-ingestion-architecture.md), -> watch-first merge readiness §4, -> [watch event ordering & attribution grace](../facts/watch-event-ordering-and-attribution-grace.md), -> [`internal/watch/target_watch.go`](../../internal/watch/target_watch.go), -> [`internal/sanitize/sanitize.go`](../../internal/sanitize/sanitize.go), -> [`internal/queue/fact_index.go`](../../internal/queue/fact_index.go), -> [`internal/webhook/audit_handler.go`](../../internal/webhook/audit_handler.go). - -## 1. The question, and the reframe - -When someone runs `kubectl delete configmaps --all -n team-a`, the API server emits **one** name-less -`deletecollection` audit event, but the watch delivers **N** independent events — one per object. State is correct -with no special code. Two things are *not* free: - -- **Attribution.** Each per-object removal runs the resolver, finds no fact (the collection event is name-less, so - it stores nothing today — §4), and ships with the explicit **unresolved** author. "Alice deleted these 12 - configmaps" is visibly unresolved rather than silently credited to the operator. The **expander** (§5) closes this. -- **Finalizers — and this is where the design got interesting.** An object with a finalizer is *not* removed by - the delete; it gets a `deletionTimestamp` and lingers in `Terminating` until a controller clears the finalizer. - An earlier revision of this doc tried to defer attribution to that eventual removal and fight the resulting - race. The **reframe** below makes it disappear: treat the delete request itself as the moment of intent. - -This doc is built on one principle — **the repository captures intent, not a byte-for-byte API mirror** — and §2 -makes that the foundation both pieces stand on. - -## 2. Deletion-as-intent — the foundational rule - -### 2.1 Two lifecycle facts - -Deletion is two distinct facts, and conflating them is what made the old design hard: - -- **Deletion intent.** The API server accepted a `DELETE` / `DELETECOLLECTION` and marked the object with - `deletionTimestamp`. The object's *desired* existence is now **absent**. -- **Final removal.** The object actually disappeared from the API after grace/finalization completed. This is - *runtime cleanup*, and it can take 5 seconds or 3 days. - -The applyable Git manifest already strips `deletionTimestamp` and `deletionGracePeriodSeconds` -([sanitize.go:103-104](../../internal/sanitize/sanitize.go#L103-L104)) because they are **server-owned runtime -metadata, not desired state** — a manifest carrying them cannot be meaningfully re-applied. The reframe simply -takes that existing truth to its conclusion: if those fields aren't desired state, then an object that *only* -differs by having them is, as desired state, **gone**. - -### 2.2 The rule - -> **A resource with `deletionTimestamp` set is treated as logically absent from the intent tree.** The first -> observation of `deletionTimestamp` (or a `DELETED` event) removes the resource from Git and attributes the -> removal to the actor who *requested* the deletion. Later finalizer updates and the eventual `DELETED` event do -> **not** create additional Git changes; they are runtime cleanup, observed operationally only. - -So when Alice runs `kubectl delete widget foo`, the Git change is `- widgets/foo.yaml`, authored "Alice", *now* — -not a commit that sets `deletionTimestamp` on the file, and not a commit deferred until finalization finishes. - -### 2.3 Why this is correct, not a shortcut - -- **It does not bypass finalizers.** Removing the file from Git is a statement about *desired state*, not an act - on the cluster. Kubernetes still keeps the object in `Terminating`, controllers still run their finalizer - cleanup (delete external resources, drain, etc.), and the object disappears from the API only when they're - done. The intent tree saying "absent" and the API still showing `Terminating` is exactly the - desired-vs-observed split GitOps is built on. -- **`deletionTimestamp` is monotonic and terminal.** Once set, a user cannot clear it; the object *will* be - removed. So "logically absent" is never a flip-flop — there is no risk of the intent oscillating. -- **It keeps the manifest re-appliable.** We never commit a file carrying `deletionTimestamp` / - `deletionGracePeriodSeconds`; the file is simply removed. The repo's invariant stays clean: **a file present - means the resource is intended to exist.** - -### 2.4 Why immediate removal is the *reversible* default - -This is the decisive reason it's the right v1, not just a defensible one. Immediate removal establishes a single -strong invariant — *the main tree contains resources intended to exist*. From there, **richer behaviour can be -added later without breaking that invariant**: `.deletions/` or `.tombstones/` records, `DeleteIntent` side -objects, commit-message trailers, status reporting. Example of an *optional, later* enrichment (explicitly not -v1): - -```text -.deletions/widgets/team-a/foo.yaml # kind: DeleteIntent, requestedBy: alice, requestedAt, finalizersAtRequest -``` - -The **reverse is much harder**: if v1 keeps `Terminating` objects in the main tree, consumers learn that "file -exists ⇒ object still in the API," and later removing them immediately silently changes what the repository -*means*. We avoid teaching that. (Captured from the owner's "what's easiest to change later?" — immediate removal -is.) - -### 2.5 Operational caveat — don't lose the terminating object - -Logical absence is a *Git* statement; we must not go blind to a stuck deletion. A long-`Terminating` object is -surfaced as **operational status / metrics / diagnostics**, never by keeping its desired-state file around. Keep -observing the object after we remove its file for: metrics, debug logs, internal cache/fact cleanup, and -stuck-finalizer diagnostics. (See §8.) - -### 2.6 What changes in code - -One local change in the watch router, plus a dependency that already holds: - -- **Reclassify on `deletionTimestamp`.** In - [`routeLiveTargetWatchEvent`](../../internal/watch/target_watch.go#L664), after the unstructured object is in - hand, an event whose object has `GetDeletionTimestamp() != nil` is routed as **Delete** regardless of whether - the watch type is `MODIFIED` or `DELETED`. The Delete path already emits no body - ([targetWatchGitEvent:719](../../internal/watch/target_watch.go#L719)), so the file is removed. (Implementation - note: this is computed *before* `ops.Match(op)`; a WatchRule that excludes deletes will, consistently, not act - on the logical delete — call this out in review.) -- **Later events fold to no-ops.** The eventual finalizer-clearing `MODIFIED` (still `Terminating`) and the final - `DELETED` re-issue "delete X" against an already-absent path → the writer diffs them to **no-op → no commit** - (merge-readiness: no-op suppression lives in the writer). No extra - bookkeeping is needed to suppress the follow-on events; the empty diff does it. - -## 3. Why collection-delete attribution joins by UID, not RV (the linchpin) - -A removal cannot join the attribution index on resourceVersion: - -- The watch event that removes the file carries the object's **current RV** (the `deletionTimestamp`-set RV, or - the deletion RV). -- The `deletecollection` **response body** lists each removed object at its **pre-delete RV** — a different - number for the same object. - -The only field **stable and identical on both sides is `metadata.uid`.** So the expander writes a **UID-keyed** -fact per item, and the removal event joins it by UID — exactly "join per-object by UID, not RV" carried from the -[superseded nudge plan](deletecollection-attribution-expander.md). Concretely, the expander writes the -**uid-only** key variant (`factKeyVariants(..., uid, "")` returns just that one key — the exact and rv-only -variants are skipped when no RV is supplied, which is precisely right since the body RV is dead). - -> The reframe is what makes this simple. Because removal happens at **intent time**, the matching fact is written -> from the *same* `deletecollection` event that triggered the removal — it is fresh, present within the grace -> window, and there is no later finalizer-clearing fact to conflict with it (that fact, if any, lands against an -> already-removed file and produces a no-op). The earlier revision's separate "delete-intent key namespace" and -> operation-aware lookup are **no longer needed**; a plain uid-only fact suffices. - -## 4. Background: what's already true (don't re-solve these) - -- **State is solved by construction** — N watch events, mark-and-sweep backstop - (merge-readiness §4). -- **A name-less event produces ONE fact about the collection.** *(This bullet described the opposite - when the expander existed: a name-less event stored nothing, and the expander was purely additive. - The name check is now "no name AND not a collection verb", so the collection request is exactly the - case that produces a fact.)* -- **Single deletes attribute through their own fact** (including finalizer ones). A single - `kubectl delete foo` has a name, so it files a per-object fact under its uid; with §2's intent rule - the finalizer single-delete is removed and attributed at intent time too. Collection facts exist - **only** for the name-less collection case, and a removal reaches them only when no per-object fact - about the deletion applies. -- **The conservative resolver fails closed** — no usable attribution fact → the explicit unresolved - author. Governing rule: - **a wrong author is worse than no author.** -- **The grace window** absorbs a watch event that arrives before its audit fact - ([author_resolver.go:40](../../internal/watch/author_resolver.go#L40)). - -## 5. RETIRED — the expander is deleted - -**This section described the per-UID fan-out expander, which no longer exists.** It was deleted with -the switch to the attribution fact stream; see -[`attribution-fact-stream.md`](../finished/attribution-fact-stream.md), which argues at length why -rebuilding N per-object facts from one request was the wrong shape. - -What replaces it, in one line: a `deletecollection` is published as **one fact describing the -collection** — actor, type, namespace, the selector the request URI expressed, the stage timestamp, -and the set of uids the API server named when it sent a body — and every removal in that scope joins -it. The join tries uid membership first, then scope. - -The rest of this document still binds. **§2, the deletion-as-intent render rule, is untouched**, and -it is what keeps the collection window short: the removal is attributed at delete-REQUEST time, when -`deletionTimestamp` is set, so finalizers do not stretch it. §3's argument that a collection member -must join by UID rather than RV also still holds, and is why the uid tier sits above the scope tier. - -## 6. The hard case — hollow / empty body (aggregated & metadata-only) - -An aggregated/external API server or a `Metadata`-level policy can report `deletecollection` with **no usable -list**. We know the actor, type, maybe namespace, maybe a label selector, and roughly when — but **not which -objects**. The owner's trap: *in a few seconds you could see more than one `deletecollection` that "fits."* - -- **Option A — keep it around and guess by scope.** Reject. Two independent mis-attribution modes: (1) two - actors deleting in the same `(type, namespace)` window → honest answer is conflict→unresolved, so it degrades to - the explicit unresolved author under exactly the load that makes the case interesting; (2) even a - single collection delete would - capture an *unrelated* plain `kubectl delete configmap x` in the same window. Selector re-matching narrows but - doesn't remove it (selectors overlap; empty selector matches all). Violates "a wrong author is worse than no - author." -- **Option B — `Co-authored-by` floor.** Honest credit under ambiguity (multiple trailers), Git author stays - unresolved. But in watch-first the deletes are driven by watch events, not the audit cause, so it needs real - plumbing to carry a scope-cause into the commit-window builder. A deliberate fast-follow. -- **Option C — commit as the explicit unresolved author, document the limit.** v1. - -**RESOLVED, and not by Option C.** The hollow-body case was shipped as **scope matching**, which is -Option A bounded until it is safe rather than rejected outright. Three things bound the -over-attribution the rejection was about, and the third is the one that changes the verdict: - -- **Namespace and selector narrow the scope.** The selector is the *intent the actor stated*, read - off the request URI, and it is present even when the body is not. An empty selector matches - everything of the type in the namespace, which is exactly what `--all` means. -- **Precedence keeps anything with its own fact out.** The scope tier is the weakest evidence the - join has, so it is reached only when every more specific tier missed. The unrelated - `kubectl delete configmap x` in the same window is claimed by its OWN fact and never reaches it. -- **The window is short**, because of §2. Under deletion-as-intent the removal happens at - delete-request time, so the window only has to cover audit batching plus clock skew — 30s by - default, against a fact TTL of ten minutes. That is what makes the scope match safe, and it is not - something the original framing, where attribution chased the eventual removal, could have offered. - -The result is the reverse of the old degradation: the aggregated and metadata-only cases that used -to ship committer-authored now resolve, and a production cluster with -`--audit-webhook-truncate-enabled` — the one MOST likely to send no body for a large collection -delete — is the one that gains most. Option B (`Co-authored-by`) is no longer needed for this case. - -## 7. Recommendation at a glance - -| Case | Behaviour | Why | -|---|---|---| -| Any delete (single or collection member) | **Remove file at intent time; never commit `deletionTimestamp`** (§2) | Git is intent; reversible invariant; manifests stay re-appliable. | -| Finalizer object | **Removed immediately, attributed to the delete-requester**; finalizer cleanup is runtime no-op in Git | Reframe dissolves the old delay/conflict; controllers still finalize in-cluster. | -| Collection delete, body present | **One collection fact**, joined by **uid membership** (`deletecollection_body_uid`) | API server states "these exact objects, by this user." | -| Collection delete, hollow body | **The same collection fact**, joined by **scope** — type, namespace, selector, window (`deletecollection_scope`) | Bounded by precedence and a short window (§6); resolves what used to degrade. | -| Stuck `Terminating` | **Operational status/metric** (§2.5), file already absent | Don't pollute intent with runtime state. | - -## 8. Observability & diagnostics - -**The `exact_deletecollection_item` result label is gone**, and so is the expander's -`op="deletecollection_expanded"` write counter. Two labels replace the one, because the match is now -two-tiered and the tiers carry different confidence: - -- **`deletecollection_body_uid`** — the removal's uid was in the set the API server said it deleted. No - over-attribution risk at all. -- **`deletecollection_scope`** — matched by namespace, selector, and window alone. Weaker evidence, and - the reason the window is short. - -Both flow onto `AttributionResolutionsTotal{result=…}`, so a dashboard can now separate *precise* -collection credit from *scoped* collection credit rather than seeing one bucket. -`attribution_collection_degraded_total{reason}` counts a collection fact published WITHOUT its uid -set, which is what turns the second tier from an inference into a measurement. See -[`interpreting-metrics.md`](../interpreting-metrics.md). - -**Stuck-finalizer / terminating diagnostics** (§2.5) are unchanged: surface long-`Terminating` -watched objects whose files we already removed, so logical absence never hides a stuck deletion. -Optional **secondary diagnostic attribution**: record *who* cleared the finalizer as a diagnostic -signal — a metric label or debug log — **never** as the Git author (that stays the delete-requester, -and the event is a no-op commit anyway). - -## 9. Tests - -### 9.1 Unit - -`internal/watch` (render rule, §2.6): - -1. A `MODIFIED` whose object has `deletionTimestamp` set is routed as **Delete** (no body) — *logical absence*. -2. A `MODIFIED` without `deletionTimestamp` is routed as **Update** (sanitized body) — unchanged. -3. A second Delete for an already-absent path diffs to **no-op** (covers finalizer-clear + eventual `DELETED` - folding to nothing; assert no commit). May reuse existing writer no-op tests. - -`internal/queue` and `internal/watch` (the collection fact and its join, §5–§6): - -1. A collection delete publishes **one** fact carrying the actor, the namespace, the selector from the - request URI, and the uid set — never one fact per object. -2. A list body larger than the uid cap drops the set and counts - `attribution_collection_degraded_total{reason="uid_cap"}`; the fact still publishes. -3. A hollow / `Status` / unparseable / absent body still publishes a fact — with no uid set. This is the - case the expander produced nothing at all for. -4. Join shape, uid tier: a removal whose uid is in the set resolves to the actor as `deletecollection_body_uid`, - even though its RV never matches (proves §3). -5. Join shape, scope tier: a removal with no uid set to consult resolves as `deletecollection_scope` when the - namespace, selector and window cover it — and does NOT resolve when the selector rejects its labels, - when it is in another namespace, or when the window has passed. -6. Precedence: an object with its own fact never reaches either collection tier. - -### 9.2 E2E — implemented - -Implemented as Ginkgo specs in `test/e2e/deletecollection_intent_e2e_test.go` -(`Describe("DeleteCollection intent & attribution")`). Attributed-author mode on (skipped in configured-author mode); a -GitTarget claims `configmaps` with a 0s commit window; deletes are issued by an **impersonated actor** carrying -OIDC name/email claims, and finalizers are cleared by a **separate** impersonated identity. Each spec scopes its -collection delete with a per-spec label selector and asserts **convergence** (these files gone/authored thus, -these survive), never a global drop count, so they run against a reused cluster. - -1. **`removes every collection member and attributes each removal to the actor` (state + attribution — "do - both").** Three configmaps; the actor runs the collection delete; all three files go and each removal commit - is **authored by the actor**, not committer. - -2. **`removes a finalizer object at intent time, authored by the actor, while it is still Terminating` (the intent - showcase).** One plain + one finalizer-guarded configmap. After the collection delete: (a) both files are - removed and authored by the actor; (b) the finalizer object **still exists** in-cluster with a - `deletionTimestamp` (Terminating); (c) the finalizer is cleared **as a different identity**, the object then - leaves the API, and `Consistently` proves **no new commit** for the path — the removal commit (authored by the - actor) stays the last one. - -3. **`removes a single finalizer object at intent time too (the rule is not collection-specific)`.** A single - named `Delete` of a finalizer-guarded configmap, as the actor: file removed at intent, authored by the actor, - object still `Terminating`; clearing the finalizer yields no further Git change. (A single delete is - attributed by its own per-object fact, never by a collection one — proving §2 is a general render rule.) - -4. **`scopes a label-selector collection delete to matching objects and leaves siblings`.** Two matching + one - non-matching sibling; a label-selector collection delete removes only the matching files (authored by the - actor) and `Consistently` confirms the sibling survives untouched. - -## 10. Definition of done - -- **§2 render rule:** `routeLiveTargetWatchEvent` reclassifies a `deletionTimestamp`-bearing event to Delete; no - manifest ever carries `deletionTimestamp`/`deletionGracePeriodSeconds` (already true via sanitize); later - finalizer/`DELETED` events fold to no-ops. Unit §9.1.1–9.1.3. -- **§5 collection fact:** one fact per collection request, carrying scope, selector and (when the body - allowed it) the uid set; finalizer items attributed like any other, since §2 removes them at intent; - defensive parsing; `deletecollection_body_uid` and `deletecollection_scope` result labels wired. Unit §9.1.4–9.1.9. -- **E2E §9.2.1–9.2.4** implemented, convergence-asserted; the finalizer showcase (§9.2.2) proves removal-at-intent - with a *different* finalizer-clearing identity. -- **Hard case:** resolved by scope matching (§6), bounded by namespace, selector, precedence and a short - window. Option B (`Co-authored-by`) is no longer needed for it. -- **Reversibility honored:** main tree = resources intended to exist; richer `.deletions/` style records left as - future enrichment, invariant intact. -- Full validation per AGENTS.md: `task fmt → generate → manifests → vet → lint → test → test-e2e` (e2e sequential). - diff --git a/docs/spec/gittarget-new-file-placement-rules.md b/docs/spec/gittarget-new-file-placement-rules.md index 86ead35c..b44efa3c 100644 --- a/docs/spec/gittarget-new-file-placement-rules.md +++ b/docs/spec/gittarget-new-file-placement-rules.md @@ -4,15 +4,17 @@ > > Status: implemented (v1 — Option B2: one `byType`/`default` placement map, > with sensitivity treated as an internal write-safety classification rather than a -> separate user-facing placement namespace) + Option C sibling inference (steps -> 1/2/4) + the kustomize-root fallback documented below. The earlier B1 surface (a -> nested `sensitive:` override block) shipped first and was superseded by B2 on the -> same branch before release; Option A and C step 3 remain deferred. See +> separate user-facing placement namespace) + the kustomize-root fallback + the +> canonical path. **Option C (sibling inference) was implemented and then removed**; +> its sections below are kept as history because the argument is worth having on the +> page, and because the risks it enumerated are what the removal answers. The earlier +> B1 surface (a nested `sensitive:` override block) shipped first and was superseded +> by B2 on the same branch before release; Option A remains deferred. See > "Sensitivity as a write-safety classifier (B2 implementation notes)" below for how > the encryption guarantee is preserved without the API-level split. -> Captured: 2026-06-05 +> Captured: 2026-06-05. Option C removed: 2026-07-29. > Related: -> [file-agnostic-placement.md](gittarget-new-file-placement-rules.md) — **the vision Option C serves**, +> [open-asks-priority.md](../design/open-asks-priority.md) — **the argument for deleting Option C**, > [contextual-namespace-and-kustomize-folder-editing.md](contextual-namespace-and-kustomize-folder-editing.md), > [gittarget-repository-validity-and-placement.md](gittarget-new-file-placement-rules.md), > [current-manifest-support-review.md](current-manifest-support-review.md), @@ -22,6 +24,17 @@ ## Summary +> **What is live, in one paragraph.** A new resource — one with no document in Git yet — +> gets its path from the GitTarget's declared `placement.byType`/`placement.default` +> (Option B2); failing that, from the folder's one supported kustomization root, if it has +> exactly one; failing that, from the built-in canonical +> `{namespaceOrCluster}/{group}/{resource}/{name}.yaml` path. Nothing reads the layout of +> the other documents of the same type. Option C did exactly that and was deleted — the +> argument is in [`open-asks-priority.md`](../design/open-asks-priority.md) and summarised +> in [its own section below](#option-c-follow-the-existing-layout-sibling-inference--removed). +> Every resolution is counted by `placements_total{source,…}`, so a repository whose +> layout needs a declaration says so in a metric rather than in a folder. + New-resource placement should become an explicit GitTarget-level policy. There are three viable shapes: @@ -48,19 +61,20 @@ they layer: identity-completeness, and append/collision rules, not which config block to read. Ordered rules (A) stay a later escape hatch only if the type map proves too limiting. -- **Option C is the default underneath it.** With no policy, placement *follows the - layout already in the repo*; on an empty repo it falls through to today's - canonical path, so default behaviour is byte-identical to now. C is what makes - "point me at an existing folder and it just works" real — the vision in - [file-agnostic-placement.md](gittarget-new-file-placement-rules.md) — and it removes the - need for B to carry a catch-all default, because the gaps B leaves are filled by - inference rather than by a hand-written fallback template. - -So the recommended shape is **B over A for the declared surface, with C as the -zero-config default that B overrides where the user has an opinion.** The rest of -this document develops B and C together; C's sharp edges get a dedicated -[problems-and-risks](#problems-and-risks-with-option-c) section because inferring -policy from mutable repo state is where the real subtlety lives. +- **Option C was the default underneath it, and is no longer.** With no policy, + placement used to *follow the layout already in the repo*. That is what made "point me + at an existing folder and it just works" a demo, and it is what made a human's edit to + the repository silently change where the operator writes. It was removed; what fills + B's gaps now is one structural fact (the folder's single kustomize root, if it has one) + and then the canonical path. See + [Option C … removed](#option-c-follow-the-existing-layout-sibling-inference--removed). + +So the shipped shape is **B for the declared surface, the kustomize root where the +folder's own structure decides, and canonical otherwise.** The rest of this document +develops B; C's sections are retained as history, and its +[problems-and-risks](#problems-and-risks-with-option-c-as-written-before-the-removal) +list is the record of why inferring policy from mutable repo state was the wrong +foundation rather than a set of edges to keep filing down. Existing manifests are still match-first: once a resource already has a document in Git, updates and deletes use that document's current location instead of @@ -409,7 +423,7 @@ type GitTargetPlacementSpec struct { The semantics are: - resolve placement from the single map: exact `byType` first, then `default`, - then sibling inference, then canonical fallback; + then the folder's single kustomize root, then canonical fallback; - independently classify the resource as sensitive or not; - for a sensitive resource, require the selected path to be identity-complete; - for a sensitive resource, write encrypted content and refuse multi-document @@ -470,7 +484,7 @@ of the API-level split is written up in "Sensitivity as a write-safety classifie The validation rules are almost the same as for ordered rules: -- omitted `placement.default` uses sibling inference and then the built-in +- omitted `placement.default` uses the kustomize-root fallback and then the built-in canonical fallback; - every `byType` key must parse as a valid resolved type key; - every referenced type should be served and watched by the GitTarget, or at @@ -494,14 +508,15 @@ Implemented shape: 1. the B2 type-map is **the** declared API — the smallest surface that covers the real "this type here, everything else there" need while keeping sensitivity as a write-safety rule; -2. Option C (sibling inference, below) is the **default** that runs when B is - absent or silent for a resource, so an unconfigured target follows the repo's - own layout instead of forcing canonical; +2. when B is absent or silent for a resource, the fallbacks run: the folder's single + supported kustomize root (a structural fact, so a new file is reachable from a render + root), and otherwise canonical. Option C's sibling inference used to sit here and was + removed; 3. ordered rules (A) remain a future extension only if users hit the type-map limit; 4. one template renderer, identity validation, encryption enforcement, and append - rules serve all of it, so B's templates and C's inferred locations flow through - one writer. + rules serve all of it, so a declared template and a fallback path flow through one + writer — and through one refusal gate, counted by one metric. ### Sensitivity as a write-safety classifier (B2 implementation notes) @@ -515,10 +530,12 @@ placement API and stays there under B2: `types.SensitiveResourcePolicy` (core Secrets always; plus operator-configured types), independent of which path placement chose. A Secret routed to `all.yaml` is still encrypted. -- **Inference never crosses the encrypted boundary.** Sibling-cohort matching - reads a document's own `CauseEncrypted` classification, so a sensitive resource - only ever infers from encrypted siblings and a plaintext one only from plaintext - siblings — no config needed. +- **The encrypted boundary is enforced at the write, not at resolution.** A sensitive + resource whose path already holds any document is refused, and a plaintext resource + routed at a file holding an encrypted document is refused — both read the document's own + `CauseEncrypted` classification, and both are counted with a bounded reason. This is + where the guarantee always actually lived: sibling inference had its own version of the + rule, and deleting it removed a second implementation rather than the guarantee. - **Sensitive never appends.** A sensitive resource whose resolved path already holds a document is refused (`finishPlacement`), never appended. - **Canonical stays SOPS.** The built-in fallback keeps the `.sops.yaml` suffix for @@ -559,31 +576,114 @@ Whether to teach the Validated gate about operator-configured sensitive types (s this becomes a fast, up-front rejection there too), and whether to add a bounded status surface for placement skips, are open questions below. -## Option C: follow the existing layout (sibling inference) +## The resolution ladder, as implemented + +Placement only ever runs for a **new** resource — one with no document in the store yet +(match-first handles everything else, and never moves it afterwards). Three steps, in order, +and each one either answers or declines: + +| Step | Mechanism | `placements_total{source}` | Decides because | +|---|---|---|---| +| 1 | declared `placement.byType`, then `placement.default` | `declared` | the GitTarget said so | +| 2 | the folder's single supported kustomization root | `kustomize_root` | a file that root cannot reach never renders | +| 3 | canonical `{namespaceOrCluster}/{group}/{resource}/{name}.yaml` | `canonical` | nothing else did | + +Every resolved path — whichever step produced it — then passes one gate before a byte is +written: [path validation](#path-validation), the append-safety rules, and the +[write-safety refusals](#sensitive-placement-and-uniqueness). A refusal is a resource the +mirror does not hold, and it is counted as `placement_refusals_total{reason}` rather than +being merely logged. + +### The kustomize-root fallback + +The canonical path is a `{namespaceOrCluster}/{group}/{resource}/{name}.yaml` tree a +kustomization's `resources:` graph can never reach. So in a folder that kustomize builds, +a new document at the canonical path is not merely oddly placed — it is **never rendered**, +and nothing applies it. That is the failure new-file placement exists to prevent, and it is +why this step survived the Option C deletion. + +When the whole *writable* subtree is governed by **exactly one** supported kustomization, +the new document is written beside that kustomization's other files and added to its +`resources:` list in the same commit (the product-level "add to the right kustomize file" +framing lives in +[`unreflectable-edits-and-write-gating.md`](../design/support-boundary/unreflectable-edits-and-write-gating.md)). + +This is a **structural fact, not an inference**: the destination follows from there being +one root, not from picking the largest matching cohort of similar documents. Consequently: + +- more than one supported kustomization under the scanned root is **ambiguous** and + declines to canonical rather than guessing which root the resource belongs to; +- an *unsupported* kustomization is never a root (the writer must not edit one), and never + a destination; +- under render-root scoping — where the scan reaches past `spec.path` into a base the + overlay reads — "one supported kustomization" means one **writable** one, so an overlay + resolves to its own root instead of declining because the read-only base counts as a + second; +- if the `resources:` entry cannot be added, the file is committed outside every render. + That is invisible in the folder, so it is counted: + `placement_kustomization_entries_total{outcome="failed"}`. + +### Namespace style follows the governing kustomization + +A document in a directory whose kustomization sets a `namespace:` transformer does not carry +`metadata.namespace` — the build context supplies it (see +[contextual-namespace-and-kustomize-folder-editing.md](contextual-namespace-and-kustomize-folder-editing.md)). +A new document placed there must follow that convention, or it breaks the folder's own +style, and this applies to **every** resolved path: a declared template pointing into a +governed directory is under the same obligation as the kustomize-root fallback. + +It is conditional on the two namespaces **matching**. Omitting `metadata.namespace` hands +the namespace to kustomize, so a transformer naming a *different* namespace would render the +document as a different object — the mirror would claim to hold a resource it does not. When +they differ the namespace is written explicitly, and the render oracle reports a folder that +cannot express the object rather than the write quietly diverging. + +## Option C: follow the existing layout (sibling inference) — REMOVED + +> **History.** Option C shipped (steps 1/2/4) and was removed on 2026-07-29. The full +> argument is in [`open-asks-priority.md`](../design/open-asks-priority.md), "The one real +> design call". The short form: +> +> - **It made a human's edit to the repository change the operator's behaviour**, with no +> Kubernetes object changing and nothing in status recording the move. Delete enough of +> one namespace's ConfigMaps from a bundle and the bundle stops being namespace-agnostic, +> so the next new ConfigMap takes a different path. Nobody approved that. +> - **Its central guard failed by cascading.** The namespace-agnosticism check was, for a +> period, vacuous on the singleton branch: one directory holding one namespace satisfied +> "all the same directory" trivially, so a new namespace's object was appended into the +> first namespace's file, which then genuinely spanned two namespaces, which legitimized +> the bundle for every later object, which collapsed a whole type into one file. The fix +> was right; the lesson is that a rule inferred from mutable state has failure modes that +> feed themselves. +> - **The explainability it required (P8, below) was declared mandatory and never built.** +> - **What it bought was smaller than the demo suggested.** "Point me at an existing repo +> and it just works" is carried by *match-first*, not by inference: every resource that +> already has a document is edited exactly where it lives, forever. Inference only fired +> for a type or namespace the target had never written — a rare event, and precisely the +> one where a wrong guess is invisible because there is no prior file to compare against. +> - **And the layout most likely to be hand-authored — per-namespace segmentation — was the +> one it could not extend anyway (P4).** The user had to declare it regardless. +> +> **What replaced it:** nothing, deliberately. A layout this operator cannot derive from one +> root is declared in `placement.byType`/`placement.default`, or it is canonical — and +> `placements_total{source="canonical"}` names the GitTarget and the type that needs the +> line, so "you need a declaration here" is a query rather than an archaeology exercise. No +> `spec.placement.mode` enum was added: an off-switch for a removed feature is a permanent +> API field bought to solve a temporary problem. +> +> The rest of this section is what the design said while it was live. A and B both ask the user to *declare* the layout in the CRD up front, in a template language, split into sensitive and normal. Option C asks for nothing. It places a new resource **where resources like it already live in the repo.** The folder is the policy. -This is not a new principle — it is the one the writer already uses, generalized -one notch. Today an existing resource is found by manifest identity and edited *in -place*: the operator already follows a file's current location instead of imposing -canonical placement ("existing manifests are still match-first", above). Option C -extends "follow the existing location" from *the same resource* to *its siblings*: -a resource the operator has never written before is placed next to the resources -it most resembles. It is the literal implementation of -[file-agnostic-placement.md](gittarget-new-file-placement-rules.md)'s goal — "point at a -real folder and it just works", "the location doesn't matter" — because the layout -already in Git, having passed acceptance, is by definition a layout the operator -will accept. - -### How it resolves a path +### How it resolved a path Placement only ever runs for a **new** resource — one with no document in the store yet (match-first handles everything else). The content-derived store already -knows every accepted document, its file path, and its effective identity. C reads -two independent facts straight off that store; it never reverse-engineers a +knows every accepted document, its file path, and its effective identity. C read +two independent facts straight off that store; it never reverse-engineered a template. 1. **Which directory** — find the nearest *cohort* of existing documents that @@ -593,237 +693,131 @@ template. |---|---|---| | 1 | same (resource type, namespace) | that cohort's directory | | 2 | same resource type, any namespace | that cohort's directory | - | 3 | same namespace, any type | that cohort's directory | - | 4 | nothing matches | canonical `ToGitPath()` directory (today's default) | + | 3 | same namespace, any type | never implemented (P5) | + | 4 | nothing matches | canonical `ToGitPath()` directory | 2. **One-per-file vs bundle** — look at how that cohort is stored: - cohort is **one resource per file** → create a new single-document file in - that directory, named `{name}.yaml` (the only filename pattern worth - inferring; anything fancier falls back to `{name}.yaml`); + that directory, named `{name}.yaml`; - cohort **shares a file** (a bundle such as `configmaps.yaml` or `all.yaml`) → - append the new resource as a document to that same bundle, via the existing - plaintext multi-document append path. - -Both decisions are *observed*, not guessed, so there is no fuzzy template -inference and the result round-trips: the store's file↔identity map (the same one -match-first relies on) is the single source of truth. - -### Sensitive stays hard-split — with no config - -The sensitive/normal split that A and B get from two config blocks, C gets for -free from the encryption classification already in the store: - -- a sensitive resource **never** infers from plaintext siblings and is **never** - appended to a plaintext bundle; -- it infers only from other encrypted/sensitive single-document siblings in the - same directory, regardless of filename suffix; otherwise it uses the built-in - **secure canonical fallback** (identity-complete and encrypted, with a - `.sops.yaml` suffix only as a convention). + append the new resource as a document to that same bundle. -So the encryption guarantee never depends on the user having configured the split -correctly — there is no split to configure. +Step 2 additionally required a candidate to **prove** it was namespace-agnostic before it +could be used for a namespace it had never seen — a bundle by already spanning two +namespaces, a singleton style by having every member in one shared directory that spans +two. That proof requirement was the fix for the cascade described above, and it is the part +worth remembering: it was correct, and it was not enough, because the feature's problem was +never the quality of its guard. -### Namespace style comes along for free +### Sensitive stayed hard-split — with no config -Because C mirrors a sibling, it also mirrors that sibling's `NamespaceSource` -(from [contextual-namespace-and-kustomize-folder-editing.md](contextual-namespace-and-kustomize-folder-editing.md)). -Drop a new file beside namespace-less documents that sit under a supported -kustomize context and it inherits that style — no `metadata.namespace` is written; -beside explicit-namespace documents the namespace is written. This is the one -option that answers file-agnostic-placement.md's "I'd like to *not* write the -namespace" without adding a `manifestStyle` knob: placement and output style -converge under "match your neighbours". (Bounded by the contextual-namespace -rules: an ambiguous context already refuses the GitTarget before placement runs.) +The sensitive/normal split that A and B get from two config blocks, C got for +free from the encryption classification already in the store: a sensitive resource never +inferred from plaintext siblings and was never appended to a plaintext bundle; it inferred +only from other encrypted siblings, and otherwise used the built-in secure canonical +fallback. That guarantee is unchanged by the removal — it is now enforced entirely by the +[write-safety refusals](#sensitive-placement-and-uniqueness), which is where it always +actually lived. ### Empty folder → canonical, then self-propagating A freshly bootstrapped repo has no siblings, so the first resource of each kind -lands on canonical `ToGitPath()` — **byte-identical to today.** From then on the -layout propagates itself. A brand-new target behaves exactly as it does now; the -power only appears once a human (or a prior import) has established any layout. - -**Amendment, decided during implementation: a kustomize-root fallback sits -between "no sibling match" and true canonical.** The rule above is exactly right -for a flat or already-populated folder, but it silently breaks the moment the -GitTarget's one kustomization-managed folder receives its *first* resource of a -brand-new type: canonical is a `{group}/{version}/{resource}/{namespace}/{name}.yaml` -tree the kustomization's `resources:` graph can never reach, so the new file -would land outside the very folder it was meant to join — precisely what this -document exists to prevent, just for a type instead of a whole folder. - -So, when steps 1/2 (sibling inference) both miss **and** the scanned subtree is -governed by exactly one supported kustomization, the new resource is placed -beside that kustomization's other files (and gets a `resources:` entry — see -`kustomize-support-boundary.md` and -`unreflectable-edits-and-write-gating.md` for the product-level "add to the right -kustomize file" framing) instead of falling to canonical. This is deliberately -narrower than the shelved step 3 (same namespace, any type) above: it never joins -an existing bundle, and it only ever fires when there is exactly one supported -kustomization for the whole GitTarget to be about — the destination follows from -there being one root, not from picking the largest matching cohort — so it cannot -become the "sink that swallows every new type" risk (P5) step 3 raised. More than -one supported kustomization under the scanned root is ambiguous and falls through -to canonical rather than guessing which root the new resource belongs to. - -True canonical — no sibling of the matching type or kind, and no single -kustomization root to fall back to — remains exactly as described above: a -brand-new, unmanaged target behaves byte-identical to today. +landed on canonical `ToGitPath()` — byte-identical to today. From then on the layout +propagated itself. This is why **cold-start repositories are unaffected by the removal**: +inference already fell to canonical there. The behaviour change is confined to a brownfield +folder that had a layout to continue. ### Determinism and ambiguity A type can legitimately live in two layouts at once (some ConfigMaps bundled, some -canonical). The lookup must be deterministic: - -- pick the cohort with the **most members**; tie-break on lexically-smallest - directory, then smallest file — stable and independent of walk order. -- Unlike contextual-namespace ambiguity (a correctness hazard, so it *refuses*), a - "wrong but valid" new-file location is cosmetic: the document is match-first the - instant it exists, so a deterministic tie-break is better than failing the - target. Sensitive identity-completeness is still enforced, so determinism never - weakens the encryption guarantee. - -### What it does not do, and how it composes - -C cannot express a layout you do not yet *have* — a greenfield "I want all -ConfigMaps bundled even though none exist yet" intent. That is exactly what A and B -are for, and the three compose cleanly: - -- **C is the default** (no config), best for brownfield / "point me at an existing - repo"; -- **B (or A) is the override:** an explicit type-map entry takes precedence, and C - fills every gap — so B no longer needs a catch-all default, because the canonical - fallback is just C's step 4; -- if a per-repo override is ever wanted *without* CRD surface, the repo-native - form — consistent with the just-landed `.gittargetignore` (an in-repo, versioned, - zero-API policy file, see - [gitpath-foreign-content-stringency.md](gitpath-foreign-content-stringency.md)) - — is the natural shape. Noted, not required: C's whole point is that the base - case needs nothing. - -### Problems and risks with Option C +canonical), so the lookup picked the cohort with the **most members**, tie-breaking on +lexically-smallest directory then file — stable and independent of walk order. The +reasoning at the time was that a "wrong but valid" location is cosmetic, since the document +is match-first the instant it exists. That is true of one document and false of a +convention: the wrong file is where every later resource of that type then goes. + +### What it could not do, and how it composed + +C could not express a layout you did not yet *have* — a greenfield "I want all ConfigMaps +bundled even though none exist yet" intent. That is what B is for, and B always outranked +C. It is also, in hindsight, the whole trade: B could express everything C could and say so +on the page. + +### Problems and risks with Option C, as written before the removal Sibling inference is powerful precisely because it reads its policy from mutable repo state — which is also exactly where every one of its sharp edges comes from. -None of these is fatal, but each needs a decided answer before C ships, and several -are the concrete reason **B exists as the override.** +This list was written as "each needs a decided answer before C ships". Read in retrospect, +it is the case for the deletion: **P1, P2, P3, P4, P6 and P8 are all one property**, stated +six times, and only P7, P9 and P10 survive as facts about the code that remains. **P1 — Placement is path-dependent on history, and the "most members" tie-break can -flip.** The cohort lookup is computed against the repo *as it is now*. If a type -lives in two layouts at once (some ConfigMaps canonical, some bundled into -`all.yaml`), the winner is "most members" — but that count moves as the repo grows. -A repo that is 6-canonical / 5-bundled routes a new ConfigMap to canonical; after a -human bundles four more it is 6-canonical / 9-bundled and the *next* new ConfigMap -goes to the bundle. Same kind, different destination, purely because of *when* it -arrived. This instability is inherent to "infer from the repo" and cannot be fully -removed; it can be tamed: (a) surface the chosen cohort, its size, and the -tie-break in the scan/dry-run output so a flip is never silent, and (b) let a -cohort the user declared in **B outrank any inferred cohort**, so the unstable case -only exists where the user expressed no preference at all. - -**P2 — Cold start and batch resync collapse to canonical.** On a fresh import or a -full resync the desired set is planned against a store that is empty (or mid-fill). -With no siblings, every resource falls to canonical — including a large initial -sync the user might have wanted bundled. Worse, if placement consulted a store that -mutates *within* a single plan, the first ConfigMap placed would become the sibling -for the rest, making a whole batch's layout depend on intra-batch ordering. -Decision: **resolve every cohort against the pre-plan store snapshot** and place a -batch together (reusing step 8's "group new creates by path"), so a batch is -order-independent and a cold start is deterministically canonical. The consequence -is blunt and must be stated: **C cannot bootstrap a layout that does not yet -exist.** A greenfield bundled layout happens only if the user declares it (B) or -seeds one file by hand first. - -**P3 — The self-fulfilling canonical bias.** Because empty → canonical and siblings -then propagate, a repo the operator bootstrapped itself stays canonical *forever* -unless a human reorganizes it. C's benefit is therefore concentrated almost -entirely on the brownfield / human-authored / imported repo; for the dominant -"operator created the repo" path, C ≡ today's behaviour. That is a safe default, -not a defect — but nobody should expect bundling to *emerge* on its own, and the -user docs must say so. +flip.** The cohort lookup is computed against the repo *as it is now*. A repo that is +6-canonical / 5-bundled routes a new ConfigMap to canonical; after a human bundles four +more it is 6-canonical / 9-bundled and the *next* new ConfigMap goes to the bundle. Same +kind, different destination, purely because of *when* it arrived. *Retired by the +deletion: the destination no longer depends on repo state at all.* + +**P2 — Cold start and batch resync collapse to canonical.** With no siblings, every +resource falls to canonical — and a store mutating *within* one plan would have made a +whole batch's layout depend on intra-batch ordering. Decision at the time: resolve every +cohort against the pre-plan store snapshot. *The snapshot rule is still in force* for the +store reads that remain (does this path already hold an append-safe file; does its +directory carry a kustomization), so a batch is still order-independent. + +**P3 — The self-fulfilling canonical bias.** A repo the operator bootstrapped itself stayed +canonical forever unless a human reorganized it, so C's benefit was concentrated on the +brownfield repo and was a no-op for the dominant path. *Retired: canonical is now simply +the documented answer, and the metric says when a declaration would be better.* **P4 — Step 2 cannot extend a custom per-namespace layout to an unseen namespace.** -Inference deliberately refuses to reverse-engineer a filename/segment template, so -when a brand-new namespace appears for a known type, step 1 (same type + namespace) -misses and step 2 (same type, any namespace) finds a cohort spread across -`…//…` directories but cannot know the `` segment without inferring the very -template it swore off. For the **canonical** layout this is harmless — step 4's -fallback *is* the canonical per-namespace path, so the result is identical. For a -**custom** per-namespace layout (e.g. `{namespace}/configmaps.yaml`) the new -namespace cannot be inferred and lands canonical, breaking the user's pattern. This -is the single clearest reason **B is the override, not a nicety:** a custom -namespace-segmented layout must be *declared*; C can continue it for namespaces it -has already seen but cannot invent the segment for new ones. Document the boundary -so it is a known limit, not a surprise. - -**P5 — Step 3 (same namespace, any type) can over-capture into a growing bundle.** -This is the most dangerous rung. A single heterogeneous bundle in a namespace can -become a sink that swallows every *new* type in that namespace just because they -share a namespace — a layout the user never asked for, growing without bound. -Mitigation: fire step 3 **only into a cohort that is already a single heterogeneous -bundle file** (the user demonstrably chose "one file for this namespace"), never to -scatter a new type into per-type files that merely happen to share a namespace. If -step 3 still feels too clever, **drop it entirely** — steps 1, 2, and 4 already -cover per-type bundles, per-type files, and canonical, and the namespace-bundle -layout can simply be required via B. (See the open question on whether step 3 ships -at all.) - -**P6 — Delete-then-recreate can move a resource.** Existing documents are -match-first, so a live resource never moves while it exists. But a resource that is -deleted (its document swept) and later recreated is "new" again and re-inferred -against whatever the repo looks like *then*, which may differ from where it lived -before. Placement is create-time and non-retroactive — the same contract A and B -carry — but C makes "create-time" depend on mutable repo state, so this churn is -more visible. Acceptable, but call it out. - -**P7 — An inferred path is still subject to the write-time ignore invariant.** A -resolved path — inferred or canonical — can collide with a `.gittargetignore` +Inference refused to reverse-engineer a path segment, so a custom `{namespace}/…` layout +could not be continued into a namespace it had never seen and fell to canonical, breaking +the user's pattern. *This was the strongest argument for the deletion: the layout most +likely to be hand-authored was the one inference could not reach, so the user had to +declare it anyway.* + +**P5 — Step 3 (same namespace, any type) can over-capture into a growing bundle.** A single +heterogeneous bundle could become a sink that swallowed every new type in a namespace. +*Never implemented, and now unreachable.* + +**P6 — Delete-then-recreate can move a resource.** A resource whose document was swept and +which was later recreated was "new" again, and re-inferred against whatever the repo looked +like then. *Retired: a recreated resource resolves the same way it did the first time, +because the answer does not depend on the folder's history.* + +**P7 — A resolved path is still subject to the write-time ignore invariant.** Any resolved +path — declared, kustomize-root, or canonical — can collide with a `.gittargetignore` pattern and trip the §4.3 `IgnoreShadowsManagedPath` precondition -([gitpath-foreign-content-stringency.md](gitpath-foreign-content-stringency.md)), -aborting the flush. Ignored and foreign files are never in the store, so they can -never *be* siblings (good) — but a new canonical-fallback path can still be -shadowed. C inherits this failure mode rather than creating it; the existing -precondition already handles it, and the diagnostic must name the inferred path and -the matching pattern. - -**P8 — Explainability becomes a hard requirement, not a nicety.** With A/B a user -reads the CRD to know where a resource will go; with C the answer lives in the repo -plus a precedence ladder. The scan/dry-run output **must** state, per new resource, -the chosen path *and* the cohort + ladder step that produced it (e.g. "matched 9 -ConfigMaps in `all.yaml` via step 1"). Without that, "why did it land there?" is -unanswerable. This is the one part of C that is genuinely *more* work than B, and it -should be treated as in-scope, not optional polish. - -**P9 — Cohort lookup cost at scale.** Naively, each new resource scans the store -(O(store size)); a large repo × a big resync batch is O(N·M). Build the per-plan -indexes (by resource type, by namespace) once from the snapshot and resolve against -them. Minor, but real at cluster scale. - -**P10 — It sharpens the stakes on exact indexing.** C trusts the store's -file↔identity map to decide *where new things go*, not only how existing ones are -edited. It is the same map match-first already depends on, so this is a sharpening -of an existing requirement rather than a new one — but a misindexed existing -document now also mis-routes future siblings, so the cost of an indexing bug is -higher under C. - -### Validation and acceptance (reuses the existing gate) - -- C adds **no policy to validate** — there is no template to parse in the base - case. The only new runtime check is the sensitive backstop: a resolved sensitive - path must be identity-complete and must use the encrypted writer. If the - inferred sibling cohort cannot prove that, placement falls back to the secure - canonical path. -- The resolved path still passes the existing path validation (under `spec.path`, - no `..`, correct suffix, inside discovery scope) and the existing - plaintext-append acceptance (never partially manage a file). - -### Keeping it small (C-specific limits) - -- infer only **directory** + **single-file-vs-bundle**; never reverse-engineer a - filename template beyond `{name}.yaml`; -- one fixed precedence ladder; no configurable matching; -- deterministic, documented tie-break; no per-resource status spam; -- sensitive never infers across the plaintext boundary; -- no retroactive moves when the repo layout changes — same rule as A and B. +([gitpath-foreign-content-stringency.md](gitpath-foreign-content-stringency.md)), aborting +the flush. *Still live, and unrelated to inference: placement inherits this failure mode +rather than creating it.* + +**P8 — Explainability becomes a hard requirement, not a nicety.** With A/B a user reads the +CRD to know where a resource will go; with C the answer lived in the repo plus a precedence +ladder, so the scan/dry-run output **must** state, per new resource, the chosen path and the +cohort and ladder step that produced it. *It was never built, which is a fact worth keeping +on this page: the feature shipped without the thing that made it defensible.* What exists +now is the smaller obligation the smaller ladder deserves — `placements_total{source}` per +(GitTarget, type), plus the per-resource log line naming the resolved path. + +**P9 — Cohort lookup cost at scale.** Each new resource scanned the store, so a large repo +× a big resync batch was O(N·M). *Retired: there are no cohorts to index.* + +**P10 — It sharpens the stakes on exact indexing.** C trusted the store's file↔identity map +to decide where *new* things go, not only how existing ones are edited, so a misindexed +document also mis-routed future siblings. *Partly retired: the map still decides +match-first and append safety, but a bad index can no longer propagate into a layout.* + +### Validation and acceptance + +- There is no inferred policy left to validate. A declared template is validated + statically (see [Path validation](#path-validation)); the resolved path is validated at + runtime on every resolution path, and the sensitive backstop (identity-complete, + encrypted writer) is enforced at the write. +- The resolved path still passes the existing path validation (under `spec.path`, no `..`, + correct suffix) and the existing plaintext-append acceptance (never partially manage a + file). ## Sensitive placement and uniqueness @@ -1189,7 +1183,7 @@ If `v1/secrets` were not covered, `default: "all.yaml"` would be invalid for a GitTarget that can write Secrets, because a sensitive resource could otherwise land in a non-identity-complete bundle. -### Brownfield import with no policy (Option C) +### Brownfield import with no policy A user points a GitTarget at an existing folder and sets **no** `placement`. The folder already looks like: @@ -1200,28 +1194,35 @@ clusters/prod/ v1/secrets/app/db.yaml # one Secret, encrypted, one-per-file ``` -A new ConfigMap `cache` in namespace `app` arrives: +A new ConfigMap `cache` in namespace `app` arrives. There is no declaration and no +kustomization, so it lands at the canonical path +**`clusters/prod/app/configmaps/cache.yaml`** — *not* appended to `all.yaml`. The same for a +new Secret, at the canonical encrypted path. The user's bundle is not extended, because +nothing told the operator it was a convention rather than a coincidence, and it counts: -- step 1 finds the type-cohort "ConfigMaps", which lives entirely in `all.yaml` (a - bundle) → the new document is **appended to `all.yaml`** — no `configmaps.yaml` - guessed, no canonical tree created. - -A new Secret `api-token` in namespace `app` arrives: +```promql +sum by (gittarget_name, group, version, resource) ( + increase(gitopsreverser_placements_total{source="canonical"}[24h]) +) +``` -- it is sensitive, so plaintext siblings are ignored; the only sensitive cohort is - `v1/secrets/app/` one-per-file → a new encrypted - **`v1/secrets/app/api-token.yaml`**. +That series is the prompt to declare what the folder means: -A new ConfigMap in a *new* namespace `billing` arrives: +```yaml +placement: + byType: + v1/configmaps: "all.yaml" +``` -- step 1 misses (no `billing` ConfigMaps yet); the type-cohort is the `all.yaml` - bundle, so it is **appended to `all.yaml`** too — the bundle is namespace-agnostic, - so the new namespace needs no new segment. +after which the same ConfigMap is appended to `all.yaml`, and the metric moves to +`source="declared", disposition="appended"`. **This is the behaviour change**: before the +Option C deletion the bundle was extended with no declaration at all. One line of YAML buys +back the old behaviour, and it says on the page what used to be a guess. -Nothing was configured; the layout the user already had simply continued. The same -target with `placement.byType: { "v1/configmaps": "{namespace}/configmaps.yaml" }` -in the B2 shape would instead route ConfigMaps into per-namespace files — B -overriding C where the user has an opinion. +If instead the folder is a kustomize overlay — one `kustomization.yaml` governing the whole +subtree — no declaration is needed: the new document lands beside it and joins its +`resources:` list, `source="kustomize_root"`, because a file that root cannot reach would +never be applied. ## Keeping it small @@ -1229,11 +1230,11 @@ The placement model can get too clever quickly. The first version should stay inside these limits: - GitTarget-level only; -- default to inference (C); declare a layout (B) only when inference cannot reach - it (a layout that does not exist yet, or a custom per-namespace pattern that must - extend to unseen namespaces — P4); -- C infers **directory + bundle-vs-file only** — never a filename or path-segment - template (that is B's job); +- three resolution steps and no more: declared, the folder's single kustomize root, + canonical. Nothing reads the layout of other documents to guess an intent (that was + Option C, removed); +- a layout the ladder cannot reach is **declared**, not inferred — and the + `placements_total{source="canonical"}` series is how its absence is noticed; - keep sensitivity as an internal write-safety classifier, not a second public placement namespace; - prefer exact type-map overrides plus defaults unless ordered matching proves @@ -1254,11 +1255,13 @@ catch-all layout. ## Implementation sketch -1. Settle the surface: **B2 is the declared API, C is the default.** +1. Settle the surface: **B2 is the declared API; the fallbacks carry no API at all.** - B2: one top-level type map (`placement.byType`) plus one `placement.default`; sensitivity is applied as write policy after placement resolves; - - C (default): no API surface — it resolves against the content-derived store; + - the kustomize-root fallback and the canonical path: no API surface, and no + off-switch — see the note on `spec.placement.mode` in + [`open-asks-priority.md`](../design/open-asks-priority.md); - A: ordered `sensitiveRules` / `normalRules`, a later escape hatch only, not implemented. 2. Add the CRD field: @@ -1273,20 +1276,16 @@ catch-all layout. } ``` -4. The default implementation is **Option C, not bare canonical.** Provide it as a - `PlacementPolicy` that resolves against the store; with no `spec.placement` and - an empty repo it falls through to `ResourceIdentifier.ToGitPath()`, so output is - byte-identical to today, and once siblings exist it follows them. When B is - present, B is consulted first and C fills every gap B leaves (canonical becomes - C's step-4 fallback, not the whole policy). The C resolver must: - - read cohorts from the **pre-plan store snapshot** only (P2), via per-plan - by-type and by-namespace indexes (P9); - - resolve a whole batch of new creates together so placement is - order-independent (P2), reusing step 8's grouping; - - never let a sensitive resource infer across the plaintext boundary (it uses - other encrypted/sensitive siblings or the secure canonical fallback); - - emit, for every new create, the chosen path plus the cohort and ladder step - that produced it, into the scan/dry-run output (P8). +4. The fallback is the **kustomize root, then bare canonical.** B is consulted first; + with no `spec.placement` the resolver looks for exactly one supported writable + kustomization and otherwise returns `ResourceIdentifier.ToGitPath()`. It must: + - read the store from the **pre-plan snapshot** only, so a batch of new creates is + order-independent — one new resource must never become another's context (P2); + - never let a sensitive resource join a plaintext file, or a plaintext resource an + encrypted one; refuse instead, with a bounded reason; + - count every resolution by `{source, disposition}` and every refusal by `{reason}`, + labelled with the GitTarget and the type, which is the smaller obligation that + replaces P8's never-built cohort trace. 5. Parse and validate path templates once per GitTarget reconcile. Sensitive resources must resolve to identity-complete paths and must be written through the encrypted writer; the selected filename suffix is not the contract. @@ -1302,11 +1301,18 @@ catch-all layout. - write or append multi-document YAML only for accepted plaintext files. 9. Add sensitive collision checks before rendering encrypted bytes; this is a runtime backstop behind the static identity-completeness validation. -10. Surface placement skips. **Implemented as:** each fail-safe skip is logged - per-resource at the skip site and counted in the resync summary - (`ResyncStats.PlacementSkipped`, distinct from the planner's `Skipped`). A - dedicated GitTarget status condition and a scan/dry-run "why here" trace remain - future work (see open questions). +10. Surface placement outcomes. **Implemented as:** every **successful** placement — one new + document actually in the mirror — increments + `gitopsreverser_placements_total{source, disposition, gittarget_namespace, + gittarget_name, group, version, resource}` and every refusal + `gitopsreverser_placement_refusals_total{reason, …}` with the same target/type labels; + a new file's `resources:` entry is counted by + `gitopsreverser_placement_kustomization_entries_total{outcome}`, whose `failed` value + is the otherwise-invisible "committed but outside every render". Each fail-safe skip + is still logged per-resource at the skip site and counted in the resync summary + (`ResyncStats.PlacementSkipped`, distinct from the planner's `Skipped`). A GitTarget + status surface (`status.layout`) remains future work — see + [`open-asks-priority.md`](../design/open-asks-priority.md), B2. 11. Update chart docs and examples after the API shape is settled. ## Tests @@ -1334,27 +1340,45 @@ Unit tests: - existing moved manifests are updated in place and do not re-run placement; - policy changes do not move existing files. -Option C (sibling inference) unit tests: - -- empty repo reproduces `ResourceIdentifier.ToGitPath()` exactly (C ≡ canonical at - cold start); -- a new resource whose type-cohort is a bundle is appended to that bundle file; -- a new resource whose type-cohort is one-per-file gets a new `{name}.yaml` beside - the siblings; -- a sensitive resource never joins a plaintext bundle and uses the secure - canonical encrypted path when only plaintext siblings exist; -- cohort tie-break is deterministic: most members, then lexically-smallest - directory, then file (P1); -- a batch of new creates against an empty snapshot is order-independent — all - canonical, regardless of input order (P2); -- a declared B entry outranks any inferred cohort (P1); -- a new namespace under a custom per-namespace layout falls back to canonical, while - a new namespace under the canonical layout is unchanged (P4); -- the new file inherits its sibling's `NamespaceSource` (namespace omitted beside - namespace-less kustomize-context siblings, written beside explicit-namespace - ones); -- an inferred path that collides with a `.gittargetignore` pattern fails via the - write-time precondition, naming the inferred path (P7). +Resolution-ladder unit tests: + +- an empty repo reproduces `ResourceIdentifier.ToGitPath()` exactly; +- **every layout sibling inference used to read resolves canonical** — a bundle of the same + type in the same namespace, one-document-per-file, a per-namespace bundle, a directory per + namespace, a single directory holding one namespace, a bundle spanning two namespaces, and + a shared directory spanning two. One table, because it is one contract: the destination + does not depend on where the other documents of the type live; +- the `kube-root-ca.crt`-shaped case keeps its own named test: an object that exists under + the same name in every namespace is never filed onto the first namespace's file, which is + the cascade the deletion retires; +- a declared entry outranks both fallbacks; +- a batch of new creates against one snapshot is order-independent; +- two supported kustomizations are ambiguous and fall to canonical; one supported + kustomization places beside it and reports the `resources:` entry to add; an unsupported + one is never edited or used as a root; +- namespace inheritance: omitted when the governing kustomization's `namespace:` matches the + resource's own namespace (for a declared path as well as the kustomize-root one), written + explicitly when it names a different namespace, and never claimed with no context; +- a sensitive resource never joins a plaintext file and never reuses an existing + `.sops.yaml` directory without a declaration; +- a declared path onto a file holding a non-editable construct never appends; +- refusals carry their bounded reason, and each is counted: an escaping template + (`invalid_path`), a sensitive collision (`sensitive_append`), a plaintext resource routed + at an encrypted file (`plaintext_onto_encrypted`), and mixed sensitivity onto one new file + (`mixed_sensitivity_new_file`); +- a resolved path that collides with a `.gittargetignore` pattern fails via the write-time + precondition, naming the path (P7). + +Metric tests (`internal/git/placement_metrics_test.go`), driving the real write path: + +- a canonical fall-back is labelled with the GitTarget and the type key a `byType` line + would name — the labels are the feature, so they are asserted rather than the count alone; +- declared and kustomize-root placements are distinguishable from canonical, and + `kustomize_root` is never counted as a fall-back; +- a declared bundle records `disposition="appended"`; +- a refused resource is counted as a refusal and **never also** as a placement; +- a `resources:` entry that could not be added is counted as `failed`; +- a resync's create carries the same target labels as a live create. Integration/e2e tests: @@ -1380,19 +1404,25 @@ Integration/e2e tests: model, or should flat discovery be dropped before placement rules land? - Should placement rule matches include `watchRuleNames` later for users who want rule-origin-aware placement without moving policy onto WatchRule? -- For Option C, should step 3 (same namespace, any type) ship at all, or only fire - into an already-heterogeneous bundle file (P5)? Dropping it keeps inference to - per-type cohorts and canonical, and pushes the namespace-bundle layout onto B. -- Should C ever offer a one-time, opt-in "adopt/normalize" pass that *does* move - existing files to a declared B layout, or is non-retroactive placement absolute? - (Today both A/B and C never move existing files; this would be a deliberate, - separate, destructive feature.) -- How much of C's cohort/ladder reasoning belongs in GitTarget *status* versus the - scan/dry-run output (P8)? Status must stay bounded; the per-resource "why here" - trace likely belongs only in the dry-run. -- When B and C disagree for a resource (B names a path, C would infer another), - confirm B always wins and C only fills B's gaps — and that this precedence is - visible in the dry-run. +- ~~For Option C, should step 3 (same namespace, any type) ship at all (P5)?~~ **Closed by + the removal**: step 3 was never implemented and the ladder it belonged to is gone. +- ~~When B and C disagree, confirm B always wins.~~ **Closed**: there is nothing left to + disagree with. A declaration is consulted first and the fallbacks are structural. +- Should a one-time, opt-in "adopt/normalize" pass ever move *existing* files to a declared + layout, or is non-retroactive placement absolute? Today nothing moves an existing + document; this would be a deliberate, separate, destructive feature. The removal makes it + more interesting, not less: a brownfield folder that wants its bundle back declares it, + and the documents already at canonical paths stay where they are. +- How much of the "why here" answer belongs in GitTarget *status* rather than in a metric? + `placements_total{source}` says which (target, type) fell back; it does not say what the + operator understood about the folder, and it expires with the scrape window. That is what + `status.layout` (B2 in the config-surface doc) is for, and the two are complements: a + status field is what a `kubectl get -o yaml` in a bug report contains. +- Whether the first canonical fall-back for a (target, type) should also raise a + `corev1.Event` on the GitTarget for timeliness. Placement runs on the branch worker, which + has no recorder and no reconcile context, so this is a hand-back over the existing + refusal→condition seam rather than a one-line addition — see the same section of + [`open-asks-priority.md`](../design/open-asks-priority.md). - Should the Validated gate learn about operator-configured *additional* sensitive types (beyond core Secrets) so a bundling `default` that could catch one is rejected up front, instead of relying on the write-time guards to skip those diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index e4a484e7..d22a4056 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -213,7 +213,8 @@ func resolvePlacementPolicy(spec *v1alpha3.GitTargetPlacementSpec) *manifestanal // the live-write path (resync resolves target.Placement directly, so the two paths // would diverge). GitTarget paths never overlap, so at most one target can match; a // base with no matching target (e.g. an event whose target metadata could not be -// resolved) gets no declared policy, falling through to sibling inference. +// resolved) gets no declared policy, falling through to the kustomize root and then the +// canonical path. func placementPolicyForBase( targets map[pendingTargetKey]ResolvedTargetMetadata, base string, diff --git a/internal/git/placement_metrics.go b/internal/git/placement_metrics.go new file mode 100644 index 00000000..f6129ac0 --- /dev/null +++ b/internal/git/placement_metrics.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "errors" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// placementTarget is the GitTarget a write batch belongs to, carried purely so the +// placement metrics can name it. A placement counter that cannot say WHICH target and +// WHICH type resolved how is not actionable — the fix for a fall-back to canonical is one +// `placement.byType` line on one GitTarget — and it is the objection the design doc raised +// against shipping a bare `placement_fell_back_total`. See +// docs/design/open-asks-priority.md and docs/interpreting-metrics.md. +// +// The label keys are gittarget_namespace / gittarget_name, never namespace / name: a +// Prometheus pod scrape with honor_labels=false overwrites a metric's `namespace` +// attribute with the scraped pod's own, which silently breaks every per-target selector. +// That convention is set by TargetReconcileCompletedTotal and PruneRetainedDocumentsTotal. +type placementTarget struct { + namespace string + name string +} + +func (t placementTarget) attrs() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("gittarget_namespace", t.namespace), + attribute.String("gittarget_name", t.name), + } +} + +// placementTargetForEvents reads the GitTarget identity off a batch's events. Every event +// in one flush shares a base and therefore a GitTarget, so the first event that carries an +// identity describes the whole batch. Events built by tests and by the CLI carry none, and +// then the labels are empty rather than absent — an unlabelled series is still a truthful +// count of placements, and inventing a placeholder would make one look like a real target. +func placementTargetForEvents(events []Event) placementTarget { + for _, ev := range events { + if ev.GitTargetName != "" || ev.GitTargetNamespace != "" { + return placementTarget{namespace: ev.GitTargetNamespace, name: ev.GitTargetName} + } + } + return placementTarget{} +} + +// resourceAttrs labels a placement by the type whose rule may be missing. group/version/ +// resource is exactly the shape of a placement.byType key (PlacementTypeKey), so a +// `source="canonical"` series reads directly as the line the GitTarget needs. The resource +// NAME is deliberately not a label: it is unbounded, and it is in the log line. +func resourceAttrs(id types.ResourceIdentifier) []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("group", id.Group), + attribute.String("version", id.Version), + attribute.String("resource", id.Resource), + } +} + +const ( + // placementDispositionNewFile is a placement that wrote a file of its own. + placementDispositionNewFile = "new_file" + // placementDispositionAppended is a placement that added a document to a file that + // already held one — only ever reachable through a declared bundling template now that + // sibling inference is gone, which is what makes the two dispositions worth splitting: + // `disposition="appended"` with `source!="declared"` should not exist. + placementDispositionAppended = "appended" +) + +// recordPlacement counts one new document actually written at a resolved path. It is +// recorded after the write, not at resolution: a refused or skipped resource is counted by +// recordPlacementRefusal instead, so the two counters partition the population — every new +// resource is mirrored (here) or not mirrored (there), and never both. +func recordPlacement( + ctx context.Context, + target placementTarget, + id types.ResourceIdentifier, + source manifestanalyzer.PlacementSource, + appended bool, +) { + if telemetry.PlacementsTotal == nil { + return + } + disposition := placementDispositionNewFile + if appended { + disposition = placementDispositionAppended + } + attrs := append(target.attrs(), resourceAttrs(id)...) + attrs = append(attrs, + attribute.String("source", string(source)), + attribute.String("disposition", disposition), + ) + telemetry.PlacementsTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) +} + +// recordPlacementRefusal counts one resource the writer declined to place. Every increment +// is a resource absent from the mirror, which is why it is a counter of its own rather than +// a `source` value: a dashboard that added the two together would report a skipped Secret +// as a successful placement. +func recordPlacementRefusal( + ctx context.Context, + target placementTarget, + id types.ResourceIdentifier, + reason manifestanalyzer.PlacementRefusalReason, +) { + if telemetry.PlacementRefusalsTotal == nil { + return + } + attrs := append(target.attrs(), resourceAttrs(id)...) + attrs = append(attrs, attribute.String("reason", string(reason))) + telemetry.PlacementRefusalsTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) +} + +const ( + kustomizationEntryAdded = "added" + kustomizationEntryNoChange = "no_change" + kustomizationEntryFailed = "failed" +) + +// recordKustomizationEntry counts one attempt to register a newly placed file in the +// resources: list of the kustomization that governs it. `failed` is the outcome worth +// alerting on and the one with no other trace: the document is committed and the entry is +// not, so kustomize never builds the file — the object is in Git, looks mirrored, and is +// applied by nothing. +// +// It carries no path label on purpose: a file path is unbounded, and the log line at the +// site already names the kustomization, the entry and the resource. +func recordKustomizationEntry(ctx context.Context, target placementTarget, outcome string) { + if telemetry.PlacementKustomizationEntriesTotal == nil { + return + } + attrs := append(target.attrs(), attribute.String("outcome", outcome)) + telemetry.PlacementKustomizationEntriesTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) +} + +// placementRefusalReason classifies a LocateNew error for the refusal counter. A refusal +// the analyzer raised carries its own bounded reason; anything else is an error shape that +// did not exist when this was written, and it is counted as `unclassified` rather than +// dropped — a refusal missing from the metric is worse than one with a vague label. +func placementRefusalReason(err error) manifestanalyzer.PlacementRefusalReason { + var refusal *manifestanalyzer.PlacementRefusedError + if errors.As(err, &refusal) { + return refusal.Reason + } + return "unclassified" +} diff --git a/internal/git/placement_metrics_test.go b/internal/git/placement_metrics_test.go new file mode 100644 index 00000000..fac9c38f --- /dev/null +++ b/internal/git/placement_metrics_test.go @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "os" + "testing" + + gogit "github.com/go-git/go-git/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + v1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +const ( + placementsMetric = "gitopsreverser_placements_total" + placementRefusalsMetric = "gitopsreverser_placement_refusals_total" + kustomizationEntriesMetric = "gitopsreverser_placement_kustomization_entries_total" + metricsTestGitTargetName = "prod-mirror" + metricsTestGitTargetNamespace = "tenant-a" +) + +// targetedConfigMapEvent is newConfigMapEvent plus the GitTarget identity the live path +// carries on every event. The identity is what makes the placement counters actionable — +// "which target needs a byType line" — so the tests drive the real path with it set. +func targetedConfigMapEvent() Event { + event := newConfigMapEvent("cache", "app") + event.GitTargetName = metricsTestGitTargetName + event.GitTargetNamespace = metricsTestGitTargetNamespace + return event +} + +func targetedSecretEvent(name, namespace string) Event { + return Event{ + Object: &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{"name": name, "namespace": namespace}, + }}, + Identifier: types.NewResourceIdentifier("", "v1", "secrets", namespace, name), + Operation: "CREATE", + GitTargetName: metricsTestGitTargetName, + GitTargetNamespace: metricsTestGitTargetNamespace, + } +} + +// placementLabels is the label set every placement sample must carry: the GitTarget that +// owns the write, and the type key a placement.byType entry would name. +func placementLabels(resource string, extra map[string]string) map[string]string { + labels := map[string]string{ + "gittarget_namespace": metricsTestGitTargetNamespace, + "gittarget_name": metricsTestGitTargetName, + "group": "", + "version": "v1", + "resource": resource, + } + for k, v := range extra { + labels[k] = v + } + return labels +} + +func flushWithPolicy( + t *testing.T, + worktree *gogit.Worktree, + policy *manifestanalyzer.PlacementPolicy, + events ...Event, +) { + t.Helper() + w := &BranchWorker{contentWriter: newContentWriter(types.SensitiveResourcePolicy{}), mapper: configMapMapper()} + _, err := w.flushEventsToWorktree(context.Background(), worktree, "", events, policy, v1alpha3.PruneOnEvent) + require.NoError(t, err) +} + +// The signal the Option C deletion owes its users: a repository whose layout this operator +// cannot derive gets the canonical path, and `source="canonical"` on placements_total names +// the GitTarget and the type that needs one `placement.byType` line. Without the labels the +// counter would only say a fall-back happened somewhere, which is not a fix anybody can act +// on — see docs/design/open-asks-priority.md. +func TestPlacementMetrics_CanonicalFallbackNamesTargetAndType(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + seedPlacedManifest(t, worktree, "overlays/test/configmap-existing.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: existing\n namespace: app\ndata:\n k: v\n") + + flushWithPolicy(t, worktree, nil, targetedConfigMapEvent()) + + got, ok := telemetry.CollectInt64Sum(reader, placementsMetric, placementLabels("configmaps", map[string]string{ + "source": "canonical", + "disposition": "new_file", + })) + require.True(t, ok, "expected a canonical placement sample labelled by target and type") + assert.Equal(t, int64(1), got) +} + +// A declared template is the answer to a canonical fall-back, so the two must be +// distinguishable in the same series: `source="declared"` is how an operator confirms the +// byType line they added is actually the one being used. +func TestPlacementMetrics_DeclaredPlacementIsCountedAsDeclared(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + policy := &manifestanalyzer.PlacementPolicy{ + ByType: map[string]string{"v1/configmaps": "{namespace}/{name}.yaml"}, + } + + flushWithPolicy(t, worktree, policy, targetedConfigMapEvent()) + + got, ok := telemetry.CollectInt64Sum(reader, placementsMetric, placementLabels("configmaps", map[string]string{ + "source": "declared", + "disposition": "new_file", + })) + require.True(t, ok, "expected a declared placement sample") + assert.Equal(t, int64(1), got) +} + +// Appending to a bundle is only reachable by declaring one now, and `disposition="appended"` +// is what proves the bundle is being grown rather than a new file written beside it — +// the difference between a bundling policy that works and one whose path is subtly wrong. +func TestPlacementMetrics_DeclaredBundleRecordsAppendedDisposition(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + seedPlacedManifest(t, worktree, "all.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: app\ndata:\n k: v\n") + policy := &manifestanalyzer.PlacementPolicy{ByType: map[string]string{"v1/configmaps": "all.yaml"}} + + flushWithPolicy(t, worktree, policy, targetedConfigMapEvent()) + + got, ok := telemetry.CollectInt64Sum(reader, placementsMetric, placementLabels("configmaps", map[string]string{ + "source": "declared", + "disposition": "appended", + })) + require.True(t, ok, "expected an appended placement sample") + assert.Equal(t, int64(1), got) +} + +// The structural fallback has its own source value, and it is the one that must NOT be read +// as a missing rule: a folder with one kustomize root is placing files where they render, +// which is the correct answer with no declaration at all. A dashboard that lumped it in with +// canonical would report every well-formed overlay as misconfigured. The successful +// resources: entry is counted too — it is the half that makes the file build. +func TestPlacementMetrics_KustomizeRootSourceAndEntryAdded(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + seedPlacedManifest(t, worktree, "overlays/test/kustomization.yaml", + "namespace: app\nresources:\n - deployment.yaml\n") + seedPlacedManifest(t, worktree, "overlays/test/deployment.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: app\n") + + flushWithPolicy(t, worktree, nil, targetedConfigMapEvent()) + + got, ok := telemetry.CollectInt64Sum(reader, placementsMetric, placementLabels("configmaps", map[string]string{ + "source": "kustomize_root", + "disposition": "new_file", + })) + require.True(t, ok, "expected a kustomize_root placement sample") + assert.Equal(t, int64(1), got) + + entries, ok := telemetry.CollectInt64Sum(reader, kustomizationEntriesMetric, map[string]string{ + "gittarget_namespace": metricsTestGitTargetNamespace, + "gittarget_name": metricsTestGitTargetName, + "outcome": "added", + }) + require.True(t, ok, "expected the resources: entry to be counted") + assert.Equal(t, int64(1), entries) +} + +// The invisible failure this counter exists for: the document is committed and its +// resources: entry is not, so kustomize never builds the file. The object is in Git, looks +// mirrored, and nothing applies it. A kustomization with no resources: sequence to append to +// is the reproducible shape of that. +func TestPlacementMetrics_KustomizationEntryFailureIsCounted(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + seedPlacedManifest(t, worktree, "overlays/test/kustomization.yaml", "namespace: app\n") + + flushWithPolicy(t, worktree, nil, targetedConfigMapEvent()) + + failed, ok := telemetry.CollectInt64Sum(reader, kustomizationEntriesMetric, map[string]string{ + "gittarget_namespace": metricsTestGitTargetNamespace, + "gittarget_name": metricsTestGitTargetName, + "outcome": "failed", + }) + require.True(t, ok, "a resources: entry that could not be added must be counted") + assert.Equal(t, int64(1), failed) +} + +// A refused resource is one the mirror does NOT hold, so it is a counter of its own with a +// bounded reason — never a `source` value on placements_total, which would report a skipped +// Secret as a successful placement. Before this it left a log line and, on the resync path +// only, one integer in a summary. +func TestPlacementMetrics_SensitiveCollisionCountsARefusal(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + seedPlacedManifest(t, worktree, "secrets/app.sops.yaml", + "apiVersion: v1\nkind: Secret\nmetadata:\n name: other\n namespace: app\nsops:\n version: \"3\"\n") + // Not identity-complete: every Secret in a namespace renders the same path, so the + // second one collides with the first. + policy := &manifestanalyzer.PlacementPolicy{ + ByType: map[string]string{"v1/secrets": "secrets/{namespace}.sops.yaml"}, + } + + flushWithPolicy(t, worktree, policy, targetedSecretEvent("api-token", "app")) + + refusals, ok := telemetry.CollectInt64Sum(reader, placementRefusalsMetric, + placementLabels("secrets", map[string]string{"reason": "sensitive_append"})) + require.True(t, ok, "expected a refusal sample naming the sensitive-append reason") + assert.Equal(t, int64(1), refusals) + + placements, _ := telemetry.CollectInt64Sum(reader, placementsMetric, placementLabels("secrets", nil)) + assert.Zero(t, placements, "a refused resource must never also count as a placement") +} + +// A declared template that escapes spec.path is refused by the runtime path gate, and it is +// worth its own reason: unlike the sensitivity refusals it is a template that can never work +// for any resource, so one series says "fix the policy" rather than "resolve a collision". +func TestPlacementMetrics_EscapingTemplateCountsAnInvalidPathRefusal(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + policy := &manifestanalyzer.PlacementPolicy{Default: "../../outside.yaml"} + + flushWithPolicy(t, worktree, policy, targetedConfigMapEvent()) + + refusals, ok := telemetry.CollectInt64Sum(reader, placementRefusalsMetric, + placementLabels("configmaps", map[string]string{"reason": "invalid_path"})) + require.True(t, ok, "expected an invalid_path refusal sample") + assert.Equal(t, int64(1), refusals) +} + +// Two resources of different sensitivity routed by one bundling default onto the SAME +// brand-new file: LocateNew cannot see the collision (it resolves each resource against the +// pre-batch snapshot), so the writer refuses the second one. That refusal is counted from the +// same closed reason set as the analyzer's, so "resources we declined to place" is one series. +func TestPlacementMetrics_MixedSensitivityNewFileCountsARefusal(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + policy := &manifestanalyzer.PlacementPolicy{Default: "{namespace}/all.yaml"} + + w := &BranchWorker{ + contentWriter: newContentWriter(types.SensitiveResourcePolicy{}), + mapper: configMapMapper(), + } + _, err = w.flushEventsToWorktree( + context.Background(), + worktree, + "", + []Event{targetedConfigMapEvent(), targetedSecretEvent("api-token", "app")}, + policy, + v1alpha3.PruneOnEvent, + ) + require.NoError(t, err, "a co-mingling refusal is skipped, not returned as a batch error") + + refusals, ok := telemetry.CollectInt64Sum(reader, placementRefusalsMetric, + map[string]string{ + "gittarget_namespace": metricsTestGitTargetNamespace, + "gittarget_name": metricsTestGitTargetName, + "reason": "mixed_sensitivity_new_file", + }) + require.True(t, ok, "expected a mixed-sensitivity refusal sample") + assert.Equal(t, int64(1), refusals) +} + +// The resync path synthesises its events from the desired snapshot, so they carry no +// GitTarget identity of their own. Its placements must still land in the labelled series: +// otherwise a fall-back to canonical would be visible for a live create and invisible for +// the reconcile that produced the same file, and which of the two ran is not something the +// operator chose. +func TestPlacementMetrics_ResyncPlacementsCarryTheTargetLabels(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + writer := newContentWriter(types.SensitiveResourcePolicy{}) + worktree := newWorktreeForTest(t) + w := &BranchWorker{contentWriter: writer, mapper: configMapMapper()} + + _, _, err = w.applyResyncToWorktree( + context.Background(), + worktree, + "", + ResolvedTargetMetadata{Name: metricsTestGitTargetName, Namespace: metricsTestGitTargetNamespace}, + []manifestanalyzer.DesiredResource{desiredCM("fresh", "red")}, + nil, + ) + require.NoError(t, err) + + got, ok := telemetry.CollectInt64Sum(reader, placementsMetric, map[string]string{ + "gittarget_namespace": metricsTestGitTargetNamespace, + "gittarget_name": metricsTestGitTargetName, + "source": "canonical", + }) + require.True(t, ok, "a resync's create must be counted with the target labels") + assert.Equal(t, int64(1), got) +} + +// The ordering guard behind that counter. `appendKustomizationResource` used to run BEFORE +// `placeNewDocument`, so a placement the writer then REFUSED still gained a resources: entry. +// The reachable shape is a declared path onto an existing multi-document file the writer will +// not overwrite (it holds a document we cannot account for): we decline to own the file, and +// registering it anyway puts foreign content into the folder's render on our say-so, counted as +// outcome="added" -- the value that is supposed to mean "the file we just wrote will build". +func TestPlacementMetrics_RefusedPlacementLeavesTheKustomizationAlone(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + worktree := newWorktreeForTest(t) + kustomization := seedPlacedManifest(t, worktree, "overlays/test/kustomization.yaml", + "namespace: app\nresources:\n - listed.yaml\n") + seedPlacedManifest(t, worktree, "overlays/test/listed.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: listed\n namespace: app\ndata:\n k: v\n") + // Two documents. Document 1 is the incoming resource (app/cache) written with a merge key, + // which manifestedit refuses to edit, so it does not claim its identity for an in-place + // match and the file stays multi-document: the writer will not rewrite it wholesale, + // because that would drop document 0. + seedPlacedManifest(t, worktree, "overlays/test/multi.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: other\n namespace: app\ndata:\n k: v\n"+ + "---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cache\n namespace: app\n"+ + "data: &d\n color: blue\nextra:\n <<: *d\n") + + before, err := os.ReadFile(kustomization) + require.NoError(t, err) + + policy := &manifestanalyzer.PlacementPolicy{ + ByType: map[string]string{"v1/configmaps": "overlays/test/multi.yaml"}, + } + flushWithPolicy(t, worktree, policy, targetedConfigMapEvent()) + + after, err := os.ReadFile(kustomization) + require.NoError(t, err) + assert.Equal(t, string(before), string(after), + "a refused placement must not register a resources: entry for a file it declined to own") + + added, _ := telemetry.CollectInt64Sum(reader, kustomizationEntriesMetric, map[string]string{ + "gittarget_namespace": metricsTestGitTargetNamespace, + "gittarget_name": metricsTestGitTargetName, + "outcome": "added", + }) + assert.Zero(t, added, "a refused resource must never count as an added resources: entry") +} diff --git a/internal/git/placement_test.go b/internal/git/placement_test.go index 1de8dbf8..d716e0a7 100644 --- a/internal/git/placement_test.go +++ b/internal/git/placement_test.go @@ -63,7 +63,12 @@ func TestPlacement_DeclaredPolicy_NewFile(t *testing.T) { assert.Contains(t, string(got), "color: blue") } -func TestPlacement_SiblingInference_BesideExistingFile(t *testing.T) { +// The write path's half of the Option C deletion: an existing document of the same type +// in the same namespace no longer attracts the new file. The folder here has no +// kustomization, so there is no structural root either, and the resource lands at the +// canonical path — where a human can see it, and where one `placement.byType` line moves +// it if the repository wants it in the overlay. +func TestPlacement_ExistingSiblingFile_DoesNotAttractTheNewFile(t *testing.T) { worktree := newWorktreeForTest(t) root := worktree.Filesystem.Root() seedPlacedManifest(t, worktree, "overlays/test/configmap-existing.yaml", @@ -72,19 +77,28 @@ func TestPlacement_SiblingInference_BesideExistingFile(t *testing.T) { changed := applyEventsWithPolicy(t, worktree, nil, newConfigMapEvent("cache", "podinfo-test")) require.True(t, changed) - got, err := os.ReadFile(filepath.Join(root, "overlays/test/cache.yaml")) - require.NoError(t, err, "the new file should land beside its sibling, not at the canonical path") + got, err := os.ReadFile(filepath.Join(root, "podinfo-test/configmaps/cache.yaml")) + require.NoError(t, err, "the new file must land at the canonical path") assert.Contains(t, string(got), "name: cache") + + _, statErr := os.Stat(filepath.Join(root, "overlays/test/cache.yaml")) + assert.True(t, os.IsNotExist(statErr), "the sibling's directory must not be inferred as the destination") } -func TestPlacement_BundleAppend_ExistingMultiDocFile(t *testing.T) { +// Appending to a bundle is still supported — it is now reached by DECLARING the bundle +// rather than by the writer noticing one. The append machinery itself (splice one +// document, leave every other byte alone) is unchanged, and this is what pins it. +func TestPlacement_DeclaredBundle_AppendsToExistingMultiDocFile(t *testing.T) { worktree := newWorktreeForTest(t) seeded := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: app\ndata:\n k: v\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: b\n namespace: app\ndata:\n k: v\n" full := seedPlacedManifest(t, worktree, "all.yaml", seeded) + policy := &manifestanalyzer.PlacementPolicy{ + ByType: map[string]string{"v1/configmaps": "all.yaml"}, + } - changed := applyEventsWithPolicy(t, worktree, nil, newConfigMapEvent("cache", "app")) + changed := applyEventsWithPolicy(t, worktree, policy, newConfigMapEvent("cache", "app")) require.True(t, changed) got, err := os.ReadFile(full) @@ -96,12 +110,12 @@ func TestPlacement_BundleAppend_ExistingMultiDocFile(t *testing.T) { "exactly one document must be added, not a replace") } -// A new resource whose siblings are in a kustomize-namespace-inferred bundle -// must not write metadata.namespace into that bundle — otherwise an incidental -// resource sharing the namespace (e.g. a cluster-injected ConfigMap watched by -// too broad a WatchRule) would break the "no namespace: in this file" -// convention every other document in the bundle already follows. -func TestPlacement_BundleAppend_OmitsNamespaceInKustomizeContext(t *testing.T) { +// A declared destination inside a kustomize context must not write metadata.namespace: +// the kustomization's namespace: transformer supplies it, and repeating it would break the +// "no namespace: in this file" convention every other document in the bundle follows. The +// obligation used to come from reading the siblings' bytes; it now comes from the +// governing kustomization, which is the thing that actually decides the rendered namespace. +func TestPlacement_DeclaredBundle_OmitsNamespaceInKustomizeContext(t *testing.T) { worktree := newWorktreeForTest(t) kustYAML := "namespace: app\nresources:\n - all.yaml\n" seedPlacedManifest(t, worktree, "overlays/test/kustomization.yaml", kustYAML) @@ -109,8 +123,11 @@ func TestPlacement_BundleAppend_OmitsNamespaceInKustomizeContext(t *testing.T) { "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: b\ndata:\n k: v\n" full := seedPlacedManifest(t, worktree, "overlays/test/all.yaml", seeded) + policy := &manifestanalyzer.PlacementPolicy{ + ByType: map[string]string{"v1/configmaps": "overlays/test/all.yaml"}, + } - changed := applyEventsWithPolicy(t, worktree, nil, newConfigMapEvent("cache", "app")) + changed := applyEventsWithPolicy(t, worktree, policy, newConfigMapEvent("cache", "app")) require.True(t, changed) got, err := os.ReadFile(full) diff --git a/internal/git/plan_flush.go b/internal/git/plan_flush.go index fd783d62..95b9c929 100644 --- a/internal/git/plan_flush.go +++ b/internal/git/plan_flush.go @@ -84,6 +84,7 @@ func (w *BranchWorker) flushEventsToWorktree( mapper := w.mapperForCluster(clusterIDForEvents(events)) batch := newWriteBatch(ctx, w.contentWriter, mapper, scoped.scan, policy, scoped.writeSubdir) batch.pruneMode = pruneMode + batch.target = placementTargetForEvents(events) if err := batch.refusal(); err != nil { return false, err } @@ -120,9 +121,14 @@ type writeBatch struct { // entitled to claim (its resources: entry can legitimately fail to be added — see // appendKustomizationResource — leaving the file written but outside every render). putToKustomize bool + // target is the GitTarget this batch writes for, carried only so the placement metrics + // can name it (see placement_metrics.go). It is set by the caller — the live path reads + // it off the events, the resync path from the request — and is empty for the CLI and for + // tests, where the counters are simply unlabelled. + target placementTarget // policy is the GitTarget's declared new-file placement policy, consulted // only for a resource with no existing document. nil means no declared policy — - // placement falls through to sibling inference and then the canonical path. + // placement falls through to the folder's one kustomize root and then the canonical path. policy *manifestanalyzer.PlacementPolicy // pruneMode is the GitTarget's effective spec.prune.mode, gating the EXPLICIT delete // path only (applyDelete). The inferred mark-and-sweep is gated a layer up, in the @@ -332,7 +338,7 @@ func wroteBytes(o upsertOutcome) bool { } // createNew resolves the placement of a resource with no existing document — -// declared policy (Option B), sibling inference (Option C), or the canonical +// declared policy (Option B), the folder's one kustomize root, or the canonical // fallback — per docs/spec/gittarget-new-file-placement-rules.md, // adds the kustomize resources: entry the placement may require, and writes the new // document: a brand-new file, or an additional document appended to an existing @@ -357,8 +363,10 @@ func (wb *writeBatch) createNew(ctx context.Context, event Event) (upsertOutcome WriteScope: wb.writeSubdir, }) if err != nil { + refusal := placementRefusalReason(err) log.FromContext(ctx).Info("Skipping new resource: placement could not be resolved safely", - "resource", event.Identifier.String(), "reason", err.Error()) + "resource", event.Identifier.String(), "refusal", string(refusal), "reason", err.Error()) + recordPlacementRefusal(ctx, wb.target, event.Identifier, refusal) return upsertSkippedUnsafe, nil } @@ -367,10 +375,6 @@ func (wb *writeBatch) createNew(ctx context.Context, event Event) (upsertOutcome // still holds both — see intentFor. live := event.Object - if placement.Kustomization != nil { - wb.appendKustomizationResource(ctx, event, placement) - } - // A destination that infers its namespace from build context (a kustomization's // namespace: transformer) must keep metadata.namespace out of the written bytes, // exactly as patchExisting already does for an in-place edit of an existing @@ -381,10 +385,32 @@ func (wb *writeBatch) createNew(ctx context.Context, event Event) (upsertOutcome event.Object.SetNamespace("") } - outcome, err := wb.placeNewDocument(ctx, event, placement, sensitive) + outcome, refusal, err := wb.placeNewDocument(ctx, event, placement, sensitive) if err != nil || !wroteBytes(outcome) { + // A skipped write is a resource the mirror does not hold. Count it with the refusals + // LocateNew raised, from the same closed reason set, so "resources we declined to + // place" is one series rather than a log line here and a metric there. + if outcome == upsertSkippedUnsafe { + recordPlacementRefusal(ctx, wb.target, event.Identifier, refusal) + } return outcome, err } + // Recorded here rather than at resolution: this is the point at which the document is + // really in the mirror at this path, so placements_total and placement_refusals_total + // partition every new resource instead of double-counting the ones that resolved and + // then could not be written. + recordPlacement(ctx, wb.target, event.Identifier, placement.Source, placement.Append) + + // AFTER the write, for the same reason the placement is counted here. placeNewDocument can + // still decline — a multi-document target it will not overwrite, or a new file that would + // mix sensitive and plaintext documents — and registering the entry first meant a resource we + // REFUSED still put its file into the folder's render. For the multi-document case that is + // foreign content we declined to own, added to resources: on our say-so; and either way it + // counted as outcome="added", the value that is supposed to mean "the file we just wrote will + // build". Pinned by TestPlacementMetrics_RefusedPlacementLeavesTheKustomizationAlone. + if placement.Kustomization != nil { + wb.appendKustomizationResource(ctx, event, placement) + } // A new document that joins a kustomization's resources: list is INSIDE a render root, so // the folder's images:/replicas: entries govern it from the moment it lands — and we do not @@ -403,14 +429,21 @@ func (wb *writeBatch) createNew(ctx context.Context, event Event) (upsertOutcome // placeNewDocument writes the new document at its resolved placement: appended to an existing // accepted bundle, folded into a same-batch cold bundle, or as a file of its own. +// +// It returns the refusal reason alongside the outcome, and only for upsertSkippedUnsafe, so +// the caller can count WHY a new resource was left out of the mirror without inspecting a log +// message. The multi-document refusal is attributed here rather than inside writeWholeFile +// because that function also serves in-place updates, where the same skip is not a placement +// decision at all. func (wb *writeBatch) placeNewDocument( ctx context.Context, event Event, placement manifestanalyzer.PlacementResult, sensitive bool, -) (upsertOutcome, error) { +) (upsertOutcome, manifestanalyzer.PlacementRefusalReason, error) { if placement.Append { - return wb.appendNewDocument(ctx, event, placement.Path) + outcome, err := wb.appendNewDocument(ctx, event, placement.Path) + return outcome, "", err } buf := wb.buffer(placement.Path) @@ -435,11 +468,16 @@ func (wb *writeBatch) placeNewDocument( log.FromContext(ctx).Info( "Skipping new resource: sensitive and plaintext resources must not share a new file", "resource", event.Identifier.String(), "file", placement.Path, "sensitive", sensitive) - return upsertSkippedUnsafe, nil + return upsertSkippedUnsafe, manifestanalyzer.PlacementRefusedMixedSensitivityNewFile, nil } - return wb.writeColdBundleMember(ctx, event, placement.Path, sensitive) + outcome, err := wb.writeColdBundleMember(ctx, event, placement.Path, sensitive) + return outcome, "", err + } + outcome, err := wb.writeWholeFile(ctx, event, placement.Path) + if outcome == upsertSkippedUnsafe { + return outcome, manifestanalyzer.PlacementRefusedMultiDocumentTarget, err } - return wb.writeWholeFile(ctx, event, placement.Path) + return outcome, "", err } // writeColdBundleMember writes a resource with no existing document to rel, a @@ -555,16 +593,26 @@ func (wb *writeBatch) appendKustomizationResource( buf := wb.buffer(k.Path) if buf.current == nil { - return // the kustomization vanished within this batch; nothing to edit + // The kustomization vanished within this batch; nothing to edit — and the file it + // would have registered is now outside every render, which is the same user-visible + // outcome as a failed edit, so it is counted as one. + recordKustomizationEntry(ctx, wb.target, kustomizationEntryFailed) + return } res, diags := manifestedit.AppendKustomizationResource(k.Path, buf.current, entry) switch res.Mode { case manifestedit.EditPatched: buf.current = res.Content + recordKustomizationEntry(ctx, wb.target, kustomizationEntryAdded) log.FromContext(ctx).Info("Added resources: entry for new file", "kustomization", k.Path, "entry", entry, "resource", event.Identifier.String()) case manifestedit.EditNoChange: + recordKustomizationEntry(ctx, wb.target, kustomizationEntryNoChange) case manifestedit.EditSkipped, manifestedit.EditDeleted, manifestedit.EditWholeReplace: + // The document is committed and its resources: entry is not, so kustomize will never + // build the file: it is in Git, it looks mirrored, and nothing applies it. The counter + // is the only signal that is not a log line. + recordKustomizationEntry(ctx, wb.target, kustomizationEntryFailed) log.FromContext(ctx).Info("Could not add resources: entry for new file", "kustomization", k.Path, "entry", entry, "resource", event.Identifier.String()) logManifestDiagnostics(ctx, diags) diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index 98ce92f4..065e83b1 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -350,6 +350,12 @@ func (w *BranchWorker) applyResyncToWorktree( target.Placement, scoped.writeSubdir, ) + // The resync's events are synthesised from the desired snapshot and carry no GitTarget + // identity of their own, so the placement metrics take it from the resolved metadata. A + // resync creates resources exactly as the live path does, and its placements must land in + // the same series — otherwise a fall-back to canonical would be visible for a live create + // and invisible for the reconcile that produced the same file. + batch.target = placementTarget{namespace: target.Namespace, name: target.Name} // First materialization is the adoption gate: refuse a subtree that holds content the // operator cannot safely manage (unsupported kustomization, duplicate identity, impure // or non-KRM files, foreign content, a catastrophic .gittargetignore) and commit nothing, diff --git a/internal/git/resync_flush_test.go b/internal/git/resync_flush_test.go index 10c4a013..341d042d 100644 --- a/internal/git/resync_flush_test.go +++ b/internal/git/resync_flush_test.go @@ -289,12 +289,14 @@ func TestResync_FoldsCreateUpdateDropTogether(t *testing.T) { _, dropErr := os.Stat(dropFull) assert.True(t, os.IsNotExist(dropErr)) - // Placement: with existing ConfigMap siblings ("keep", "drop") each in their own file - // under apps/, a genuinely new ConfigMap follows that established layout - // (Option C sibling inference) rather than the canonical GVR-tree path. - freshInferred := filepath.Join(root, "apps", "fresh.yaml") - _, freshErr := os.Stat(freshInferred) - assert.NoError(t, freshErr, "the created resource lands beside its siblings under apps/") + // Placement: the existing ConfigMaps under apps/ do not decide where a genuinely new + // one goes. With no declared policy and no kustomize root, the create lands at the + // canonical path — the same answer the live-event path gives, which is the point: a + // resync must not place a resource anywhere a steady-state create would not. + _, siblingErr := os.Stat(filepath.Join(root, "apps", "fresh.yaml")) + assert.True(t, os.IsNotExist(siblingErr), "apps/ is not inferred from the siblings") + _, freshErr := os.Stat(filepath.Join(root, "default", "configmaps", "fresh.yaml")) + assert.NoError(t, freshErr, "the created resource lands at the canonical path") } // A fail-safe placement refusal during resync is counted in PlacementSkipped, not diff --git a/internal/git/types.go b/internal/git/types.go index 0a11b42c..611a21ae 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -240,7 +240,8 @@ type ResolvedTargetMetadata struct { EncryptionConfig *ResolvedEncryptionConfig // Placement is the GitTarget's declared new-file placement policy, resolved // from spec.placement. Nil when the GitTarget declares none, in which case new - // resources are placed by sibling inference and then the canonical path. + // resources are placed beside the folder's one kustomize root, if it has exactly one, + // and otherwise at the canonical path. Placement *manifestanalyzer.PlacementPolicy // PruneMode is the GitTarget's EFFECTIVE spec.prune.mode — always a concrete value, // because it is resolved through EffectivePruneMode and an omitted policy is onEvent. diff --git a/internal/manifestanalyzer/placement.go b/internal/manifestanalyzer/placement.go index 509ccece..e7a4a28b 100644 --- a/internal/manifestanalyzer/placement.go +++ b/internal/manifestanalyzer/placement.go @@ -7,7 +7,6 @@ import ( "fmt" "path" "regexp" - "sort" "strings" "github.com/ConfigButler/gitops-reverser/internal/types" @@ -24,11 +23,11 @@ import ( // There is no sensitive/normal split here: sensitivity is a write-safety property // (encrypt the content, keep the path identity-complete, never append or // co-mingle) enforced after resolution — in finishPlacement (sensitive never -// appends), in the writer (encrypt by classification), and in cohortMembers -// (inference never crosses the encrypted boundary) — not a second map to configure. +// appends) and in the writer (encrypt by classification) — not a second map to +// configure. // // A nil *PlacementPolicy, or one with no matching ByType entry and no Default, -// falls through to sibling inference (Option C) and then the canonical fallback. +// falls through to the kustomize-root fallback and then the canonical path. type PlacementPolicy struct { ByType map[string]string Default string @@ -45,28 +44,41 @@ type PlacementRequest struct { // WriteScope is the write jail relative to the scanned (render) root, set only when // render-root scoping re-rooted the scan past spec.path into a base an overlay reads. // Placement is documented as relative to spec.path, so a resolved path that would land - // outside the jail (a declared/canonical path resolved against the render anchor, or an - // inference from a read-only base sibling) is rebased under WriteScope rather than escaping - // it. Empty for a self-contained subtree, where the scan root IS spec.path and every - // resolved path is already in scope. + // outside the jail (a declared or canonical path resolved against the render anchor) is + // rebased under WriteScope rather than escaping it. Empty for a self-contained subtree, + // where the scan root IS spec.path and every resolved path is already in scope. WriteScope string } -// PlacementSource names which mechanism produced a PlacementResult's Path, for -// logging and the scan/dry-run "why here" trace (P8 in the design doc). +// PlacementSource names which mechanism produced a PlacementResult's Path. It is +// the "why here" answer for one new document, and it is reported three ways: the +// write path's log line, the placements_total metric's `source` label, and the +// scan/dry-run trace. The values are a public observability contract — they are +// metric label values — so they are lower_snake_case and are not renamed lightly. +// +// There are exactly three, and the list is closed by construction: a declaration, +// one structural fact about the folder, and the built-in path. Nothing here reads +// the repository's *layout* to guess an intent — that was Option C's sibling-cohort +// ladder, and it is gone (see the deletion argument in +// docs/design/open-asks-priority.md). type PlacementSource string const ( // PlacementSourceDeclared is Option B: an explicit placement.byType/default // template matched. PlacementSourceDeclared PlacementSource = "declared" - // PlacementSourceInferred is Option C: no declared template matched, but an - // existing sibling cohort determined the destination. - PlacementSourceInferred PlacementSource = "inferred" + // PlacementSourceKustomizeRoot is the structural fallback: no declared template + // matched, and the whole writable subtree is governed by exactly one supported + // kustomization, so the new document goes beside it and into its resources: list + // (see resolveKustomizeRoot). It is a fact about reachability, not a guess about + // convention: a file that root cannot reach would never render at all. + PlacementSourceKustomizeRoot PlacementSource = "kustomize_root" // PlacementSourceCanonical is the built-in, versionless // {namespaceOrCluster}/{group}/{resource}/{name}.yaml fallback: no declared - // template and no sibling to follow (e.g. an empty repository, or the - // type/namespace is new). + // template, and no single kustomize root to hang the file off. For a repository + // with a hand-authored layout this is the signal that a placement.byType or + // placement.default line is missing — which is why it is counted per + // (GitTarget, type) rather than only logged. See recordPlacement in internal/git. PlacementSourceCanonical PlacementSource = "canonical" ) @@ -80,41 +92,92 @@ type PlacementResult struct { Append bool // Source names which mechanism produced Path. Source PlacementSource - // Cohort describes the sibling cohort and ladder step that produced Path; - // empty unless Source is PlacementSourceInferred. - Cohort string // Kustomization is set when Path's directory carries a supported // kustomization.yaml whose resources: list does not already name Path — the // writer must add it as part of the same commit so kustomize picks the file // up ("add to the right kustomize file"). Kustomization *KustomizationInfo // NamespaceInherited is true when Path's destination infers its namespace - // from build context (a kustomization.yaml's namespace: transformer) rather - // than from metadata.namespace in the file — mirroring - // DocumentModel.NamespaceInheritedFromContext for a document that does not - // exist yet. The writer must keep metadata.namespace out of the written + // from build context (a kustomization.yaml's namespace: transformer set to this + // resource's own namespace) rather than from metadata.namespace in the file — + // mirroring DocumentModel.NamespaceInheritedFromContext for a document that does + // not exist yet. The writer must keep metadata.namespace out of the written // bytes, exactly as it already does for an in-place edit of an existing - // document in the same context (see design doc: "the new file inherits its - // sibling's NamespaceSource"). + // document in the same context. NamespaceInherited bool } +// PlacementRefusalReason names WHY a placement was refused, from a closed set. It is a +// metric label value (placement_refusals_total{reason}) as well as a log field, so the +// strings are lower_snake_case and are part of the observability contract. +// +// A refusal is a resource the operator did NOT mirror. It has to be countable per +// (GitTarget, type): before this it left a log line and, on the resync path, a single +// integer in a summary — neither of which a dashboard or an alert can reach, so a +// misconfigured template that silently skipped one Secret on every reconcile was +// invisible unless somebody read the logs. +type PlacementRefusalReason string + +const ( + // PlacementRefusedInvalidPath is a resolved path that failed the write-jail gate: + // empty, absolute, unclean, escaping via "..", or not a YAML file name. In practice + // a declared template whose literal text is wrong. + PlacementRefusedInvalidPath PlacementRefusalReason = "invalid_path" + // PlacementRefusedSensitiveAppend is a sensitive resource whose resolved path already + // holds a document. Sensitive documents are never appended, so the write is skipped + // rather than co-mingled — usually a declared template that is not identity-complete. + PlacementRefusedSensitiveAppend PlacementRefusalReason = "sensitive_append" + // PlacementRefusedPlaintextOntoEncrypted is a plaintext resource routed onto a file + // that already holds an encrypted document: appending would produce a + // partially-encrypted file and overwriting would destroy the encrypted document. + PlacementRefusedPlaintextOntoEncrypted PlacementRefusalReason = "plaintext_onto_encrypted" + // PlacementRefusedMixedSensitivityNewFile is two resources of different sensitivity + // resolving to the SAME brand-new path within one batch. LocateNew cannot see this + // (it resolves every resource against the pre-batch snapshot), so the writer refuses + // it; the value is defined here to keep one closed label domain for every refusal. + PlacementRefusedMixedSensitivityNewFile PlacementRefusalReason = "mixed_sensitivity_new_file" + // PlacementRefusedMultiDocumentTarget is a resolved path that holds a multi-document + // file the writer cannot append to (one of its documents is not cleanly editable), so + // the write would have to overwrite it and drop the siblings. Raised by the writer. + PlacementRefusedMultiDocumentTarget PlacementRefusalReason = "multi_document_target" +) + +// PlacementRefusedError is a placement that resolved but cannot be honoured safely. The +// caller must skip creating that resource and surface the refusal rather than writing. +// It carries a bounded Reason so the write path can count refusals by cause without +// matching on message text. +type PlacementRefusedError struct { + Reason PlacementRefusalReason + Resource string + Path string + detail string + cause error +} + +func (e *PlacementRefusedError) Error() string { return e.detail } + +// Unwrap exposes the underlying validation error for an invalid-path refusal, so +// errors.Is/As still reach it. +func (e *PlacementRefusedError) Unwrap() error { return e.cause } + // LocateNew resolves the placement of a resource with no existing document, per -// docs/spec/gittarget-new-file-placement-rules.md: a declared -// template (Option B) wins when present; otherwise an existing sibling cohort -// decides (Option C, steps 1/2 — same type+namespace, then same type+any -// namespace); otherwise the canonical path. +// docs/spec/gittarget-new-file-placement-rules.md: a declared template (Option B) +// wins when present; otherwise the folder's one supported kustomize root, if it has +// exactly one; otherwise the canonical path. // -// store MUST be the pre-plan snapshot for the whole batch and must never be mutated -// mid-batch, so a batch of several new creates resolves order-independently -// regardless of event order — a new resource never becomes another new resource's -// sibling within the same commit (P2 of the design doc). +// There is no step that reads the layout of the *other* documents of this type. +// Sibling-cohort inference (Option C) was removed: it let a human's edit to the +// repository change where the operator writes, with no Kubernetes object changing +// and nothing in status recording the move, and its central namespace-agnosticism +// guard had already failed once by cascading. The argument, and what replaced it +// (a declared byType line, plus the placements_total metric that says which +// (GitTarget, type) needs one), is in docs/design/open-asks-priority.md. // -// Step 3 (same namespace, any type) is deliberately not implemented: the design -// doc's own P5 discussion flags it as the highest-risk rung (an unbounded -// namespace-wide bundle that swallows every new type sharing a namespace), and -// steps 1/2/4 already cover the launch use cases (per-type bundles, per-type files, -// canonical). A namespace-bundle layout remains reachable via Option B. +// store is still the pre-plan snapshot for the whole batch and must never be mutated +// mid-batch: the remaining store reads — does the resolved path already hold an +// append-safe file, does its directory carry a kustomization — must answer the same +// way for every resource in one batch, so a batch of several new creates resolves +// order-independently regardless of event order (P2 of the design doc). // // An error is returned only when the resolved placement cannot be honoured safely // — currently, a sensitive resource whose resolved path already exists (sensitive @@ -125,42 +188,30 @@ func LocateNew(store *ManifestStore, policy *PlacementPolicy, req PlacementReque vars := placementVars(req) if path, ok, err := resolveDeclared(policy, req, vars); err == nil && ok { - return finishPlacement(store, req, path, PlacementSourceDeclared, "", false) - } - - if path, cohort, nsInherited, ok := resolveInferred(store, req); ok { - return finishPlacement(store, req, path, PlacementSourceInferred, cohort, nsInherited) + return finishPlacement(store, req, path, PlacementSourceDeclared) } - if path, ok, nsInherited := resolveKustomizeRoot(store, req); ok { - return finishPlacement( - store, req, path, PlacementSourceInferred, "the GitTarget's one kustomization root", nsInherited, - ) + if path, ok := resolveKustomizeRoot(store, req); ok { + return finishPlacement(store, req, path, PlacementSourceKustomizeRoot) } - return finishPlacement(store, req, canonicalPath(req), PlacementSourceCanonical, "", false) + return finishPlacement(store, req, canonicalPath(req), PlacementSourceCanonical) } -// resolveKustomizeRoot is a narrow, placement-specific fallback for when no sibling cohort -// exists (steps 1/2 both miss) — typically a resource whose type has never before -// appeared in this GitTarget. The canonical path (step 4) is a -// {group}/{version}/{resource}/{namespace}/{name}.yaml tree a kustomization's -// resources: graph can never reach, so a brand-new type in an otherwise -// kustomize-managed folder would silently land outside the folder's own -// convention — precisely the problem new-file placement exists to fix. When the whole scanned -// subtree is governed by exactly one supported kustomization (today's -// single-context baseline), the new resource belongs beside that kustomization's -// other files instead. +// resolveKustomizeRoot is the one non-declared, non-canonical placement, and it is a +// structural fact rather than a reading of the repository's conventions. The canonical +// path is a {namespaceOrCluster}/{group}/{resource}/{name}.yaml tree a kustomization's +// resources: graph can never reach, so a new document in an otherwise kustomize-managed +// folder would land outside every render — not merely oddly placed, but never applied. +// When the whole writable subtree is governed by exactly one supported kustomization +// (today's single-context baseline), the new resource belongs beside that +// kustomization's other files, and finishPlacement adds the resources: entry. // -// This is intentionally narrower than the design doc's shelved step 3 (same -// namespace, any type): it never appends into an existing bundle file, and it only -// ever fires when there is exactly one supported kustomization for the whole -// GitTarget to be about — the destination follows from there being one root, not -// from picking the largest matching cohort — so it cannot become the "sink that -// swallows every new type" risk (P5) the doc's own step 3 raised. More than one -// supported kustomization under the scanned root is ambiguous and declines rather -// than guessing. -func resolveKustomizeRoot(store *ManifestStore, req PlacementRequest) (string, bool, bool) { +// The destination follows from there being ONE root, not from picking a cohort: more +// than one supported kustomization under the scanned root is ambiguous and declines +// rather than guessing. That is why this survived the Option C deletion — deleting it +// would reintroduce the unreachable-file bug it was added to fix. +func resolveKustomizeRoot(store *ManifestStore, req PlacementRequest) (string, bool) { var only *KustomizationInfo for _, k := range store.Kustomizations { if k.Unsupported { @@ -174,18 +225,18 @@ func resolveKustomizeRoot(store *ManifestStore, req PlacementRequest) (string, b continue } if only != nil { - return "", false, false + return "", false } only = k } if only == nil { - return "", false, false + return "", false } name := req.Identifier.Name + ".yaml" if req.Sensitive { name = req.Identifier.Name + ".sops.yaml" } - return cleanJoin(slashDir(only.Path), name), true, only.Namespace != "" + return cleanJoin(slashDir(only.Path), name), true } // finishPlacement fills in the parts of a PlacementResult that depend only on the @@ -207,26 +258,30 @@ func finishPlacement( req PlacementRequest, resolvedPath string, source PlacementSource, - cohort string, - namespaceInherited bool, ) (PlacementResult, error) { // Render-root scoping re-roots the scan at the common ancestor of spec.path and the bases // it reads, so a resolved path is anchored there, not at spec.path. Placement is documented // as relative to spec.path, so rebase a path that would land outside the write jail back // under it before it is validated, checked for append, or matched to a kustomization. resolvedPath = rebaseIntoWriteScope(req.WriteScope, resolvedPath) - // This is the one gate every resolution path — declared, inferred, the - // kustomize-root fallback, and canonical alike — funnels through before a + // This is the one gate every resolution path — declared, the kustomize-root + // fallback, and canonical alike — funnels through before a // byte is ever written, so a rendered path can never escape the GitTarget's // spec.path regardless of which mechanism produced it. See "Path validation" // in the design doc: non-empty, a clean relative path, no "..", and a YAML // suffix. if err := ValidateResolvedPlacementPath(resolvedPath); err != nil { - return PlacementResult{}, fmt.Errorf( - "placement for resource %s resolved to an invalid path: %w", req.Identifier.String(), err, - ) + return PlacementResult{}, &PlacementRefusedError{ + Reason: PlacementRefusedInvalidPath, + Resource: req.Identifier.String(), + Path: resolvedPath, + detail: fmt.Sprintf( + "placement for resource %s resolved to an invalid path: %v", req.Identifier.String(), err, + ), + cause: err, + } } - res := PlacementResult{Path: resolvedPath, Source: source, Cohort: cohort, NamespaceInherited: namespaceInherited} + res := PlacementResult{Path: resolvedPath, Source: source} // A resolved path that already holds a file is only a safe append target when // every document already in it is cleanly editable. A file that tolerates a // non-editable construct (an anchor, alias, or other disallowed pattern) may @@ -241,11 +296,16 @@ func finishPlacement( res.Append = true } if req.Sensitive && res.Append { - return PlacementResult{}, fmt.Errorf( - "placement for sensitive resource %s resolved to %q, which already holds a document; "+ - "sensitive resources are never appended to an existing file", - req.Identifier.String(), resolvedPath, - ) + return PlacementResult{}, &PlacementRefusedError{ + Reason: PlacementRefusedSensitiveAppend, + Resource: req.Identifier.String(), + Path: resolvedPath, + detail: fmt.Sprintf( + "placement for sensitive resource %s resolved to %q, which already holds a document; "+ + "sensitive resources are never appended to an existing file", + req.Identifier.String(), resolvedPath, + ), + } } // A plaintext resource must never join a file that already holds an encrypted // document: appending would sit its cleartext beside SOPS-encrypted data (a @@ -255,19 +315,54 @@ func finishPlacement( // runtime guard (not a separate sensitive placement block) is what keeps the two // classes from co-mingling for every sensitive type, core or operator-configured. if res.Append && !req.Sensitive && fileHoldsEncryptedDocument(fm) { - return PlacementResult{}, fmt.Errorf( - "placement for resource %s resolved to %q, which already holds an encrypted document; "+ - "a plaintext resource is never appended to an encrypted file", - req.Identifier.String(), resolvedPath, - ) + return PlacementResult{}, &PlacementRefusedError{ + Reason: PlacementRefusedPlaintextOntoEncrypted, + Resource: req.Identifier.String(), + Path: resolvedPath, + detail: fmt.Sprintf( + "placement for resource %s resolved to %q, which already holds an encrypted document; "+ + "a plaintext resource is never appended to an encrypted file", + req.Identifier.String(), resolvedPath, + ), + } } - if k := governingKustomization(store, req.WriteScope, resolvedPath); k != nil && !k.Unsupported && - !kustomizationListsResource(k, resolvedPath) { - res.Kustomization = k + if k := governingKustomization(store, req.WriteScope, resolvedPath); k != nil && !k.Unsupported { + if !kustomizationListsResource(k, resolvedPath) { + res.Kustomization = k + } + res.NamespaceInherited = namespaceIsInheritedFromContext(k, req) } return res, nil } +// namespaceIsInheritedFromContext reports whether a new document at a path this +// kustomization governs must OMIT metadata.namespace, because the build context already +// supplies it. Two conditions, and the second one is the safety half: +// +// - the kustomization sets a namespace: transformer at all, and +// - it sets it to THIS resource's own namespace. +// +// The second condition is what keeps the write honest. Omitting metadata.namespace hands +// the namespace to kustomize, so if the transformer named a DIFFERENT namespace the +// document would render as another object entirely — the mirror would claim to hold a +// resource it does not. Writing the namespace explicitly in that case is not a +// convention break; it is the only truthful bytes available, and the render oracle then +// reports the folder as unable to express this object rather than silently mis-rendering +// it. (kustomize's namespace transformer overrides an explicit metadata.namespace, so the +// explicit line is redundant when the two agree and load-bearing when they do not.) +// +// A cluster-scoped resource has no namespace, so it never inherits one. +// +// This applies to every resolved path, not just the kustomize-root fallback: a DECLARED +// template pointing into a governed directory has exactly the same obligation, and before +// this it silently wrote a namespace: line the folder's own documents omit. +func namespaceIsInheritedFromContext(k *KustomizationInfo, req PlacementRequest) bool { + if k.Namespace == "" || req.Identifier.Namespace == "" { + return false + } + return k.Namespace == req.Identifier.Namespace +} + // governingKustomization returns the kustomization whose resources: list a new file at // resolvedPath must join to render: the one in its own directory, or — under render-root // scoping, when the file lands in a subdirectory of the overlay that has no kustomization of @@ -561,125 +656,16 @@ func IdentityCompletePlacementTemplate(tmpl string, narrowedToOneType bool) bool strings.Contains(tmpl, "{resource}") } -// --- Option C: sibling inference ------------------------------------------------- - -// resolveInferred implements Option C steps 1 and 2. See LocateNew's doc comment for -// why step 3 is not implemented. -func resolveInferred(store *ManifestStore, req PlacementRequest) (string, string, bool, bool) { - id := req.Identifier - // Render-root scoping puts the read-only base documents in the store too; a new write must - // never be inferred to sit beside one — that path is outside the jail, and rebasing it would - // duplicate a base object under the overlay. Restrict every cohort to writable siblings. - // The identity when there is no jail (WriteScope == ""), so self-contained placement is - // unchanged. - writable := writableCohort(store, req.WriteScope) - - if members := writable(cohortMembers( - store, - id.Group, - id.Version, - id.Resource, - id.Namespace, - true, - req.Sensitive, - )); len( - members, - ) > 0 { - if path, cohort, nsInherited, ok := cohortDestination( - store, - members, - req, - "same type and namespace", - false, - ); ok { - return path, cohort, nsInherited, true - } - } - // Step 2 matches across namespaces, so — unlike step 1, where every candidate - // already shares the new resource's own namespace — a candidate here must prove - // it is namespace-agnostic before it can be trusted for a namespace it has never - // seen (P4 of the design doc): a per-namespace-segmented layout (a dedicated - // bundle or directory per namespace) must NOT be extended by guessing one of the - // existing namespaces' files/directories for a brand-new namespace. cohortDestination - // disqualifies any candidate that has not demonstrated it already spans more than - // one namespace (a bundle) or lives in a single shared directory regardless of - // namespace (singleton style); an unseen namespace then correctly falls through to - // the canonical path, which builds the right namespace segment directly. - if members := writable( - cohortMembers(store, id.Group, id.Version, id.Resource, "", false, req.Sensitive), - ); len(members) > 0 { - if path, cohort, nsInherited, ok := cohortDestination( - store, - members, - req, - "same type, any namespace", - true, - ); ok { - return path, cohort, nsInherited, true - } - } - return "", "", false, false -} - -// writableCohort returns a filter that keeps only documents inside the write jail, so sibling -// inference never places a new write beside a read-only base document render-root scoping -// pulled into the store. With no jail (writeScope == "") it is the identity, so self-contained -// placement is byte-for-byte unchanged. DocumentLocations is resolved once, only when a jail -// is in force. -func writableCohort(store *ManifestStore, writeScope string) func([]*DocumentModel) []*DocumentModel { - if writeScope == "" { - return func(dms []*DocumentModel) []*DocumentModel { return dms } - } - locs := store.DocumentLocations() - return func(dms []*DocumentModel) []*DocumentModel { - out := make([]*DocumentModel, 0, len(dms)) - for _, dm := range dms { - if pathWithin(locs[dm].FilePath, writeScope) { - out = append(out, dm) - } - } - return out - } -} - -// cohortMembers collects every existing document of the given type (optionally -// pinned to namespace) whose sensitivity matches the resource being placed. A -// document's sensitivity is read off the analyzer's own encrypted-document -// classification (CauseEncrypted) rather than a separately threaded policy, so a -// sensitive resource can never infer from a plaintext sibling or vice versa (the -// design doc's "sensitive stays hard-split — with no config"). -func cohortMembers( - store *ManifestStore, - group, version, resource, namespace string, - matchNamespace, sensitive bool, -) []*DocumentModel { - var out []*DocumentModel - for rid, dm := range store.ByResourceIdentity { - if rid.Group != group || rid.Version != version || rid.Resource != resource { - continue - } - if matchNamespace && rid.Namespace != namespace { - continue - } - if isSensitiveDocument(dm) != sensitive { - continue - } - out = append(out, dm) - } - return out -} - -func isSensitiveDocument(dm *DocumentModel) bool { - return dm.Cause.Kind == CauseEncrypted -} +// --- Write-safety helpers for an already-occupied destination ------------------ // fileIsAppendSafe reports whether every document already in fm is cleanly // editable or an ordinary encrypted document — never a document tolerated despite // an unsupported construct (CauseNonEditable: an anchor, alias, or other disallowed // pattern), which does not claim its identity and so cannot be vouched for. Such a -// file is excluded from both bundle and singleton-style candidacy (cohortDestination) -// and from the append decision (finishPlacement): a genuinely new resource must -// never be joined to a file the writer cannot fully account for. +// file is excluded from the append decision (finishPlacement): a genuinely new +// resource must never be joined to a file the writer cannot fully account for. The +// caller falls back to writeWholeFile, whose own multi-document guard refuses rather +// than overwrites. func fileIsAppendSafe(fm *FileModel) bool { if fm == nil { return false @@ -708,169 +694,3 @@ func fileHoldsEncryptedDocument(fm *FileModel) bool { } return false } - -// cohortDestination decides, for one matched cohort, whether the repository's -// established pattern is "one resource per file" or "resources of this cohort share -// a file" (a bundle), and resolves the concrete destination: -// -// - every file holding >1 document is a candidate bundle, keyed by path, weighted -// by how many cohort members it holds; -// - every file holding exactly one document is "singleton style," aggregated into -// one virtual candidate regardless of how many separate files/directories it -// spans, weighted by its total member count; -// - the candidate with the most members wins; ties (including "no bundle beats -// the singleton style") favour singleton style, the more conservative choice, -// since it never grows an existing bundle the repository's siblings do not -// clearly favour. Among multiple bundle files tied for the lead, the -// lexicographically smallest file path wins; the singleton style's directory is -// the lexicographically smallest directory among its members. -// -// This is deterministic and independent of map/walk iteration order (P1 of the -// design doc): the result depends only on the (path -> member count) shape of the -// pre-plan snapshot, never on the order LocateNew is called for other resources in -// the same batch. -// -// namespaceAgnostic is true only for step 2 (any namespace). It disqualifies a -// candidate that has not demonstrated it is independent of namespace: a bundle file -// must already hold members from more than one distinct namespace, and singleton -// style must have every member in exactly one directory (P4 — see resolveInferred). -// A step-1 candidate (namespaceAgnostic false) is never disqualified this way, -// because every member there already shares the new resource's own namespace. -func cohortDestination( - store *ManifestStore, - members []*DocumentModel, - req PlacementRequest, - step string, - namespaceAgnostic bool, -) (string, string, bool, bool) { - docLoc := store.DocumentLocations() - perFile := map[string][]*DocumentModel{} - for _, m := range members { - if p := docLoc[m].FilePath; p != "" { - perFile[p] = append(perFile[p], m) - } - } - if len(perFile) == 0 { - return "", "", false, false - } - - singletonDirs, bestPath, bestCount, dirReps, bundleReps := classifyCohortLocations( - store, - perFile, - namespaceAgnostic, - ) - if bestPath == "" && len(singletonDirs) == 0 { - return "", "", false, false - } - - cohort := fmt.Sprintf("%d sibling(s) via %s", len(members), step) - if bestCount > len(singletonDirs) { - nsInherited := bundleReps[bestPath] != nil && bundleReps[bestPath].NamespaceInheritedFromContext() - return bestPath, cohort, nsInherited, true - } - - sort.Strings(singletonDirs) - winDir := singletonDirs[0] - name := req.Identifier.Name + ".yaml" - if req.Sensitive { - name = req.Identifier.Name + ".sops.yaml" - } - nsInherited := dirReps[winDir] != nil && dirReps[winDir].NamespaceInheritedFromContext() - return cleanJoin(winDir, name), cohort, nsInherited, true -} - -// classifyCohortLocations partitions a cohort's members by where they live: -// every file holding more than one document is a bundle candidate (keyed by path, -// weighted by member count); every file holding exactly one document contributes to -// the singleton-style candidate. A tainted file (fileIsAppendSafe false) is -// excluded from both. namespaceAgnostic applies the P4 safety rule (see -// resolveInferred): a candidate must PROVE it is namespace-agnostic by already -// spanning more than one namespace — a bundle through its own documents, singleton -// style through the documents of its one shared directory — or it is dropped. -// It returns the eligible singleton directories, the winning bundle -// (path/count), and one representative document per singleton directory / bundle -// path — used to decide whether the destination's namespace is inherited from -// build context (see PlacementResult.NamespaceInherited) — determined -// independently of map iteration order by scanning candidate paths in sorted order. -func classifyCohortLocations( - store *ManifestStore, - perFile map[string][]*DocumentModel, - namespaceAgnostic bool, -) ([]string, string, int, map[string]*DocumentModel, map[string]*DocumentModel) { - var singletonDirs []string - var singletonDocs []*DocumentModel - dirReps := map[string]*DocumentModel{} - bundleReps := map[string]*DocumentModel{} - bundleCounts := map[string]int{} - for p, ms := range perFile { - fm := store.FilesByPath[p] - if !fileIsAppendSafe(fm) { - continue // a tainted file is never a placement destination - } - if len(fm.Documents) > 1 { - if namespaceAgnostic && !spansMultipleNamespaces(ms) { - continue // unproven: looks like a per-namespace-segmented bundle (P4) - } - bundleCounts[p] = len(ms) - bundleReps[p] = ms[0] - continue - } - dir := slashDir(p) - singletonDirs = append(singletonDirs, dir) - singletonDocs = append(singletonDocs, ms...) - if _, seen := dirReps[dir]; !seen { - dirReps[dir] = ms[0] - } - } - // P4 for singleton style, symmetric with the bundle branch above: reuse needs POSITIVE - // proof of namespace-agnosticism, not merely the absence of contrary evidence. One - // directory holding one namespace's documents is exactly as consistent with a - // per-namespace-segmented layout whose second namespace has simply never been written as - // it is with a shared directory — and guessing wrong files another namespace's object - // under the first namespace's folder. Declining costs nothing: the canonical path builds - // the correct namespace segment directly. - if namespaceAgnostic && (!allSameDir(singletonDirs) || !spansMultipleNamespaces(singletonDocs)) { - singletonDirs = nil // unproven: directories look namespace-segmented (P4) - } - - bundlePaths := make([]string, 0, len(bundleCounts)) - for p := range bundleCounts { - bundlePaths = append(bundlePaths, p) - } - sort.Strings(bundlePaths) - bestPath, bestCount := "", 0 - for _, p := range bundlePaths { - if bundleCounts[p] > bestCount { - bestCount, bestPath = bundleCounts[p], p - } - } - return singletonDirs, bestPath, bestCount, dirReps, bundleReps -} - -// spansMultipleNamespaces reports whether ms (all documents sharing one file) -// carry more than one distinct namespace, proving the file is namespace-agnostic -// rather than one namespace's dedicated bundle. -func spansMultipleNamespaces(ms []*DocumentModel) bool { - seen := map[string]struct{}{} - for _, m := range ms { - if m.ResourceIdentity == nil { - continue - } - seen[m.ResourceIdentity.Namespace] = struct{}{} - if len(seen) > 1 { - return true - } - } - return false -} - -// allSameDir reports whether every directory in dirs is identical (trivially true -// for zero or one element). -func allSameDir(dirs []string) bool { - for i := 1; i < len(dirs); i++ { - if dirs[i] != dirs[0] { - return false - } - } - return true -} diff --git a/internal/manifestanalyzer/placement_test.go b/internal/manifestanalyzer/placement_test.go index 445c4664..30bca795 100644 --- a/internal/manifestanalyzer/placement_test.go +++ b/internal/manifestanalyzer/placement_test.go @@ -132,9 +132,130 @@ func TestLocateNew_WriteScope_RebasesDeclared(t *testing.T) { } } -func TestLocateNew_BundleCohort_Appends(t *testing.T) { +// Every layout sibling inference used to read, in one table, all resolving to the +// canonical path. This is the deletion's contract, stated as the behaviour rather than +// as an absence: the destination of a new document depends on the GitTarget's +// declaration and on whether the folder has one kustomize root — never on where the +// repository happens to keep the OTHER documents of the same type. +// +// The last two rows are the ones the ladder used to accept, and they are the cost the +// deletion is paying deliberately (docs/design/open-asks-priority.md, "The cost, stated +// plainly"): a bundle or a directory that had already proven itself namespace-agnostic +// was extended. It is not extended now. A `placement.byType` line is how a repository +// asks for either, and it is now the only way. +func TestLocateNew_LayoutsThatUsedToBeInferred_AllResolveCanonical(t *testing.T) { + cases := []struct { + name string + files map[string]string + namespace string + }{ + { + name: "a bundle of the same type in the same namespace", + files: map[string]string{"all.yaml": configMapYAML("a", "app") + "---\n" + configMapYAML("b", "app")}, + namespace: "app", + }, + { + name: "one document per file, same type and namespace", + files: map[string]string{"overlays/test/configmap-a.yaml": configMapYAML("a", "app")}, + namespace: "app", + }, + { + name: "a per-namespace bundle, for an unseen namespace", + files: map[string]string{ + "ns1/configmaps.yaml": configMapYAML("a", "ns1") + "---\n" + configMapYAML("b", "ns1"), + }, + namespace: "ns2", + }, + { + name: "a directory per namespace, for an unseen namespace", + files: map[string]string{ + "ns1/configmap-a.yaml": configMapYAML("a", "ns1"), + "ns2/configmap-b.yaml": configMapYAML("b", "ns2"), + }, + namespace: "ns3", + }, + { + name: "one directory holding one namespace, for an unseen namespace", + files: map[string]string{"ns1/configmap-a.yaml": configMapYAML("a", "ns1")}, + namespace: "ns2", + }, + { + name: "a bundle that already spans two namespaces", + files: map[string]string{"all.yaml": configMapYAML("a", "ns1") + "---\n" + configMapYAML("b", "ns2")}, + namespace: "ns3", + }, + { + name: "a shared directory that already spans two namespaces", + files: map[string]string{ + "shared/configmap-a.yaml": configMapYAML("a", "ns1"), + "shared/configmap-b.yaml": configMapYAML("b", "ns2"), + }, + namespace: "ns3", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fsys := fstest.MapFS{} + for path, body := range tc.files { + fsys[path] = &fstest.MapFile{Data: []byte(body)} + } + store := placementStore(t, fsys) + req := newConfigMapRequest("cache", tc.namespace) + + res, err := LocateNew(store, nil, req) + if err != nil { + t.Fatalf("LocateNew: %v", err) + } + if res.Path != req.Identifier.ToGitPath() || res.Source != PlacementSourceCanonical { + t.Fatalf("got %+v, want the canonical path %q", res, req.Identifier.ToGitPath()) + } + if res.Append { + t.Fatalf("got %+v, want a file of its own: no existing document's file is ever joined", res) + } + }) + } +} + +// The production shape of the cascade the deletion retires, kept as its own named test +// because it is the failure the argument rests on. Objects that exist under the SAME NAME +// in every namespace (kube-root-ca.crt is in all of them) made the inferred path collide +// exactly with the first namespace's file, so the second namespace's object was appended +// as an extra document. That file then genuinely spanned two namespaces, so every later +// object of the type legitimately preferred the bundle and the whole type collapsed into +// one file — one wrong inference cascading into total collapse. With no ladder there is +// no first wrong step to cascade from. +func TestLocateNew_SameNameInANewNamespace_NeverLandsOnTheFirstNamespacesFile(t *testing.T) { fsys := fstest.MapFS{ - "all.yaml": {Data: []byte(configMapYAML("a", "app") + "---\n" + configMapYAML("b", "app"))}, + "ns1/configmaps/kube-root-ca.crt.yaml": {Data: []byte(configMapYAML("kube-root-ca.crt", "ns1"))}, + } + store := placementStore(t, fsys) + req := newConfigMapRequest("kube-root-ca.crt", "ns2") + + res, err := LocateNew(store, nil, req) + if err != nil { + t.Fatalf("LocateNew: %v", err) + } + if res.Append || res.Path == "ns1/configmaps/kube-root-ca.crt.yaml" { + t.Fatalf("ns2's object was filed onto ns1's own file: %+v", res) + } + if res.Path != req.Identifier.ToGitPath() || res.Source != PlacementSourceCanonical { + t.Fatalf("got %+v, want canonical fallback carrying ns2's own namespace segment", res) + } +} + +// A folder whose one kustomization sets a namespace: transformer for this resource's own +// namespace means the new document must omit metadata.namespace — the build context +// supplies it, and repeating it would break the convention every document in that +// context follows. The destination here comes from there being ONE kustomize root, not +// from the sibling: what the sibling's own bytes look like no longer enters into it. +func TestLocateNew_KustomizeContextNamespace_NewFileOmitsNamespace(t *testing.T) { + fsys := fstest.MapFS{ + "overlays/test/kustomization.yaml": { + Data: []byte("namespace: app\nresources:\n - configmap-a.yaml\n"), + }, + "overlays/test/configmap-a.yaml": { + Data: []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n"), + }, } store := placementStore(t, fsys) req := newConfigMapRequest("cache", "app") @@ -143,12 +264,14 @@ func TestLocateNew_BundleCohort_Appends(t *testing.T) { if err != nil { t.Fatalf("LocateNew: %v", err) } - if res.Path != "all.yaml" || !res.Append || res.Source != PlacementSourceInferred { - t.Fatalf("got %+v, want append to all.yaml via inference", res) + if !res.NamespaceInherited { + t.Fatalf("got %+v, want NamespaceInherited since the sibling omits metadata.namespace", res) } } -func TestLocateNew_SingletonCohort_NewFileBesideSiblings(t *testing.T) { +// No kustomize context means no inherited namespace: the document carries its own +// metadata.namespace, because nothing else will supply it. +func TestLocateNew_NoKustomizeContext_NewFileKeepsNamespace(t *testing.T) { fsys := fstest.MapFS{ "overlays/test/configmap-a.yaml": {Data: []byte(configMapYAML("a", "app"))}, } @@ -159,24 +282,21 @@ func TestLocateNew_SingletonCohort_NewFileBesideSiblings(t *testing.T) { if err != nil { t.Fatalf("LocateNew: %v", err) } - want := "overlays/test/cache.yaml" - if res.Path != want || res.Append || res.Source != PlacementSourceInferred { - t.Fatalf("got %+v, want a new file %q beside the sibling", res, want) + if res.NamespaceInherited { + t.Fatalf("got %+v, want NamespaceInherited false: no build context supplies a namespace", res) } } -// A sibling whose namespace is inherited from a kustomization's namespace: -// transformer (no metadata.namespace in its own bytes) means a new document -// placed beside it must also omit metadata.namespace — otherwise the write -// would silently break the convention every document in that context follows -// (this is what let an incidental resource sharing the namespace, e.g. a -// cluster-injected ConfigMap, write a namespace: line into a hand-curated -// bundle file in production; see the design doc's Option C test plan, "the new -// file inherits its sibling's NamespaceSource"). -func TestLocateNew_SiblingNamespaceInheritedFromKustomize_NewFileOmitsNamespace(t *testing.T) { +// The safety half of the inheritance rule, and the case the old kustomize-root path got +// wrong: a kustomization whose namespace: transformer names a DIFFERENT namespace than the +// resource's own must not make the write omit metadata.namespace. Omitting it would hand +// the namespace to kustomize, which renders the document as an object in the +// transformer's namespace — a different object than the one being mirrored. The explicit +// line stays, and the render oracle reports the folder as unable to express this object. +func TestLocateNew_KustomizeContextNamespaceMismatch_NewFileKeepsItsOwnNamespace(t *testing.T) { fsys := fstest.MapFS{ "overlays/test/kustomization.yaml": { - Data: []byte("namespace: app\nresources:\n - configmap-a.yaml\n"), + Data: []byte("namespace: other\nresources:\n - configmap-a.yaml\n"), }, "overlays/test/configmap-a.yaml": { Data: []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n"), @@ -189,32 +309,41 @@ func TestLocateNew_SiblingNamespaceInheritedFromKustomize_NewFileOmitsNamespace( if err != nil { t.Fatalf("LocateNew: %v", err) } - if !res.NamespaceInherited { - t.Fatalf("got %+v, want NamespaceInherited since the sibling omits metadata.namespace", res) + if res.NamespaceInherited { + t.Fatalf("got %+v, want the namespace written explicitly: the transformer names another namespace", res) } } -// A sibling with an explicit metadata.namespace (no kustomize context) means a -// new document beside it keeps writing its namespace explicitly too. -func TestLocateNew_SiblingNamespaceExplicit_NewFileKeepsNamespace(t *testing.T) { +// A DECLARED template pointing into a governed directory carries the same obligation as +// the kustomize-root fallback: the context supplies the namespace, so the file must not +// repeat it. Before the Option C deletion this only ever came out of a sibling's own +// bytes, so a declared placement silently wrote a namespace: line the folder omits. +func TestLocateNew_DeclaredIntoKustomizeContext_OmitsNamespace(t *testing.T) { fsys := fstest.MapFS{ - "overlays/test/configmap-a.yaml": {Data: []byte(configMapYAML("a", "app"))}, + "overlays/test/kustomization.yaml": { + Data: []byte("namespace: app\nresources:\n - configmap-a.yaml\n"), + }, + "overlays/test/configmap-a.yaml": { + Data: []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n"), + }, } store := placementStore(t, fsys) - req := newConfigMapRequest("cache", "app") + policy := &PlacementPolicy{Default: "overlays/test/{name}.yaml"} - res, err := LocateNew(store, nil, req) + res, err := LocateNew(store, policy, newConfigMapRequest("cache", "app")) if err != nil { t.Fatalf("LocateNew: %v", err) } - if res.NamespaceInherited { - t.Fatalf("got %+v, want NamespaceInherited false: the sibling writes its namespace explicitly", res) + if res.Source != PlacementSourceDeclared { + t.Fatalf("expected a declared placement, got %s", res.Source) + } + if !res.NamespaceInherited { + t.Fatalf("got %+v, want NamespaceInherited: the declared path lands in a namespaced context", res) } } -// resolveKustomizeRoot's fallback (no sibling of this type yet) must also flag -// NamespaceInherited when the one kustomization declares a namespace: -// transformer, for the same reason. +// resolveKustomizeRoot's fallback must flag NamespaceInherited when the one +// kustomization declares a namespace: transformer for this resource's namespace. func TestLocateNew_KustomizeRootWithNamespaceTransformer_NewFileOmitsNamespace(t *testing.T) { fsys := fstest.MapFS{ "overlays/test/kustomization.yaml": { @@ -256,7 +385,11 @@ func TestLocateNew_Sensitive_NeverJoinsPlaintextBundle(t *testing.T) { } } -func TestLocateNew_Sensitive_JoinsSensitiveSiblingDirectory(t *testing.T) { +// A sensitive resource gets no sibling reuse either: with the ladder gone, the existing +// .sops.yaml directory does not attract the new Secret, and the canonical SOPS path — which +// is identity-complete by construction — is what it gets. A repository that wants its +// secrets kept together says so with one placement.byType line. +func TestLocateNew_Sensitive_ExistingSopsDirectoryIsNotReused(t *testing.T) { fsys := fstest.MapFS{ "secrets/app/db.sops.yaml": {Data: []byte(secretYAML("db", "app"))}, } @@ -267,57 +400,16 @@ func TestLocateNew_Sensitive_JoinsSensitiveSiblingDirectory(t *testing.T) { if err != nil { t.Fatalf("LocateNew: %v", err) } - want := "secrets/app/api-token.sops.yaml" - if res.Path != want || res.Append || res.Source != PlacementSourceInferred { - t.Fatalf("got %+v, want a new single-doc SOPS file beside the sensitive sibling %q", res, want) - } -} - -func TestLocateNew_TieBreak_SingletonWinsWhenAheadOrTied(t *testing.T) { - cases := []struct { - name string - singletons int - bundleSize int - wantBundle bool - }{ - {"singleton strictly ahead", 3, 2, false}, - {"tie favours singleton", 2, 2, false}, - {"bundle strictly ahead", 2, 3, true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - fsys := fstest.MapFS{} - for i := range tc.singletons { - fsys[fmt.Sprintf("solo-%d.yaml", i)] = &fstest.MapFile{ - Data: []byte(configMapYAML(fmt.Sprintf("solo-%d", i), "app")), - } - } - var bundle strings.Builder - for i := range tc.bundleSize { - if i > 0 { - bundle.WriteString("---\n") - } - bundle.WriteString(configMapYAML(fmt.Sprintf("bundled-%d", i), "app")) - } - if tc.bundleSize > 0 { - fsys["bundle.yaml"] = &fstest.MapFile{Data: []byte(bundle.String())} - } - - store := placementStore(t, fsys) - res, err := LocateNew(store, nil, newConfigMapRequest("new", "app")) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - if got := res.Path == "bundle.yaml"; got != tc.wantBundle { - t.Fatalf("path = %q (append=%v), wantBundle=%v", res.Path, res.Append, tc.wantBundle) - } - }) + want := "app/secrets/api-token.sops.yaml" + if res.Path != want || res.Append || res.Source != PlacementSourceCanonical { + t.Fatalf("got %+v, want the canonical SOPS path %q", res, want) } } -func TestLocateNew_DeclaredOutranksInferred(t *testing.T) { +func TestLocateNew_DeclaredOutranksTheKustomizeRoot(t *testing.T) { fsys := fstest.MapFS{ - "all.yaml": {Data: []byte(configMapYAML("a", "app"))}, + "overlays/test/kustomization.yaml": {Data: []byte("resources:\n - configmap-a.yaml\n")}, + "overlays/test/configmap-a.yaml": {Data: []byte(configMapYAML("a", "app"))}, } store := placementStore(t, fsys) policy := &PlacementPolicy{ @@ -330,135 +422,7 @@ func TestLocateNew_DeclaredOutranksInferred(t *testing.T) { } want := "app/configmaps.yaml" if res.Path != want || res.Source != PlacementSourceDeclared { - t.Fatalf("got %+v, want the declared template %q to win over inference", res, want) - } -} - -func TestLocateNew_Step2_NewNamespaceUnderPerNamespaceBundle_FallsToCanonical(t *testing.T) { - fsys := fstest.MapFS{ - "ns1/configmaps.yaml": { - Data: []byte( - configMapYAML("a", "ns1") + "---\n" + configMapYAML("b", "ns1") + "---\n" + configMapYAML("c", "ns1"), - ), - }, - } - store := placementStore(t, fsys) - req := newConfigMapRequest("cache", "ns2") - - res, err := LocateNew(store, nil, req) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - // P4: a per-namespace-segmented bundle must not be guessed for an unseen - // namespace; the new namespace's ConfigMap must fall through to canonical, - // never land in ns1/configmaps.yaml. - if res.Path != req.Identifier.ToGitPath() || res.Source != PlacementSourceCanonical { - t.Fatalf("got %+v, want canonical fallback (never ns1's bundle)", res) - } -} - -func TestLocateNew_Step2_NamespaceAgnosticBundle_IsReused(t *testing.T) { - fsys := fstest.MapFS{ - "all.yaml": {Data: []byte(configMapYAML("a", "ns1") + "---\n" + configMapYAML("b", "ns2"))}, - } - store := placementStore(t, fsys) - req := newConfigMapRequest("cache", "ns3") - - res, err := LocateNew(store, nil, req) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - if res.Path != "all.yaml" || !res.Append || res.Source != PlacementSourceInferred { - t.Fatalf("got %+v, want the namespace-agnostic bundle reused for the new namespace", res) - } -} - -func TestLocateNew_Step2_NewNamespaceUnderPerNamespaceDirectories_FallsToCanonical(t *testing.T) { - // Two distinct singleton directories, one per namespace, is what proves a - // per-namespace-segmented convention (P4) — a single existing directory would - // be indistinguishable from coincidence, so this needs at least two. - fsys := fstest.MapFS{ - "ns1/configmap-a.yaml": {Data: []byte(configMapYAML("a", "ns1"))}, - "ns2/configmap-b.yaml": {Data: []byte(configMapYAML("b", "ns2"))}, - } - store := placementStore(t, fsys) - req := newConfigMapRequest("cache", "ns3") - - res, err := LocateNew(store, nil, req) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - if res.Path != req.Identifier.ToGitPath() || res.Source != PlacementSourceCanonical { - t.Fatalf("got %+v, want canonical fallback, never ns1/ or ns2/ for a resource in ns3", res) - } -} - -func TestLocateNew_Step2_NewNamespaceUnderTheOnlyExistingDirectory_FallsToCanonical(t *testing.T) { - // ONE directory holding ONE namespace is the case that used to slip through: "all the - // singleton directories agree" is trivially true of a single directory, so a - // per-namespace-segmented layout whose second namespace had simply never been written - // was indistinguishable from a shared one — and the new namespace's object was filed - // under the first namespace's folder. Absence of contrary evidence is not proof. - fsys := fstest.MapFS{ - "ns1/configmap-a.yaml": {Data: []byte(configMapYAML("a", "ns1"))}, - } - store := placementStore(t, fsys) - req := newConfigMapRequest("cache", "ns2") - - res, err := LocateNew(store, nil, req) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - if res.Path != req.Identifier.ToGitPath() || res.Source != PlacementSourceCanonical { - t.Fatalf("got %+v, want canonical fallback, never ns1/ for a resource in ns2", res) - } -} - -func TestLocateNew_Step2_SameNameInANewNamespace_IsNeverAppendedOntoTheFirstNamespacesFile(t *testing.T) { - // The production shape of the bug. Objects that exist under the SAME NAME in every - // namespace (kube-root-ca.crt is in all of them) made the inferred path collide exactly - // with the first namespace's file, so the second namespace's object was appended as an - // extra document. That file then genuinely spanned two namespaces, so every later object - // of the type legitimately preferred the bundle and the whole type collapsed into one - // file — one wrong inference cascading into total collapse. - fsys := fstest.MapFS{ - "ns1/configmaps/kube-root-ca.crt.yaml": {Data: []byte(configMapYAML("kube-root-ca.crt", "ns1"))}, - } - store := placementStore(t, fsys) - req := newConfigMapRequest("kube-root-ca.crt", "ns2") - - res, err := LocateNew(store, nil, req) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - if res.Append { - t.Fatalf("got %+v, want a distinct file; appending merges two namespaces' objects", res) - } - if res.Path == "ns1/configmaps/kube-root-ca.crt.yaml" { - t.Fatalf("ns2's object was filed onto ns1's own file: %+v", res) - } - if res.Path != req.Identifier.ToGitPath() || res.Source != PlacementSourceCanonical { - t.Fatalf("got %+v, want canonical fallback carrying ns2's own namespace segment", res) - } -} - -func TestLocateNew_Step2_ProvenNamespaceAgnosticDirectory_IsStillReused(t *testing.T) { - // The other side of the rule: one shared directory that ALREADY holds more than one - // namespace has proven itself namespace-agnostic, so a third namespace still joins it. - // Requiring proof must not degrade into refusing every singleton-style layout. - fsys := fstest.MapFS{ - "shared/configmap-a.yaml": {Data: []byte(configMapYAML("a", "ns1"))}, - "shared/configmap-b.yaml": {Data: []byte(configMapYAML("b", "ns2"))}, - } - store := placementStore(t, fsys) - req := newConfigMapRequest("cache", "ns3") - - res, err := LocateNew(store, nil, req) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - if res.Path != "shared/cache.yaml" || res.Source != PlacementSourceInferred { - t.Fatalf("got %+v, want the proven shared directory reused for the new namespace", res) + t.Fatalf("got %+v, want the declared template %q to win over the kustomize root", res, want) } } @@ -646,51 +610,30 @@ func TestPlacementVars_GroupedClusterScoped(t *testing.T) { } } -// A sensitive and a normal document of the SAME type (e.g. one ConfigMap -// encrypted as .sops.yaml, one plain) must not be conflated: cohortMembers must -// skip the mismatched-sensitivity sibling rather than only relying on the type -// filter (which cannot tell them apart, since sensitivity is an encryption fact, -// not a type fact). -func TestLocateNew_MixedSensitivityConfigMapsInSameNamespace_NeverConflated(t *testing.T) { - fsys := fstest.MapFS{ - "normal.yaml": {Data: []byte(configMapYAML("a", "app") + "---\n" + configMapYAML("b", "app"))}, - "secret.sops.yaml": { - Data: []byte( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: sensitive-cm\n namespace: app\nsops:\n version: \"3\"\n", - ), - }, - } - store := placementStore(t, fsys) - - res, err := LocateNew(store, nil, newConfigMapRequest("cache", "app")) - if err != nil { - t.Fatalf("LocateNew: %v", err) - } - want := "normal.yaml" - if res.Path != want || !res.Append { - t.Fatalf("got %+v, want the new normal ConfigMap appended to its normal bundle %q, "+ - "never to the encrypted sibling", res, want) - } -} - -// A file tolerated despite a non-editable construct (e.g. a YAML anchor) must -// never be joined — classifyCohortLocations excludes it from both bundle and -// singleton candidacy, so a genuinely new sibling falls through past it instead -// of silently landing beside content the writer cannot vouch for. -func TestLocateNew_TaintedSiblingNeverJoined(t *testing.T) { +// A declared template resolving onto a file the writer cannot vouch for — one holding a +// document tolerated despite a non-editable construct (a YAML anchor) — must not be +// appended to. Append stays false and the caller falls through to writeWholeFile, whose +// multi-document guard refuses rather than overwriting. This used to be enforced twice, +// once for cohort candidacy and once here; the append gate is now the only place, so it is +// worth pinning directly. +func TestLocateNew_DeclaredOntoTaintedFile_NeverAppends(t *testing.T) { tainted := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: anchored\n namespace: app\n" + "data: &d\n color: blue\nextra:\n <<: *d\n" fsys := fstest.MapFS{ "tainted.yaml": {Data: []byte(tainted)}, } store := placementStore(t, fsys) + policy := &PlacementPolicy{Default: "tainted.yaml"} - res, err := LocateNew(store, nil, newConfigMapRequest("cache", "app")) + res, err := LocateNew(store, policy, newConfigMapRequest("cache", "app")) if err != nil { t.Fatalf("LocateNew: %v", err) } - if res.Path == "tainted.yaml" || res.Source != PlacementSourceCanonical { - t.Fatalf("got %+v, want the tainted file excluded and canonical fallback used", res) + if res.Path != "tainted.yaml" { + t.Fatalf("got %+v, want the declared path honoured", res) + } + if res.Append { + t.Fatalf("got %+v, want Append false: the file holds a document the writer cannot account for", res) } } @@ -843,30 +786,6 @@ func TestFileIsAppendSafe(t *testing.T) { } } -func TestSpansMultipleNamespaces(t *testing.T) { - if spansMultipleNamespaces(nil) { - t.Error("no members cannot span multiple namespaces") - } - unresolved := []*DocumentModel{{ResourceIdentity: nil}} - if spansMultipleNamespaces(unresolved) { - t.Error("a document with no resolved ResourceIdentity contributes no namespace") - } - oneNamespace := []*DocumentModel{ - {ResourceIdentity: &types.ResourceIdentifier{Namespace: "a"}}, - {ResourceIdentity: &types.ResourceIdentifier{Namespace: "a"}}, - } - if spansMultipleNamespaces(oneNamespace) { - t.Error("members sharing one namespace do not span multiple namespaces") - } - twoNamespaces := []*DocumentModel{ - {ResourceIdentity: &types.ResourceIdentifier{Namespace: "a"}}, - {ResourceIdentity: &types.ResourceIdentifier{Namespace: "b"}}, - } - if !spansMultipleNamespaces(twoNamespaces) { - t.Error("members in two distinct namespaces must span multiple namespaces") - } -} - func TestValidateResolvedPlacementPath(t *testing.T) { cases := []struct { name string diff --git a/internal/queue/author_fact.go b/internal/queue/author_fact.go index 83743373..68959c99 100644 --- a/internal/queue/author_fact.go +++ b/internal/queue/author_fact.go @@ -79,7 +79,7 @@ const ( // // It is also the only tier the TTL does not bound. A uid is unique across space and time, so the // statement can never be superseded; its horizon is the index's caps instead. See - // docs/design/attribution-deletion-intent-actor.md. + // docs/spec/attribution.md. AttributionDeleteSticky AttributionResult = "delete_sticky" // AttributionExact is an exact UID+resourceVersion match: this actor produced this exact version. AttributionExact AttributionResult = "exact" diff --git a/internal/queue/fact_index_store.go b/internal/queue/fact_index_store.go index f99fdc9d..6e6739f0 100644 --- a/internal/queue/fact_index_store.go +++ b/internal/queue/fact_index_store.go @@ -199,7 +199,7 @@ func (s *scopeFacts) putLatest(uid string, entry *indexedFact) { // DELETION stamped, so both facts are filed under the same (uid, rv) and the same uid. Every other // structure here is last-writer-wins, so the deleter's fact is not outranked by the controller's — // it is replaced, and no tier ordering can recover what is no longer stored. See -// docs/design/attribution-deletion-intent-actor.md. +// docs/spec/attribution.md. // // A later removal fact may replace it, because that is a statement about the same question. func (s *scopeFacts) putRemoval(uid string, entry *indexedFact) { diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index 68855dc5..a903fd8b 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -43,6 +43,44 @@ var ( // is the configured behaviour, never a fault. PruneRetainedDocumentsTotal metric.Int64Counter + // PlacementsTotal counts new-file placements resolved for a resource with no document in + // Git yet — the only case placement runs for — labelled by {source, disposition, + // gittarget_namespace, gittarget_name, group, version, resource}. source is which + // mechanism chose the path (declared / kustomize_root / canonical) and disposition is what + // it did with it (new_file / appended). + // + // It exists because sibling inference was deleted (docs/design/open-asks-priority.md): a + // repository with a hand-authored layout now needs a placement.byType line, and + // `source="canonical"` is how its operator learns which type in which target is missing + // one, WITHOUT reading the folder. The (GitTarget, type) labels are the whole point — a + // bare "a fall-back happened somewhere" counter is not actionable, which is why the + // design doc argued against leading with one. Cardinality is bounded by targets × + // watched types, and placement fires only for a type/name the target has never written. + // + // Every increment is a resource that WAS mirrored; a resource the writer refused is + // PlacementRefusalsTotal instead, never a value of source here. + PlacementsTotal metric.Int64Counter + // PlacementRefusalsTotal counts resources the writer declined to place, labelled by + // {reason, gittarget_namespace, gittarget_name, group, version, resource}. Every + // increment is a resource NOT in the mirror: a declared template that escapes spec.path + // or is not identity-complete, a sensitive resource whose path is already taken, a + // plaintext resource routed at an encrypted file, or two resources of mixed sensitivity + // racing onto one brand-new file. The write is retried on the next event or resync, so a + // steady non-zero rate means a policy that needs fixing rather than a transient. + // + // This is the counter that did not exist: a refusal left a log line at the skip site and, + // on the resync path only, ResyncStats.PlacementSkipped — a field in a summary, not a + // series anything can alert on. + PlacementRefusalsTotal metric.Int64Counter + // PlacementKustomizationEntriesTotal counts attempts to add a new file to the + // resources: list of the kustomization that governs it, labelled by {outcome, + // gittarget_namespace, gittarget_name}. outcome is added, no_change, or failed. + // + // `failed` is the one to watch, and it is otherwise invisible: the document is committed + // and the entry is not, so kustomize never builds the file. The object is in Git, looks + // mirrored, and is not applied by anything. + PlacementKustomizationEntriesTotal metric.Int64Counter + // TargetReconcileCompletedTotal counts completed watch recovery passes per // GitTarget: each increment marks either a streaming-snapshot resync applied on // the branch worker or a cursor-backed watch resume (see Manager.recordTargetReconcileCompleted). @@ -261,6 +299,12 @@ func registerCounters() error { {"gitopsreverser_commits_total", &CommitsTotal}, {"gitopsreverser_resync_sweep_deletes_total", &ResyncSweepDeletesTotal}, {"gitopsreverser_prune_retained_documents_total", &PruneRetainedDocumentsTotal}, + {"gitopsreverser_placements_total", &PlacementsTotal}, + {"gitopsreverser_placement_refusals_total", &PlacementRefusalsTotal}, + { + "gitopsreverser_placement_kustomization_entries_total", + &PlacementKustomizationEntriesTotal, + }, {"gitopsreverser_target_reconcile_completed_total", &TargetReconcileCompletedTotal}, {"gitopsreverser_resync_background_failures_total", &ResyncBackgroundFailuresTotal}, {"gitopsreverser_audit_events_total", &AuditEventsTotal}, diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index b44a8d4b..5fd17082 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -916,7 +916,7 @@ func operationForWatchEvent(eventType watch.EventType) string { // the writer folds to a no-op against the already-absent path. deletionTimestamp is // server-owned runtime metadata (sanitize strips it), never desired state, so the // intent tree's invariant — a file present means the resource is intended to exist — -// holds. See docs/spec/deletecollection-attribution-expander.md §2. +// holds. See docs/spec/attribution.md §1. func operationForLiveTargetWatchEvent(eventType watch.EventType, u *unstructured.Unstructured) string { if u != nil && u.GetDeletionTimestamp() != nil { return string(configv1alpha3.OperationDelete) diff --git a/test/e2e/audit_route_attribution_e2e_test.go b/test/e2e/audit_route_attribution_e2e_test.go index e9f8949b..86ac56eb 100644 --- a/test/e2e/audit_route_attribution_e2e_test.go +++ b/test/e2e/audit_route_attribution_e2e_test.go @@ -29,7 +29,7 @@ import ( // source_namespace_e2e_test.go creates dedicated in-cluster providers but asserts Git paths, not // commit authors. The gap between them is exactly where the loss lived. // -// See docs/design/attribution-fact-identity.md. +// See docs/spec/attribution.md. var _ = Describe("Audit route attribution", Label("manager"), Ordered, func() { const ( // The apiserver's webhook-config.yaml posts to /audit-webhook/default, so this is the only diff --git a/test/e2e/deletecollection_intent_e2e_test.go b/test/e2e/deletecollection_intent_e2e_test.go index 2bee5301..ddf8d94a 100644 --- a/test/e2e/deletecollection_intent_e2e_test.go +++ b/test/e2e/deletecollection_intent_e2e_test.go @@ -36,7 +36,7 @@ import ( // The state correctness of collection deletes is solved by construction in watch-first // (one watch event per object); this spec adds the intent semantics and the attribution // that the headline claim depends on. See -// docs/spec/deletecollection-attribution-expander.md. +// docs/spec/attribution.md. // // Not Serial: the GitProvider uses a 0s commit window, so every watched event commits // immediately as its own commit, and every assertion reads the author/state scoped to a diff --git a/test/e2e/inplace_edit_e2e_test.go b/test/e2e/inplace_edit_e2e_test.go index 8637f454..e2a68f55 100644 --- a/test/e2e/inplace_edit_e2e_test.go +++ b/test/e2e/inplace_edit_e2e_test.go @@ -251,7 +251,9 @@ var _ = Describe( g.Expect(nestedBody).NotTo(ContainSubstring("namespace:")) kustomizationBody := readRepoFile(g, kustomizationFullPath) - g.Expect(kustomizationBody).To(Equal(readRepoFile(g, renderedKustomization))) + expectKustomizationOnlyGainedResourceEntries( + g, readRepoFile(g, renderedKustomization), kustomizationBody, + ) for _, name := range []string{bundleConfigMapName, nestedConfigMapName} { canonicalPath := filepath.Join(repo.CheckoutDir, gitPath, testNs, "configmaps", name+".yaml") @@ -266,6 +268,49 @@ var _ = Describe( }, ) +// expectKustomizationOnlyGainedResourceEntries asserts that the committed kustomization still +// holds every line the human wrote, in order, and that anything added to it is a `resources:` +// entry. +// +// It replaces a byte-equality assertion, which is no longer the right question. This namespace +// holds a ConfigMap nobody in the test created — the cluster's own `kube-root-ca.crt` — and the +// WatchRule selects every ConfigMap, so the operator has a watched resource with no document in +// Git. Placement gives it a file beside this folder's one kustomization and registers it, which +// is the documented kustomize-root behaviour (and what the new-file-placement spec asserts +// directly). Before sibling inference was deleted, that resource was appended to the existing +// bundle instead, and the bundle was already listed — so the build file happened to stay +// byte-identical. Byte equality was therefore pinning a side effect of inference, not the +// property this spec is about: a hand-authored build file must not be reordered, reformatted, or +// have anything taken out of it. +func expectKustomizationOnlyGainedResourceEntries(g Gomega, original, committed string) { + GinkgoHelper() + + committedLines := strings.Split(committed, "\n") + next := 0 + for _, want := range strings.Split(original, "\n") { + found := false + for ; next < len(committedLines); next++ { + if committedLines[next] == want { + next++ + found = true + break + } + g.Expect(committedLines[next]).To(MatchRegexp(`^\s+- \S+\.ya?ml$`), + "the committed kustomization gained a line that is not a resources: entry: %q", + committedLines[next]) + } + g.Expect(found).To(BeTrue(), + "the committed kustomization lost (or reordered) a line the fixture wrote: %q\n%s", want, committed) + } + for ; next < len(committedLines); next++ { + if committedLines[next] == "" { + continue + } + g.Expect(committedLines[next]).To(MatchRegexp(`^\s+- \S+\.ya?ml$`), + "trailing line is not a resources: entry: %q", committedLines[next]) + } +} + // seedCommentIntoRepoFile inserts a YAML comment under the data block of the // committed manifest and pushes it to main, authenticating the local checkout's // origin from the GitTarget's Git Secret. It retries once over a remote race by diff --git a/test/mutationlab/README.md b/test/mutationlab/README.md index 7f6283ac..411c663a 100644 --- a/test/mutationlab/README.md +++ b/test/mutationlab/README.md @@ -54,7 +54,7 @@ The finding is in the last column: the human's `delete` and the controller's fin return a body, and both bodies carry the resourceVersion the DELETION stamped, so the two facts they produce collide on one key. `LAB_FINALIZER_HOLD` tunes the gap between the phases, which is what decides whether both land in one audit batch. See -[attribution-deletion-intent-actor.md](../../docs/design/attribution-deletion-intent-actor.md). +[the attribution spec](../../docs/spec/attribution.md). All seventeen catalogued scenarios are now captured. Rows 15a and 15b are not catalog rows: they extend row 15 to the removal verbs, because the create alone could not say