CCIX-213: Expose lifecycle UNKNOWN causes

ARCHITECTURE.md

@@ -328,7 +328,36 @@ type VersionPolicy interface {
 - 🔴 **RED**: Past EOL, deprecated, or extended support expired
 - 🟡 **YELLOW**: In extended support or approaching EOL (< 90 days)
 - 🟢 **GREEN**: In standard support, current version
-- ⚪ **UNKNOWN**: Version not found in EOL database
+- ⚪ **UNKNOWN**: Lifecycle lookup or classification was inconclusive
+
+UNKNOWN findings use a bounded cause vocabulary: `product_not_found`,
+`cycle_not_found`, `source_error`, `malformed_cycle`,
+`empty_inventory_version`, `lifecycle_mismatch`,
+`indeterminate_lifecycle`, and `unattributed`.
+
+### Lifecycle attribution and diagnostics
+
+The endoflife.date client records where each lifecycle response came from.
+Requests sent directly to the default `https://endoflife.date/api` endpoint
+resolve to `endoflife_date`. The nginx override shim marks local JSON responses
+as `local_override` and proxied upstream responses as `endoflife_date` using
+the trusted `X-Version-Guard-EOL-Source` response header. Custom or otherwise
+untrusted endpoints default to `unknown` unless they provide one of those
+recognized header values; arbitrary values are not propagated.
+
+Each snapshot finding's `eol` object preserves `unknown_cause`, `data_source`,
+`engine`, and `version` for drill-down. Prometheus uses only bounded labels:
+`version_guard_detection_unknown_resources{resource_type,cause}` and
+`version_guard_detection_lifecycle_resources{resource_type,source}`. Engine
+and version are intentionally excluded from labels to avoid unbounded
+cardinality.
+
+Local overrides are governed by the machine-readable
+[`deploy/endoflife-override/manifest.json`](./deploy/endoflife-override/manifest.json).
+The [override policy and validation workflow](./deploy/endoflife-override/README.md)
+requires ownership, provenance, review dates, one-to-one manifest/file
+coverage, and runtime-compatible lifecycle data. Overdue reviews warn; invalid
+metadata or lifecycle data fails validation.
 
 ### 4. Detection Pipeline
 

README.md

@@ -188,6 +188,21 @@ The same OpenMetrics endpoint exports `temporal_*`, `version_guard_*`,
 `go_*`, and `process_*` series. Datadog/BPCI scrape configuration must allow
 all four families for the RCA dashboard panels to populate.
 
+Lifecycle diagnostics are exposed through two bounded-cardinality gauges:
+
+- `version_guard_detection_unknown_resources{resource_type,cause}` reports the
+  latest UNKNOWN count. `cause` is one of `product_not_found`,
+  `cycle_not_found`, `source_error`, `malformed_cycle`,
+  `empty_inventory_version`, `lifecycle_mismatch`,
+  `indeterminate_lifecycle`, or `unattributed`.
+- `version_guard_detection_lifecycle_resources{resource_type,source}` reports
+  the latest resource count by lifecycle source. `source` is one of
+  `endoflife_date`, `local_override`, or `unknown`.
+
+Prometheus deliberately does not label these metrics with engine or version,
+which would create unbounded series. Use the snapshot `eol` object's
+`unknown_cause`, `data_source`, `engine`, and `version` fields for drill-down.
+
 #### End-to-end with `make compose-*`
 
 The same commands work for everyone — they auto-detect whether a webhook-style emitter is present and adjust accordingly:
@@ -473,7 +488,13 @@ See `./bin/version-guard --help` for all options.
 | 🔴 **RED** | Past EOL, deprecated, extended support expired | Urgent upgrade required |
 | 🟡 **YELLOW** | In extended support (costly), approaching EOL (< 90 days) | Plan upgrade soon |
 | 🟢 **GREEN** | In standard support, current version | Compliant |
-| ⚪ **UNKNOWN** | Version not found in EOL database | Investigate |
+| ⚪ **UNKNOWN** | Lifecycle lookup or classification was inconclusive (see bounded causes below) | Investigate |
+
+UNKNOWN is attributed to one of: `product_not_found`, `cycle_not_found`,
+`source_error`, `malformed_cycle`, `empty_inventory_version`,
+`lifecycle_mismatch`, `indeterminate_lifecycle`, or `unattributed`. The
+snapshot `eol` object preserves the cause and lifecycle source alongside the
+engine and version for diagnosis.
 
 ## 🔌 Extending Version Guard
 
@@ -565,6 +586,7 @@ constants used in tests.
           "version": "5.7",
           "engine": "mysql",
           "source": "endoflife-date-api",
+          "data_source": "local_override",
           "is_supported": true,
           "is_deprecated": true,
           "is_extended_support": true,

USAGE.md

@@ -364,6 +364,28 @@ Useful SDK metrics include:
 - `temporal_request_failure_total`
 - `temporal_request_latency_seconds`
 
+Version Guard also exposes lifecycle diagnostic gauges:
+
+- `version_guard_detection_unknown_resources{resource_type,cause}` — latest
+  UNKNOWN resources by resource type and bounded cause. Causes are
+  `product_not_found`, `cycle_not_found`, `source_error`, `malformed_cycle`,
+  `empty_inventory_version`, `lifecycle_mismatch`,
+  `indeterminate_lifecycle`, and `unattributed`.
+- `version_guard_detection_lifecycle_resources{resource_type,source}` — latest
+  resources by resource type and lifecycle source. Sources are
+  `endoflife_date`, `local_override`, and `unknown`.
+
+Engine and version are intentionally not Prometheus labels. For a specific
+resource, inspect its snapshot `eol.unknown_cause`, `eol.data_source`,
+`eol.engine`, and `eol.version` fields instead. Direct requests to the default
+endoflife.date API resolve to `endoflife_date`; nginx-served local files resolve
+to `local_override`; custom endpoints without a recognized
+`X-Version-Guard-EOL-Source` header resolve to `unknown`.
+
+Operators adding, reviewing, or removing local overrides must update
+[`deploy/endoflife-override/manifest.json`](./deploy/endoflife-override/manifest.json)
+and follow its [validation policy](./deploy/endoflife-override/README.md).
+
 Set `TEMPORAL_METRICS_ENABLED=false` to disable the handler, or
 `TEMPORAL_METRICS_LISTEN_ADDRESS=0.0.0.0:9091` to change the listen address.
 
@@ -780,7 +802,11 @@ A: Next scan will detect the new version and auto-resolve the finding.
 A: No, Version Guard only detects and reports. You must upgrade manually.
 
 **Q: What if my resource version isn't in the EOL database?**
-A: Finding will show status UNKNOWN. You can extend the EOL provider to add version data.
+A: The finding will show UNKNOWN with a bounded `eol.unknown_cause`. UNKNOWN
+also covers source errors, malformed lifecycle data, empty inventory versions,
+lifecycle mismatches, and indeterminate lifecycle records—not only missing
+versions. Inspect `eol.data_source`, `eol.engine`, and `eol.version` in the
+snapshot to choose the remediation.
 
 **Q: How do I add a new resource type?**
 A: See [Runbook 1](#runbook-1-onboarding-new-resource-type) above.

deploy/endoflife-override/README.md

@@ -29,18 +29,25 @@ curl -s https://deploy-preview-9534--endoflife-date.netlify.app/api/amazon-auror
   | python3 -m json.tool > api/amazon-aurora-mysql.json
 ```
 
-2. Restart docker-compose — no rebuild needed:
+2. Add or update the corresponding entry in `manifest.json`. The source URL,
+owner, reason, review date, and review due date are required. Reviews may be
+scheduled at most 30 days apart.
+
+3. Run the override package tests, then restart docker-compose — no rebuild needed:
 
 ```bash
+go test ./deploy/endoflife-override
 docker compose restart endoflife
 ```
 
-## Current Overrides
+The validator checks the manifest schema and metadata, one-to-one coverage of
+manifest entries and `api/*.json` files, and lifecycle data using the same
+validation as the runtime provider. Malformed metadata, missing files, invalid
+URLs, and invalid lifecycle data fail validation. An expired review due date is
+warn-only so CI continues to run while making the overdue review visible.
 
-| File | Reason | Upstream PR |
-|------|--------|-------------|
-| `amazon-aurora-mysql.json` | Product not yet on endoflife.date | [#9534](https://github.com/endoflife-date/endoflife.date/pull/9534) |
-| `amazon-opensearch.json` | Missing cycles 3.3 and 3.5 | [#9919](https://github.com/endoflife-date/endoflife.date/pull/9919) |
+`manifest.json` is the machine-readable source of truth for current overrides.
+Update it whenever an override is added, reviewed, or removed.
 
 ## Configuration
 
@@ -57,3 +64,7 @@ When `EOL_BASE_URL` is not set, Version Guard connects directly to `https://endo
 ## Removing Overrides
 
 Once an upstream PR is merged, delete the local JSON file. Nginx will then proxy that product to the upstream API automatically.
+
+Delete its `manifest.json` entry in the same change. Nginx marks local and
+upstream responses with authoritative `X-Version-Guard-EOL-Source` headers;
+those values flow into snapshot findings and lifecycle source metrics.

deploy/endoflife-override/manifest.go

@@ -0,0 +1,231 @@
+package override
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/block/Version-Guard/pkg/eol/endoflife"
+)
+
+const (
+	manifestSchemaVersion = 1
+	dateLayout            = "2006-01-02"
+	maximumReviewInterval = 30 * 24 * time.Hour
+)
+
+type manifest struct {
+	Overrides     []manifestOverride `json:"overrides"`
+	SchemaVersion int                `json:"schema_version"`
+}
+
+type manifestOverride struct {
+	Product     string `json:"product"`
+	Path        string `json:"path"`
+	Reason      string `json:"reason"`
+	Owner       string `json:"owner"`
+	SourceURL   string `json:"source_url"`
+	ReviewedOn  string `json:"reviewed_on"`
+	ReviewDueOn string `json:"review_due_on"`
+}
+
+func validateManifest(root string, now time.Time, warnings io.Writer) error {
+	m, err := readManifest(filepath.Join(root, "manifest.json"))
+	if err != nil {
+		return err
+	}
+	if m.SchemaVersion != manifestSchemaVersion {
+		return fmt.Errorf("schema_version must be %d", manifestSchemaVersion)
+	}
+	if warnings == nil {
+		warnings = io.Discard
+	}
+
+	apiDirectory := filepath.Join(root, "api")
+	apiInfo, err := os.Lstat(apiDirectory)
+	if err != nil {
+		return fmt.Errorf("stat API directory: %w", err)
+	}
+	if apiInfo.Mode()&os.ModeSymlink != 0 {
+		return fmt.Errorf("API directory must not be a symlink")
+	}
+	if !apiInfo.IsDir() {
+		return fmt.Errorf("API directory is not a directory")
+	}
+
+	products := make(map[string]struct{}, len(m.Overrides))
+	paths := make(map[string]struct{}, len(m.Overrides))
+	for index := range m.Overrides {
+		override := &m.Overrides[index]
+		if validationErr := validateOverride(root, override, now.UTC(), warnings, products, paths); validationErr != nil {
+			return fmt.Errorf("override %d: %w", index, validationErr)
+		}
+	}
+
+	apiFiles, err := filepath.Glob(filepath.Join(apiDirectory, "*.json"))
+	if err != nil {
+		return fmt.Errorf("list API files: %w", err)
+	}
+	for _, apiFile := range apiFiles {
+		relative, err := filepath.Rel(root, apiFile)
+		if err != nil {
+			return fmt.Errorf("resolve API file %q: %w", apiFile, err)
+		}
+		relative = filepath.ToSlash(relative)
+		if _, ok := paths[relative]; !ok {
+			return fmt.Errorf("API file %q has no manifest entry", relative)
+		}
+	}
+	return nil
+}
+
+func readManifest(path string) (*manifest, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open manifest: %w", err)
+	}
+	defer file.Close()
+	decoder := json.NewDecoder(file)
+	decoder.DisallowUnknownFields()
+	var m manifest
+	if err := decoder.Decode(&m); err != nil {
+		return nil, fmt.Errorf("decode manifest: %w", err)
+	}
+	if err := ensureJSONEOF(decoder); err != nil {
+		return nil, fmt.Errorf("decode manifest: %w", err)
+	}
+	return &m, nil
+}
+
+//nolint:gocyclo // Validation intentionally reports the first field-specific policy violation.
+func validateOverride(root string, override *manifestOverride, now time.Time, warnings io.Writer, products, paths map[string]struct{}) error {
+	required := []struct {
+		name  string
+		value string
+	}{
+		{"product", override.Product}, {"path", override.Path}, {"reason", override.Reason},
+		{"owner", override.Owner}, {"source_url", override.SourceURL},
+		{"reviewed_on", override.ReviewedOn}, {"review_due_on", override.ReviewDueOn},
+	}
+	for _, field := range required {
+		if strings.TrimSpace(field.value) == "" {
+			return fmt.Errorf("%s is required", field.name)
+		}
+	}
+	if _, exists := products[override.Product]; exists {
+		return fmt.Errorf("duplicate product %q", override.Product)
+	}
+	products[override.Product] = struct{}{}
+	if _, exists := paths[override.Path]; exists {
+		return fmt.Errorf("duplicate path %q", override.Path)
+	}
+	paths[override.Path] = struct{}{}
+
+	if !strings.HasPrefix(override.SourceURL, "https://") {
+		return fmt.Errorf("source_url must use https")
+	}
+	if _, err := parseHTTPSURL(override.SourceURL); err != nil {
+		return err
+	}
+	reviewedOn, err := parseManifestDate("reviewed_on", override.ReviewedOn)
+	if err != nil {
+		return err
+	}
+	reviewDueOn, err := parseManifestDate("review_due_on", override.ReviewDueOn)
+	if err != nil {
+		return err
+	}
+	interval := reviewDueOn.Sub(reviewedOn)
+	if interval < 0 {
+		return fmt.Errorf("review_due_on is before reviewed_on")
+	}
+	if interval > maximumReviewInterval {
+		return fmt.Errorf("review interval exceeds 30 days")
+	}
+	if !now.Before(reviewDueOn.AddDate(0, 0, 1)) {
+		fmt.Fprintf(warnings, "warning: review overdue for %s (due %s)\n", override.Product, override.ReviewDueOn)
+	}
+
+	cleanPath := filepath.ToSlash(filepath.Clean(override.Path))
+	filename := strings.TrimPrefix(cleanPath, "api/")
+	if cleanPath != override.Path || filename == cleanPath || filename == "" ||
+		strings.ContainsAny(filename, `/\`) || filepath.Ext(filename) != ".json" {
+		return fmt.Errorf("path %q must be a direct api/<filename>.json path", override.Path)
+	}
+	if strings.TrimSuffix(filename, ".json") != override.Product {
+		return fmt.Errorf("path filename must match product %q", override.Product)
+	}
+	fullPath := filepath.Join(root, filepath.FromSlash(cleanPath))
+	info, err := os.Lstat(fullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("path %q does not exist", override.Path)
+		}
+		return fmt.Errorf("stat path %q: %w", override.Path, err)
+	}
+	if info.Mode()&os.ModeSymlink != 0 {
+		return fmt.Errorf("path %q must not be a symlink", override.Path)
+	}
+	if !info.Mode().IsRegular() {
+		return fmt.Errorf("path %q is not a regular file", override.Path)
+	}
+	return validateAPIFile(fullPath)
+}
+
+func parseHTTPSURL(raw string) (*url.URL, error) {
+	parsed, err := url.ParseRequestURI(raw)
+	if err != nil || parsed.Scheme != "https" || parsed.Host == "" {
+		return nil, fmt.Errorf("source_url must be a valid https URL")
+	}
+	return parsed, nil
+}
+
+func parseManifestDate(name, value string) (time.Time, error) {
+	parsed, err := time.Parse(dateLayout, value)
+	if err != nil || parsed.Format(dateLayout) != value {
+		return time.Time{}, fmt.Errorf("%s must use YYYY-MM-DD", name)
+	}
+	return parsed, nil
+}
+
+func validateAPIFile(path string) error {
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("open API file %q: %w", path, err)
+	}
+	defer file.Close()
+	decoder := json.NewDecoder(file)
+	decoder.DisallowUnknownFields()
+	var cycles []*endoflife.ProductCycle
+	if err := decoder.Decode(&cycles); err != nil {
+		return fmt.Errorf("decode API file %q: %w", path, err)
+	}
+	if cycles == nil {
+		return fmt.Errorf("API file %q must contain a top-level array", path)
+	}
+	if err := ensureJSONEOF(decoder); err != nil {
+		return fmt.Errorf("decode API file %q: %w", path, err)
+	}
+	for index, cycle := range cycles {
+		if err := endoflife.ValidateProductCycle(cycle); err != nil {
+			return fmt.Errorf("API file %q cycle %d: %w", path, index, err)
+		}
+	}
+	return nil
+}
+
+func ensureJSONEOF(decoder *json.Decoder) error {
+	var extra any
+	if err := decoder.Decode(&extra); err != io.EOF {
+		if err == nil {
+			return fmt.Errorf("multiple JSON values")
+		}
+		return err
+	}
+	return nil
+}