diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ffd8d..d3549a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## Unreleased + +### Added + +- Per-environment log-view scoping: `log_view` (with optional `log_bucket`, `log_location`) confines queries to a single Cloud Logging log view instead of the whole project. When `log_view` is unset, behavior is unchanged (project scope). +- `fx-ci-scoped` / `community-tc-scoped` / `staging-scoped` example environments, intended for untrusted agents/containers: a token minted from the narrow `tc-logview-reader` service account can read only TaskCluster's per-namespace tenant log bucket. Under `-v`, the active scope is printed as `Scope: `. + +### Changed + +- `TASKCLUSTER_ROOT_URL` auto-detection now ignores log-view-scoped environments (they share a `root_url` with their broad counterpart). Scoped envs must be selected explicitly with `--env`; auto-detect resolves deterministically to the broad env. +- Result cache keys now include the environment's project and log-view scope, so scoped and broad environments no longer share cache entries. + +### Notes + +- Infrastructure presets (`k8s.*`, `cloudsql.*`) are not available on `*-scoped` envs — those logs live in other projects/buckets; querying one now returns a clear error (instead of silently hitting the wrong project). Use the broad envs with your own ADC. + ## v1.3.1 - 2026-05-29 ### Changed diff --git a/README.md b/README.md index b872065..e9ee8ac 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,32 @@ environments: cluster: "taskcluster-dev" root_url: "https://tc.dev.taskcluster.mozgcp.net" # key_path omitted — uses ADC by default + + # Scoped envs for untrusted agents — query a single log view, not the project. + fx-ci-scoped: + project_id: "moz-fx-taskcluster-prod" + cluster: "webservices-high-prod" + namespace: "taskcluster-prod" + log_bucket: "gke-taskcluster-prod-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://firefox-ci-tc.services.mozilla.com" + community-tc-scoped: + project_id: "moz-fx-taskcluster-prod" + cluster: "webservices-high-prod" + namespace: "taskcluster-communitytc" + log_bucket: "gke-taskcluster-communitytc-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://community-tc.services.mozilla.com" + staging-scoped: + project_id: "moz-fx-webservices-high-nonpro" + cluster: "webservices-high-nonprod" + namespace: "taskcluster-stage" + log_bucket: "gke-taskcluster-stage-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://stage.taskcluster.nonprod.cloudops.mozgcp.net" ``` ### 2. Authenticate to GCP @@ -106,6 +132,43 @@ When `TC_LOGVIEW_ACCESS_TOKEN` is set it takes priority over `key_path` and ADC. Re-mint and re-inject the token before it expires for long-running containers. An expired token surfaces as a `401` from the query path with a hint pointing back at the re-mint command. +#### Scoping a token to TaskCluster logs only (`*-scoped` envs) + +An access token minted from your own ADC inherits your privileges — too broad for an untrusted agent. To hand an agent a token that can read **only** TaskCluster's logs, mint it from a dedicated, narrowly-scoped reader service account and point tc-logview at a **log view** instead of the whole project. + +This is what the `*-scoped` environments are for. They set three optional fields that confine every query to a single Cloud Logging log view: + +| Field | Meaning | Default | +|---|---|---| +| `log_view` | View ID; when set, queries are scoped to this view (otherwise project scope) | — (project scope) | +| `log_bucket` | Bucket holding the view | `_Default` | +| `log_location` | Bucket location | `global` | + +The `tc-logview-reader` service account is granted `roles/logging.viewAccessor` on exactly that view (a per-namespace tenant log bucket), so a token minted from it can read nothing else: + +```bash +# Host: mint a narrow, short-lived token by impersonating the reader SA. +TOKEN=$(gcloud auth print-access-token \ + --impersonate-service-account=tc-logview-reader@moz-fx-taskcluster-prod.iam.gserviceaccount.com) + +# Agent/container: scoped env queries only the TaskCluster log view. +TC_LOGVIEW_ACCESS_TOKEN=$TOKEN tc-logview query -e fx-ci-scoped --type monitor.error --since 1h +``` + +The same `tc-logview-reader` service account backs all scoped envs: + +| Scoped env | Project | Log bucket | +|---|---|---| +| `fx-ci-scoped` | `moz-fx-taskcluster-prod` | `gke-taskcluster-prod-log-bucket` | +| `community-tc-scoped` | `moz-fx-taskcluster-prod` | `gke-taskcluster-communitytc-log-bucket` | +| `staging-scoped` | `moz-fx-webservices-high-nonpro` | `gke-taskcluster-stage-log-bucket` | + +The reader SA lives in the prod project; for `staging-scoped` it's granted `viewAccessor` on the stage bucket in the nonprod project, so the same impersonation command works for all three. + +> Infrastructure presets (`k8s.*`, `cloudsql.*`) are **not** available on `*-scoped` envs — those logs live in other projects/buckets the reader SA can't see. Use the broad envs (with your own ADC) for infra queries. + +> Auto-detection via `TASKCLUSTER_ROOT_URL` always resolves to the broad env (scoped envs share its `root_url`); select a scoped env explicitly with `-e`. + ### 3. Sync references ```bash diff --git a/cmd/config_init.go b/cmd/config_init.go index 0348424..2081c73 100644 --- a/cmd/config_init.go +++ b/cmd/config_init.go @@ -75,6 +75,36 @@ environments: cluster: "taskcluster-dev" root_url: "https://tc.dev.taskcluster.mozgcp.net" # key_path omitted — uses ADC by default + + # Scoped environments for untrusted agents/containers. These query a single + # Cloud Logging log view (a per-namespace tenant bucket) instead of the whole + # project, so an injected TC_LOGVIEW_ACCESS_TOKEN minted from the narrow + # tc-logview-reader service account can read ONLY TaskCluster service logs. + # Infra/k8s/CloudSQL presets are not available here — use the broad envs above. + fx-ci-scoped: + project_id: "moz-fx-taskcluster-prod" + cluster: "webservices-high-prod" + namespace: "taskcluster-prod" + log_bucket: "gke-taskcluster-prod-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://firefox-ci-tc.services.mozilla.com" + community-tc-scoped: + project_id: "moz-fx-taskcluster-prod" + cluster: "webservices-high-prod" + namespace: "taskcluster-communitytc" + log_bucket: "gke-taskcluster-communitytc-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://community-tc.services.mozilla.com" + staging-scoped: + project_id: "moz-fx-webservices-high-nonpro" + cluster: "webservices-high-nonprod" + namespace: "taskcluster-stage" + log_bucket: "gke-taskcluster-stage-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://stage.taskcluster.nonprod.cloudops.mozgcp.net" ` if err := os.WriteFile(cfgPath, []byte(example), 0o644); err != nil { return fmt.Errorf("writing config: %w", err) diff --git a/cmd/query.go b/cmd/query.go index 85f876a..415ab0e 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -94,6 +94,14 @@ func runQuery(cmd *cobra.Command, args []string) error { } if preset != nil { + // Infra presets (k8s.*, cloudsql.*) read logs that live outside the scoped + // log view (other projects/buckets), so they are not available on + // log-view-scoped environments. Fail loud rather than silently querying the + // wrong project (or relying on an IAM denial). + if env.LogViewResource() != "" { + return fmt.Errorf("preset %q is not available on log-view-scoped environments (those with log_view set); use a non-scoped environment for k8s/CloudSQL presets", preset.Name) + } + // Preset mode: use preset's filter and field mappings presetFilter := preset.Filter skipCluster := false @@ -136,7 +144,7 @@ func runQuery(cmd *cobra.Command, args []string) error { // Query, cache, format — same as TC path resultsCache := cache.New(filepath.Join(config.CacheDir(), "results"), resultsCacheTTL) - cacheKey := resultsCache.Key(env.Cluster, fromTime.Format(time.RFC3339), toTime.Format(time.RFC3339), filterStr) + cacheKey := resultsCache.Key(env.Cluster, env.ProjectID, env.LogViewResource(), fromTime.Format(time.RFC3339), toTime.Format(time.RFC3339), filterStr) var rawEntries []map[string]interface{} var totalCount int @@ -169,7 +177,9 @@ func runQuery(cmd *cobra.Command, args []string) error { defer client.Close() logInfo("Querying GCP Cloud Logging...") - result, err := client.Query(ctx, filterStr, queryLimit+queryOffset) + // Presets (k8s/CloudSQL) query at project scope; they are not served + // by a tenant log view, so resourceName is intentionally empty. + result, err := client.Query(ctx, filterStr, "", queryLimit+queryOffset) if err != nil { wrapped := fmt.Errorf("querying logs (auth=%s): %w", gcp.AuthModeLabel(auth), err) if hint := authHint(err, auth); hint != "" { @@ -291,7 +301,7 @@ func runQuery(cmd *cobra.Command, args []string) error { // Check results cache resultsCache := cache.New(filepath.Join(config.CacheDir(), "results"), resultsCacheTTL) - cacheKey := resultsCache.Key(env.Cluster, fromTime.Format(time.RFC3339), toTime.Format(time.RFC3339), filterStr) + cacheKey := resultsCache.Key(env.Cluster, env.ProjectID, env.LogViewResource(), fromTime.Format(time.RFC3339), toTime.Format(time.RFC3339), filterStr) var rawEntries []map[string]interface{} var totalCount int @@ -320,8 +330,11 @@ func runQuery(cmd *cobra.Command, args []string) error { } defer client.Close() + if rn := env.LogViewResource(); rn != "" { + logInfo("Scope: %s", rn) + } logInfo("Querying GCP Cloud Logging...") - result, err := client.Query(ctx, filterStr, queryLimit+queryOffset) + result, err := client.Query(ctx, filterStr, env.LogViewResource(), queryLimit+queryOffset) if err != nil { wrapped := fmt.Errorf("querying logs (auth=%s): %w", gcp.AuthModeLabel(auth), err) if hint := authHint(err, auth); hint != "" { diff --git a/cmd/root.go b/cmd/root.go index 6f906e3..7bb5008 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -124,12 +124,25 @@ func resolveEnv() (*config.Environment, error) { } rootURL := os.Getenv("TASKCLUSTER_ROOT_URL") if rootURL != "" { + // Auto-detection deliberately ignores log-view-scoped environments: + // multiple envs (broad + scoped) can share a root_url, and a scoped env + // uses different credentials/scope. Scoped access must be opted into + // explicitly with --env, so detection resolves to the broad env only. + var scopedMatch string for name, env := range cfg.Environments { - if env.RootURL == rootURL { - e := env - logInfo("Auto-detected environment: %s", name) - return &e, nil + if env.RootURL != rootURL { + continue } + if env.LogViewResource() != "" { + scopedMatch = name + continue + } + e := env + logInfo("Auto-detected environment: %s", name) + return &e, nil + } + if scopedMatch != "" { + return nil, fmt.Errorf("TASKCLUSTER_ROOT_URL=%q matches only the scoped environment %q; select it explicitly with --env", rootURL, scopedMatch) } return nil, fmt.Errorf("TASKCLUSTER_ROOT_URL=%q does not match any environment", rootURL) } diff --git a/cmd/root_test.go b/cmd/root_test.go index 2ffe3fb..dadd039 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/taskcluster/tc-logview/internal/config" "github.com/taskcluster/tc-logview/internal/gcp" ) @@ -53,6 +54,55 @@ func TestAuthModeMessage(t *testing.T) { } } +func TestResolveEnvAutoDetectSkipsScoped(t *testing.T) { + prevCfg, prevFlag := cfg, envFlag + t.Cleanup(func() { cfg, envFlag = prevCfg, prevFlag }) + + envFlag = "" + t.Setenv("TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com") + cfg = &config.Config{Environments: map[string]config.Environment{ + "fx-ci": { + ProjectID: "moz-fx-webservices-high-prod", + RootURL: "https://firefox-ci-tc.services.mozilla.com", + }, + "fx-ci-scoped": { + ProjectID: "moz-fx-taskcluster-prod", + RootURL: "https://firefox-ci-tc.services.mozilla.com", + LogBucket: "gke-taskcluster-prod-log-bucket", + LogLocation: "global", + LogView: "_AllLogs", + }, + }} + + // Map iteration is randomized; a nondeterministic resolveEnv would eventually + // pick the scoped env. Run enough times to catch that. + for i := 0; i < 50; i++ { + env, err := resolveEnv() + if err != nil { + t.Fatalf("resolveEnv: %v", err) + } + if env.ProjectID != "moz-fx-webservices-high-prod" || env.LogViewResource() != "" { + t.Fatalf("auto-detect selected scoped env (project=%s, view=%q); want broad env", + env.ProjectID, env.LogViewResource()) + } + } +} + +func TestResolveEnvScopedOnlyRequiresExplicit(t *testing.T) { + prevCfg, prevFlag := cfg, envFlag + t.Cleanup(func() { cfg, envFlag = prevCfg, prevFlag }) + + envFlag = "" + t.Setenv("TASKCLUSTER_ROOT_URL", "https://example.com") + cfg = &config.Config{Environments: map[string]config.Environment{ + "only-scoped": {ProjectID: "p", RootURL: "https://example.com", LogView: "_AllLogs"}, + }} + + if _, err := resolveEnv(); err == nil { + t.Fatal("expected error when only a scoped env matches TASKCLUSTER_ROOT_URL") + } +} + func TestAuthHint(t *testing.T) { adcHintFragment := "gcloud auth application-default login" tokenHintFragment := "may be expired" diff --git a/internal/config/config.go b/internal/config/config.go index 46e34fb..0f8bb7c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,13 +10,16 @@ import ( ) type Environment struct { - ProjectID string `yaml:"project_id"` + ProjectID string `yaml:"project_id"` CloudSQLProjectID string `yaml:"cloudsql_project_id,omitempty"` - Cluster string `yaml:"cluster"` - Namespace string `yaml:"namespace,omitempty"` - RootURL string `yaml:"root_url"` - KeyPath string `yaml:"key_path,omitempty"` - CloudSQLInstance string `yaml:"cloudsql_instance,omitempty"` + Cluster string `yaml:"cluster"` + Namespace string `yaml:"namespace,omitempty"` + RootURL string `yaml:"root_url"` + KeyPath string `yaml:"key_path,omitempty"` + CloudSQLInstance string `yaml:"cloudsql_instance,omitempty"` + LogBucket string `yaml:"log_bucket,omitempty"` + LogLocation string `yaml:"log_location,omitempty"` + LogView string `yaml:"log_view,omitempty"` } // CloudSQLProject returns the GCP project that holds the CloudSQL instance. @@ -28,6 +31,26 @@ func (e Environment) CloudSQLProject() string { return e.ProjectID } +// LogViewResource returns the fully-qualified Cloud Logging log view resource +// name to scope queries to, or "" to query at project scope (the default, +// unchanged behavior). A non-empty LogView is the trigger; LogBucket defaults +// to "_Default" and LogLocation to "global" when unset. +func (e Environment) LogViewResource() string { + if e.LogView == "" { + return "" + } + bucket := e.LogBucket + if bucket == "" { + bucket = "_Default" + } + location := e.LogLocation + if location == "" { + location = "global" + } + return fmt.Sprintf("projects/%s/locations/%s/buckets/%s/views/%s", + e.ProjectID, location, bucket, e.LogView) +} + type Config struct { Environments map[string]Environment `yaml:"environments"` } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index fe92a13..745f29f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -116,6 +116,69 @@ func TestCloudSQLProject(t *testing.T) { } } +func TestLogViewResource(t *testing.T) { + tests := []struct { + name string + env Environment + want string + }{ + { + name: "no log_view -> project scope (empty)", + env: Environment{ProjectID: "p"}, + want: "", + }, + { + name: "full fields", + env: Environment{ + ProjectID: "moz-fx-taskcluster-prod", + LogBucket: "gke-taskcluster-prod-log-bucket", + LogLocation: "global", + LogView: "_AllLogs", + }, + want: "projects/moz-fx-taskcluster-prod/locations/global/buckets/gke-taskcluster-prod-log-bucket/views/_AllLogs", + }, + { + name: "defaults applied when bucket/location omitted", + env: Environment{ProjectID: "p", LogView: "myview"}, + want: "projects/p/locations/global/buckets/_Default/views/myview", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.env.LogViewResource(); got != tt.want { + t.Errorf("LogViewResource() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestLogViewResourceRoundTrip(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.yaml") + content := `environments: + fx-ci-scoped: + project_id: "moz-fx-taskcluster-prod" + cluster: "webservices-high-prod" + namespace: "taskcluster-prod" + log_bucket: "gke-taskcluster-prod-log-bucket" + log_location: "global" + log_view: "_AllLogs" + root_url: "https://firefox-ci-tc.services.mozilla.com" +` + if err := os.WriteFile(cfgPath, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := LoadFrom(cfgPath) + if err != nil { + t.Fatalf("LoadFrom: %v", err) + } + env := cfg.Environments["fx-ci-scoped"] + want := "projects/moz-fx-taskcluster-prod/locations/global/buckets/gke-taskcluster-prod-log-bucket/views/_AllLogs" + if got := env.LogViewResource(); got != want { + t.Errorf("LogViewResource() = %q, want %q", got, want) + } +} + func TestExpandHome(t *testing.T) { home, _ := os.UserHomeDir() tests := []struct { diff --git a/internal/gcp/client.go b/internal/gcp/client.go index 24955a5..8a6ccfa 100644 --- a/internal/gcp/client.go +++ b/internal/gcp/client.go @@ -79,9 +79,11 @@ func AuthModeLabel(auth AuthConfig) string { } // Query executes a GCP Cloud Logging filter query and returns up to limit -// entries, newest first (reversed before display). -func (c *Client) Query(ctx context.Context, filter string, limit int) (*QueryResult, error) { - it := c.adminClient.Entries(ctx, logadmin.Filter(filter), logadmin.NewestFirst()) +// entries, newest first (reversed before display). When resourceName is +// non-empty, the query is scoped to that resource (e.g. a log view) instead of +// the client's project; an empty resourceName preserves project-scope behavior. +func (c *Client) Query(ctx context.Context, filter, resourceName string, limit int) (*QueryResult, error) { + it := c.adminClient.Entries(ctx, queryOptions(filter, resourceName)...) result := &QueryResult{} for i := 0; i < limit; i++ { @@ -100,6 +102,17 @@ func (c *Client) Query(ctx context.Context, filter string, limit int) (*QueryRes return result, nil } +// queryOptions builds the logadmin EntriesOptions for a query. When +// resourceName is non-empty, a ResourceNames option scopes the query to that +// resource (e.g. a log view), overriding the client's default project scope. +func queryOptions(filter, resourceName string) []logadmin.EntriesOption { + opts := []logadmin.EntriesOption{logadmin.Filter(filter), logadmin.NewestFirst()} + if resourceName != "" { + opts = append(opts, logadmin.ResourceNames([]string{resourceName})) + } + return opts +} + // Close closes the underlying logadmin client. func (c *Client) Close() error { return c.adminClient.Close() diff --git a/internal/gcp/client_test.go b/internal/gcp/client_test.go index fbba070..197d4ac 100644 --- a/internal/gcp/client_test.go +++ b/internal/gcp/client_test.go @@ -304,6 +304,18 @@ func TestAuthModeLabel(t *testing.T) { } } +func TestQueryOptions(t *testing.T) { + // Project scope: filter + newestFirst only, no ResourceNames. + if got := len(queryOptions("jsonPayload.Type=\"x\"", "")); got != 2 { + t.Errorf("queryOptions with empty resourceName: got %d options, want 2", got) + } + // View scope: adds a ResourceNames option. + rn := "projects/p/locations/global/buckets/b/views/_AllLogs" + if got := len(queryOptions("jsonPayload.Type=\"x\"", rn)); got != 3 { + t.Errorf("queryOptions with resourceName: got %d options, want 3", got) + } +} + func TestExtractFieldsWithPaths_FallbackToStandard(t *testing.T) { raw := map[string]interface{}{ "timestamp": "2026-03-10T12:00:00Z",