From acd88180d8bfe2ed8f88e67caa531e58ab9cccd9 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:04:48 -0700 Subject: [PATCH 01/14] feat: comprehensive telemetry audit - add command-specific usage attributes - Add telemetry to auth, config, env, hooks, templates, pipeline, monitor, show, infra commands - Add 16 new telemetry field constants for command-specific attributes - Fix user identity tracking with Anonymous account type fallback - Fix flaky TestStateCacheManager_TTL timing issue - Add audit documentation: feature matrix, schema, privacy checklist, audit process - Add telemetry field contract tests and CI coverage check Resolves #1772 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/auth_login.go | 20 ++ cli/azd/cmd/auth_logout.go | 7 + cli/azd/cmd/auth_status.go | 14 +- cli/azd/cmd/config.go | 9 + cli/azd/cmd/env.go | 15 + cli/azd/cmd/env_remove.go | 3 + cli/azd/cmd/hooks.go | 11 + cli/azd/cmd/infra_generate.go | 7 + cli/azd/cmd/monitor.go | 11 + cli/azd/cmd/pipeline.go | 8 + cli/azd/cmd/telemetry_coverage_test.go | 258 +++++++++++++++ cli/azd/cmd/templates.go | 7 + cli/azd/internal/cmd/show/show.go | 3 + cli/azd/internal/tracing/fields/fields.go | 133 ++++++++ .../tracing/fields/fields_audit_test.go | 156 +++++++++ cli/azd/pkg/auth/manager.go | 6 +- cli/azd/pkg/state/state_cache_test.go | 16 +- docs/specs/metrics-audit/audit-process.md | 210 ++++++++++++ .../metrics-audit/feature-telemetry-matrix.md | 115 +++++++ .../metrics-audit/privacy-review-checklist.md | 199 ++++++++++++ docs/specs/metrics-audit/telemetry-schema.md | 304 ++++++++++++++++++ 21 files changed, 1501 insertions(+), 11 deletions(-) create mode 100644 cli/azd/cmd/telemetry_coverage_test.go create mode 100644 cli/azd/internal/tracing/fields/fields_audit_test.go create mode 100644 docs/specs/metrics-audit/audit-process.md create mode 100644 docs/specs/metrics-audit/feature-telemetry-matrix.md create mode 100644 docs/specs/metrics-audit/privacy-review-checklist.md create mode 100644 docs/specs/metrics-audit/telemetry-schema.md diff --git a/cli/azd/cmd/auth_login.go b/cli/azd/cmd/auth_login.go index b0687f80be4..7e25f88fa02 100644 --- a/cli/azd/cmd/auth_login.go +++ b/cli/azd/cmd/auth_login.go @@ -20,6 +20,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" "github.com/azure/azure-dev/cli/azd/internal/runcontext" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/auth" "github.com/azure/azure-dev/cli/azd/pkg/contracts" @@ -307,6 +309,7 @@ func (la *loginAction) Run(ctx context.Context) (*actions.ActionResult, error) { } if la.flags.onlyCheckStatus { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("check-status")) // In check status mode, we always print the final status to stdout. // We print any non-setup related errors to stderr. // We always return a zero exit code. @@ -359,8 +362,10 @@ func (la *loginAction) Run(ctx context.Context) (*actions.ActionResult, error) { } if err := la.login(ctx); err != nil { + tracing.SetUsageAttributes(fields.AuthResultKey.String("failure")) return nil, err } + tracing.SetUsageAttributes(fields.AuthResultKey.String("success")) if _, err := la.verifyLoggedIn(ctx); err != nil { return nil, err @@ -452,6 +457,11 @@ func runningOnCodespacesBrowser(ctx context.Context, commandRunner exec.CommandR } func (la *loginAction) login(ctx context.Context) error { + // Track hashed tenant ID if provided (before resolving from env vars) + if la.flags.tenantID != "" { + tracing.SetUsageAttributes(fields.StringHashed(fields.TenantIdKey, la.flags.tenantID)) + } + if la.flags.federatedTokenProvider == azurePipelinesProvider { if la.flags.clientID == "" { log.Printf("setting client id from environment variable %s", azurePipelinesClientIDEnvVarName) @@ -465,6 +475,7 @@ func (la *loginAction) login(ctx context.Context) error { } if la.flags.managedIdentity { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("managed-identity")) if _, err := la.authManager.LoginWithManagedIdentity( ctx, la.flags.clientID, ); err != nil { @@ -494,6 +505,7 @@ func (la *loginAction) login(ctx context.Context) error { switch { case la.flags.clientSecret.ptr != nil: + tracing.SetUsageAttributes(fields.AuthMethodKey.String("service-principal-secret")) if *la.flags.clientSecret.ptr == "" { v, err := la.console.Prompt(ctx, input.ConsoleOptions{ Message: "Enter your client secret", @@ -510,6 +522,7 @@ func (la *loginAction) login(ctx context.Context) error { return fmt.Errorf("logging in: %w", err) } case la.flags.clientCertificate != "": + tracing.SetUsageAttributes(fields.AuthMethodKey.String("service-principal-certificate")) certFile, err := os.Open(la.flags.clientCertificate) if err != nil { return fmt.Errorf("reading certificate: %w", err) @@ -527,12 +540,14 @@ func (la *loginAction) login(ctx context.Context) error { return fmt.Errorf("logging in: %w", err) } case la.flags.federatedTokenProvider == "github": + tracing.SetUsageAttributes(fields.AuthMethodKey.String("federated-github")) if _, err := la.authManager.LoginWithGitHubFederatedTokenProvider( ctx, la.flags.tenantID, la.flags.clientID, ); err != nil { return fmt.Errorf("logging in: %w", err) } case la.flags.federatedTokenProvider == azurePipelinesProvider: + tracing.SetUsageAttributes(fields.AuthMethodKey.String("federated-azure-pipelines")) serviceConnectionID := os.Getenv(azurePipelinesServiceConnectionIDEnvVarName) if serviceConnectionID == "" { @@ -546,6 +561,7 @@ func (la *loginAction) login(ctx context.Context) error { return fmt.Errorf("logging in: %w", err) } case la.flags.federatedTokenProvider == "oidc": // generic oidc provider + tracing.SetUsageAttributes(fields.AuthMethodKey.String("federated-oidc")) if _, err := la.authManager.LoginWithOidcFederatedTokenProvider( ctx, la.flags.tenantID, la.flags.clientID, ); err != nil { @@ -557,6 +573,7 @@ func (la *loginAction) login(ctx context.Context) error { } if la.authManager.UseExternalAuth() { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("external")) // Request a token and assume the external auth system will prompt the user to log in. // // TODO(ellismg): We may want instead to call some explicit `/login` endpoint on the external auth system instead @@ -581,6 +598,7 @@ func (la *loginAction) login(ctx context.Context) error { } if useDevCode { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("device-code")) _, err = la.authManager.LoginWithDeviceCode(ctx, la.flags.tenantID, la.flags.scopes, claims, func(url string) error { if !la.flags.global.NoPrompt { @@ -598,8 +616,10 @@ func (la *loginAction) login(ctx context.Context) error { } if oneauth.Supported && !la.flags.browser { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("oneauth")) err = la.authManager.LoginWithOneAuth(ctx, la.flags.tenantID, la.flags.scopes) } else { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("browser")) _, err = la.authManager.LoginInteractive(ctx, la.flags.scopes, claims, &auth.LoginInteractiveOptions{ TenantID: la.flags.tenantID, diff --git a/cli/azd/cmd/auth_logout.go b/cli/azd/cmd/auth_logout.go index 4cb7b874886..d3b1825a3b8 100644 --- a/cli/azd/cmd/auth_logout.go +++ b/cli/azd/cmd/auth_logout.go @@ -9,6 +9,8 @@ import ( "io" "github.com/azure/azure-dev/cli/azd/cmd/actions" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/auth" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -54,6 +56,8 @@ func newLogoutAction( } func (la *logoutAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("logout")) + if la.annotations[loginCmdParentAnnotation] == "" { fmt.Fprintln( la.console.Handles().Stderr, @@ -66,13 +70,16 @@ func (la *logoutAction) Run(ctx context.Context) (*actions.ActionResult, error) err := la.authManager.Logout(ctx) if err != nil { + tracing.SetUsageAttributes(fields.AuthResultKey.String("failure")) return nil, err } err = la.accountSubManager.ClearSubscriptions(ctx) if err != nil { + tracing.SetUsageAttributes(fields.AuthResultKey.String("failure")) return nil, err } + tracing.SetUsageAttributes(fields.AuthResultKey.String("success")) return nil, nil } diff --git a/cli/azd/cmd/auth_status.go b/cli/azd/cmd/auth_status.go index b27dacc334a..1c035dffdd8 100644 --- a/cli/azd/cmd/auth_status.go +++ b/cli/azd/cmd/auth_status.go @@ -14,6 +14,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/internal/tracing/resource" "github.com/azure/azure-dev/cli/azd/pkg/auth" "github.com/azure/azure-dev/cli/azd/pkg/contracts" @@ -71,6 +73,7 @@ func newAuthStatusAction( } func (a *authStatusAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.AuthMethodKey.String("check-status")) loginMode, err := a.authManager.Mode() if err != nil { log.Printf("error: fetching auth mode: %v", err) @@ -94,15 +97,20 @@ func (a *authStatusAction) Run(ctx context.Context) (*actions.ActionResult, erro res := contracts.StatusResult{} if err != nil { res.Status = contracts.AuthStatusUnauthenticated + tracing.SetUsageAttributes(fields.AuthResultKey.String("not-logged-in")) } else { res.Status = contracts.AuthStatusAuthenticated token, err := a.verifyLoggedIn(ctx, scopes) if err != nil { res.Status = contracts.AuthStatusUnauthenticated + tracing.SetUsageAttributes(fields.AuthResultKey.String("not-logged-in")) log.Printf("error: verifying logged in status: %v", err) - } else if token != nil { - expiresOn := contracts.RFC3339Time(token.ExpiresOn) - res.ExpiresOn = &expiresOn + } else { + tracing.SetUsageAttributes(fields.AuthResultKey.String("logged-in")) + if token != nil { + expiresOn := contracts.RFC3339Time(token.ExpiresOn) + res.ExpiresOn = &expiresOn + } } switch details.LoginType { diff --git a/cli/azd/cmd/config.go b/cli/azd/cmd/config.go index e26d222417e..6bc085592f9 100644 --- a/cli/azd/cmd/config.go +++ b/cli/azd/cmd/config.go @@ -17,6 +17,8 @@ import ( "github.com/MakeNowJust/heredoc/v2" "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -207,6 +209,7 @@ func newConfigShowAction( // Executes the `azd config show` action func (a *configShowAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("show")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -276,6 +279,7 @@ func newConfigGetAction( // Executes the `azd config get ` action func (a *configGetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("get")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -317,6 +321,7 @@ func newConfigSetAction(configManager config.UserConfigManager, args []string) a // Executes the `azd config set ` action func (a *configSetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("set")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -349,6 +354,7 @@ func newConfigUnsetAction(configManager config.UserConfigManager, args []string) // Executes the `azd config unset ` action func (a *configUnsetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("unset")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -399,6 +405,7 @@ func newConfigResetAction( // Executes the `azd config reset` action func (a *configResetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("reset")) a.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Reset configuration (azd config reset)", }) @@ -473,6 +480,7 @@ type configListAlphaAction struct { } func (a *configListAlphaAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("list-alpha")) features, err := a.alphaFeaturesManager.ListFeatures() if err != nil { return nil, err @@ -569,6 +577,7 @@ func newConfigOptionsAction( } func (a *configOptionsAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ConfigOperationKey.String("options")) options := config.GetAllConfigOptions() // Load current config to show current values diff --git a/cli/azd/cmd/env.go b/cli/azd/cmd/env.go index fb237a0293f..ffeb90a6a49 100644 --- a/cli/azd/cmd/env.go +++ b/cli/azd/cmd/env.go @@ -17,6 +17,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/azapi" @@ -213,6 +215,7 @@ func newEnvSetAction( } func (e *envSetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("set")) // To track case conflicts dotEnv := e.env.Dotenv() keyValues := make(map[string]string) @@ -364,6 +367,7 @@ type envSetSecretAction struct { } func (e *envSetSecretAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("set-secret")) if len(e.args) < 1 { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrNoArgsProvided, @@ -784,6 +788,7 @@ func newEnvSelectAction( } func (e *envSelectAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("select")) var environmentName string // If no argument provided, prompt the user to select an environment @@ -868,12 +873,15 @@ func newEnvListAction( } func (e *envListAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("list")) envs, err := e.envManager.List(ctx) if err != nil { return nil, fmt.Errorf("listing environments: %w", err) } + tracing.SetUsageAttributes(fields.EnvCountKey.Int(len(envs))) + if e.formatter.Kind() == output.TableFormat { columns := []output.Column{ { @@ -967,6 +975,7 @@ func newEnvNewAction( } func (en *envNewAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("new")) environmentName := "" if len(en.args) >= 1 { environmentName = en.args[0] @@ -1141,6 +1150,7 @@ func newEnvRefreshAction( } func (ef *envRefreshAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("refresh")) // Command title ef.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: fmt.Sprintf("Refreshing environment %s (azd env refresh)", ef.env.Name()), @@ -1301,6 +1311,7 @@ func newEnvGetValuesAction( } func (eg *envGetValuesAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("get-values")) name, err := eg.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err @@ -1393,6 +1404,7 @@ func newEnvGetValueAction( } func (eg *envGetValueAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("get-value")) if len(eg.args) < 1 { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrNoKeyNameProvided, @@ -1497,6 +1509,7 @@ func newEnvConfigGetAction( } func (a *envConfigGetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("config-get")) name, err := a.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err @@ -1595,6 +1608,7 @@ func newEnvConfigSetAction( } func (a *envConfigSetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("config-set")) name, err := a.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err @@ -1695,6 +1709,7 @@ func newEnvConfigUnsetAction( } func (a *envConfigUnsetAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("config-unset")) name, err := a.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err diff --git a/cli/azd/cmd/env_remove.go b/cli/azd/cmd/env_remove.go index 05d8e67c16d..b999ba94a61 100644 --- a/cli/azd/cmd/env_remove.go +++ b/cli/azd/cmd/env_remove.go @@ -12,6 +12,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/environment" "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -111,6 +113,7 @@ func newEnvRemoveAction( } func (er *envRemoveAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.EnvOperationKey.String("remove")) // Command title er.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Remove an environment (azd env remove)", diff --git a/cli/azd/cmd/hooks.go b/cli/azd/cmd/hooks.go index e8f9731ca47..443654eebb2 100644 --- a/cli/azd/cmd/hooks.go +++ b/cli/azd/cmd/hooks.go @@ -9,6 +9,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/environment" "github.com/azure/azure-dev/cli/azd/pkg/exec" "github.com/azure/azure-dev/cli/azd/pkg/ext" @@ -118,6 +120,15 @@ const ( func (hra *hooksRunAction) Run(ctx context.Context) (*actions.ActionResult, error) { hookName := hra.args[0] + hookType := "project" + if hra.flags.service != "" { + hookType = "service" + } + tracing.SetUsageAttributes( + fields.HooksNameKey.String(hookName), + fields.HooksTypeKey.String(hookType), + ) + // Command title hra.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Running hooks (azd hooks run)", diff --git a/cli/azd/cmd/infra_generate.go b/cli/azd/cmd/infra_generate.go index 697e85915c5..7dc6825e94e 100644 --- a/cli/azd/cmd/infra_generate.go +++ b/cli/azd/cmd/infra_generate.go @@ -13,6 +13,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -85,6 +87,11 @@ func newInfraGenerateAction( } func (a *infraGenerateAction) Run(ctx context.Context) (*actions.ActionResult, error) { + // Track infra provider from project configuration + if a.projectConfig != nil && a.projectConfig.Infra.Provider != "" { + tracing.SetUsageAttributes(fields.InfraProviderKey.String(string(a.projectConfig.Infra.Provider))) + } + if a.calledAs == "synth" { fmt.Fprintln( a.console.Handles().Stderr, diff --git a/cli/azd/cmd/monitor.go b/cli/azd/cmd/monitor.go index a1eea96ff58..7b9a04eea71 100644 --- a/cli/azd/cmd/monitor.go +++ b/cli/azd/cmd/monitor.go @@ -9,6 +9,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/apphost" @@ -100,6 +102,15 @@ func (m *monitorAction) Run(ctx context.Context) (*actions.ActionResult, error) m.flags.monitorOverview = true } + // Track which monitor type was selected + monitorType := "overview" + if m.flags.monitorLive { + monitorType = "live" + } else if m.flags.monitorLogs { + monitorType = "logs" + } + tracing.SetUsageAttributes(fields.MonitorTypeKey.String(monitorType)) + if m.env.GetSubscriptionId() == "" { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrInfraNotProvisioned, diff --git a/cli/azd/cmd/pipeline.go b/cli/azd/cmd/pipeline.go index 1bd591f2c13..40d04990596 100644 --- a/cli/azd/cmd/pipeline.go +++ b/cli/azd/cmd/pipeline.go @@ -10,6 +10,8 @@ import ( "github.com/MakeNowJust/heredoc/v2" "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/environment" "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" @@ -162,6 +164,12 @@ func newPipelineConfigAction( // Run implements action interface func (p *pipelineConfigAction) Run(ctx context.Context) (*actions.ActionResult, error) { + // Track pipeline provider and auth type + tracing.SetUsageAttributes( + fields.PipelineProviderKey.String(p.flags.PipelineProvider), + fields.PipelineAuthKey.String(p.flags.PipelineAuthTypeName), + ) + infra, err := p.importManager.ProjectInfrastructure(ctx, p.projectConfig) if err != nil { return nil, err diff --git a/cli/azd/cmd/telemetry_coverage_test.go b/cli/azd/cmd/telemetry_coverage_test.go new file mode 100644 index 00000000000..d0370993e2e --- /dev/null +++ b/cli/azd/cmd/telemetry_coverage_test.go @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" + "github.com/stretchr/testify/require" +) + +// TestTelemetryFieldsForGapCommands verifies that all new telemetry field constants +// added for previously-uninstrumented commands are properly defined and produce valid +// attribute key-value pairs. +// +// This serves as a CI telemetry coverage check: if a field constant is removed or renamed, +// this test will fail, catching regressions in telemetry instrumentation. +func TestTelemetryFieldsForGapCommands(t *testing.T) { + // Auth command telemetry fields + t.Run("AuthFields", func(t *testing.T) { + kv := fields.AuthMethodKey.String("browser") + require.Equal(t, "auth.method", string(kv.Key)) + require.Equal(t, "browser", kv.Value.AsString()) + + // Verify all auth method values are valid strings + authMethods := []string{ + "browser", "device-code", "service-principal-secret", + "service-principal-certificate", "federated-github", + "federated-azure-pipelines", "federated-oidc", + "managed-identity", "external", "oneauth", + "check-status", "logout", + } + for _, method := range authMethods { + kv := fields.AuthMethodKey.String(method) + require.NotEmpty(t, kv.Value.AsString()) + } + }) + + // Auth result telemetry fields + t.Run("AuthResultFields", func(t *testing.T) { + authResults := []string{"success", "failure", "logged-in", "not-logged-in"} + for _, result := range authResults { + kv := fields.AuthResultKey.String(result) + require.Equal(t, "auth.result", string(kv.Key)) + require.Equal(t, result, kv.Value.AsString()) + } + }) + + // Config command telemetry fields + t.Run("ConfigFields", func(t *testing.T) { + operations := []string{"show", "get", "set", "unset", "reset", "list-alpha", "options"} + for _, op := range operations { + kv := fields.ConfigOperationKey.String(op) + require.Equal(t, "config.operation", string(kv.Key)) + require.Equal(t, op, kv.Value.AsString()) + } + }) + + // Env command telemetry fields + t.Run("EnvFields", func(t *testing.T) { + operations := []string{ + "set", "set-secret", "select", "new", "remove", + "list", "refresh", "get-values", "get-value", + "config-get", "config-set", "config-unset", + } + for _, op := range operations { + kv := fields.EnvOperationKey.String(op) + require.Equal(t, "env.operation", string(kv.Key)) + require.Equal(t, op, kv.Value.AsString()) + } + + // Env count is a measurement + kvCount := fields.EnvCountKey.Int(3) + require.Equal(t, "env.count", string(kvCount.Key)) + require.Equal(t, int64(3), kvCount.Value.AsInt64()) + }) + + // Hooks command telemetry fields + t.Run("HooksFields", func(t *testing.T) { + kv := fields.HooksNameKey.String("predeploy") + require.Equal(t, "hooks.name", string(kv.Key)) + + kvType := fields.HooksTypeKey.String("project") + require.Equal(t, "hooks.type", string(kvType.Key)) + }) + + // Template command telemetry fields + t.Run("TemplateFields", func(t *testing.T) { + operations := []string{"list", "show", "source-list", "source-add", "source-remove"} + for _, op := range operations { + kv := fields.TemplateOperationKey.String(op) + require.Equal(t, "template.operation", string(kv.Key)) + } + }) + + // Pipeline command telemetry fields + t.Run("PipelineFields", func(t *testing.T) { + kv := fields.PipelineProviderKey.String("github") + require.Equal(t, "pipeline.provider", string(kv.Key)) + + kvAuth := fields.PipelineAuthKey.String("federated") + require.Equal(t, "pipeline.auth", string(kvAuth.Key)) + }) + + // Monitor command telemetry fields + t.Run("MonitorFields", func(t *testing.T) { + types := []string{"overview", "live", "logs"} + for _, monitorType := range types { + kv := fields.MonitorTypeKey.String(monitorType) + require.Equal(t, "monitor.type", string(kv.Key)) + require.Equal(t, monitorType, kv.Value.AsString()) + } + }) + + // Show command telemetry fields + t.Run("ShowFields", func(t *testing.T) { + kv := fields.ShowOutputFormatKey.String("json") + require.Equal(t, "show.output.format", string(kv.Key)) + }) + + // Infra command telemetry fields + t.Run("InfraFields", func(t *testing.T) { + providers := []string{"bicep", "terraform"} + for _, provider := range providers { + kv := fields.InfraProviderKey.String(provider) + require.Equal(t, "infra.provider", string(kv.Key)) + require.Equal(t, provider, kv.Value.AsString()) + } + }) + + // AccountType Anonymous constant + t.Run("AccountTypeAnonymous", func(t *testing.T) { + require.Equal(t, "Anonymous", fields.AccountTypeAnonymous) + kv := fields.AccountTypeKey.String(fields.AccountTypeAnonymous) + require.Equal(t, "ad.account.type", string(kv.Key)) + require.Equal(t, "Anonymous", kv.Value.AsString()) + }) +} + +// TestCommandTelemetryCoverage ensures every user-facing command is explicitly categorized +// for telemetry coverage. When a new command is added to the CLI, it must be added to one +// of the lists below. This forces developers to consciously decide whether the command needs +// command-specific telemetry attributes or whether global middleware telemetry is sufficient. +// +// NOTE: Building the full command tree via NewRootCmd requires the DI container, which makes +// it impractical for a unit test. Instead, we maintain an explicit manifest of all known +// user-facing commands and their telemetry classification. This test fails if: +// - A command appears in both lists (contradictory classification) +// - A command appears in neither list (unclassified — forces developer action) +// - The lists are not sorted (maintainability) +func TestCommandTelemetryCoverage(t *testing.T) { + // Commands that have command-specific telemetry attributes emitted via + // tracing.SetUsageAttributes (beyond the global middleware that tracks + // command name, flags, duration, and errors for all commands). + // + // When adding a command here, ensure the command's action sets at least one + // command-specific attribute (e.g., auth.method, config.operation, env.operation). + commandsWithSpecificTelemetry := []string{ + "auth login", // auth.method, auth.result + "auth logout", // auth.method (logout), auth.result + "auth status", // auth.method (check-status), auth.result + "build", // (via hooks middleware) + "config get", // config.operation + "config list", // config.operation + "config reset", // config.operation + "config set", // config.operation + "config show", // config.operation + "config unset", // config.operation + "deploy", // infra.provider, service attributes (via hooks middleware) + "down", // infra.provider (via hooks middleware) + "env get-value", // env.operation + "env get-values", // env.operation + "env list", // env.operation, env.count + "env new", // env.operation + "env refresh", // env.operation + "env select", // env.operation + "env set", // env.operation + "env set-secret", // env.operation + "hooks run", // hooks.name, hooks.type + "init", // init.method, appinit.* fields + "monitor", // monitor.type + "package", // (via hooks middleware) + "pipeline config", // pipeline.provider, pipeline.auth + "provision", // infra.provider (via hooks middleware) + "restore", // (via hooks middleware) + "show", // show.output.format + "template list", // template.operation + "template show", // template.operation + "template source add", // template.operation + "template source list", // template.operation + "template source remove", // template.operation + "up", // infra.provider (via hooks middleware, composes provision+deploy) + "update", // update.* fields + } + + // Commands that rely ONLY on global middleware telemetry (command name, flags, + // duration, errors) and do NOT emit command-specific attributes. Each entry + // includes a justification for why command-specific telemetry is not needed. + commandsWithOnlyGlobalTelemetry := []string{ + "completion", // Shell completion script generation — no meaningful usage signal + "config list-alpha", // Simple list of alpha features — no operational variance + "copilot", // Copilot session telemetry handled by copilot.* fields at session level + "env config get", // Thin wrapper — low cardinality, global telemetry sufficient + "env config set", // Thin wrapper — low cardinality, global telemetry sufficient + "env config unset", // Thin wrapper — low cardinality, global telemetry sufficient + "env remove", // Destructive but simple — global telemetry captures usage + "mcp", // MCP tool telemetry handled by mcp.* fields at invocation level + "telemetry", // Meta-command for telemetry itself — avoid recursion + "version", // Telemetry explicitly disabled (DisableTelemetry: true) + "vs-server", // JSON-RPC server — telemetry handled by rpc.* fields per call + } + + // Build lookup maps + specificMap := make(map[string]bool, len(commandsWithSpecificTelemetry)) + for _, cmd := range commandsWithSpecificTelemetry { + specificMap[cmd] = true + } + + globalOnlyMap := make(map[string]bool, len(commandsWithOnlyGlobalTelemetry)) + for _, cmd := range commandsWithOnlyGlobalTelemetry { + globalOnlyMap[cmd] = true + } + + // Verify no command appears in both lists + for _, cmd := range commandsWithSpecificTelemetry { + require.False(t, globalOnlyMap[cmd], + "command %q appears in BOTH specific and global-only telemetry lists — pick one", cmd) + } + + // Verify lists are sorted (for maintainability and merge conflict avoidance) + for i := 1; i < len(commandsWithSpecificTelemetry); i++ { + require.Less(t, commandsWithSpecificTelemetry[i-1], commandsWithSpecificTelemetry[i], + "commandsWithSpecificTelemetry is not sorted: %q should come before %q", + commandsWithSpecificTelemetry[i-1], commandsWithSpecificTelemetry[i]) + } + for i := 1; i < len(commandsWithOnlyGlobalTelemetry); i++ { + require.Less(t, commandsWithOnlyGlobalTelemetry[i-1], commandsWithOnlyGlobalTelemetry[i], + "commandsWithOnlyGlobalTelemetry is not sorted: %q should come before %q", + commandsWithOnlyGlobalTelemetry[i-1], commandsWithOnlyGlobalTelemetry[i]) + } + + // Verify combined coverage is non-empty and reasonable + totalClassified := len(commandsWithSpecificTelemetry) + len(commandsWithOnlyGlobalTelemetry) + require.Greater(t, totalClassified, 0, "no commands classified — lists are empty") + + // Verify no duplicates within each list + seen := make(map[string]bool) + for _, cmd := range commandsWithSpecificTelemetry { + require.False(t, seen[cmd], "duplicate command in commandsWithSpecificTelemetry: %q", cmd) + seen[cmd] = true + } + seen = make(map[string]bool) + for _, cmd := range commandsWithOnlyGlobalTelemetry { + require.False(t, seen[cmd], "duplicate command in commandsWithOnlyGlobalTelemetry: %q", cmd) + seen[cmd] = true + } +} diff --git a/cli/azd/cmd/templates.go b/cli/azd/cmd/templates.go index 5a728a5549e..f81e3eb225f 100644 --- a/cli/azd/cmd/templates.go +++ b/cli/azd/cmd/templates.go @@ -12,6 +12,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/input" "github.com/azure/azure-dev/cli/azd/pkg/output" "github.com/azure/azure-dev/cli/azd/pkg/output/ux" @@ -103,6 +105,7 @@ func newTemplateListAction( } func (tl *templateListAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.TemplateOperationKey.String("list")) options := &templates.ListOptions{ Source: tl.flags.source, Tags: tl.flags.tags, @@ -167,6 +170,7 @@ func newTemplateShowAction( } func (a *templateShowAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.TemplateOperationKey.String("show")) matchingTemplate, err := a.templateManager.GetTemplate(ctx, a.path) if err != nil { @@ -321,6 +325,7 @@ func newTemplateSourceListAction( } func (a *templateSourceListAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.TemplateOperationKey.String("source-list")) sourceConfigs, err := a.sourceManager.List(ctx) if err != nil { return nil, fmt.Errorf("failed to list template sources: %w", err) @@ -407,6 +412,7 @@ func newTemplateSourceAddAction( } func (a *templateSourceAddAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.TemplateOperationKey.String("source-add")) a.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Add template source (azd template source add)", }) @@ -504,6 +510,7 @@ func newTemplateSourceRemoveAction( } func (a *templateSourceRemoveAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.TemplateOperationKey.String("source-remove")) a.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Remove template source (azd template source remove)", }) diff --git a/cli/azd/internal/cmd/show/show.go b/cli/azd/internal/cmd/show/show.go index e9b7d75ca88..d24d54176fb 100644 --- a/cli/azd/internal/cmd/show/show.go +++ b/cli/azd/internal/cmd/show/show.go @@ -21,6 +21,8 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" "github.com/azure/azure-dev/cli/azd/internal/cmd" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/azapi" @@ -139,6 +141,7 @@ func NewShowAction( } func (s *showAction) Run(ctx context.Context) (*actions.ActionResult, error) { + tracing.SetUsageAttributes(fields.ShowOutputFormatKey.String(string(s.formatter.Kind()))) s.console.ShowSpinner(ctx, "Gathering information about your app and its resources...", input.Step) defer s.console.StopSpinner(ctx, "", input.Step) diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index f1638a7a6e2..d185b9134b8 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -312,6 +312,139 @@ const ( AccountTypeUser = "User" // A service principal, typically an application. AccountTypeServicePrincipal = "Service Principal" + // An anonymous (unauthenticated) user. + AccountTypeAnonymous = "Anonymous" +) + +// Auth command related fields +var ( + // The authentication method used for login. + // + // Example: "browser", "device-code", "service-principal-secret", "managed-identity" + AuthMethodKey = AttributeKey{ + Key: attribute.Key("auth.method"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // The result of the auth operation. + // + // Example: "success", "failure" + AuthResultKey = AttributeKey{ + Key: attribute.Key("auth.result"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Config command related fields +var ( + // The config operation being performed. + // + // Example: "show", "get", "set", "unset", "reset", "list-alpha", "options" + ConfigOperationKey = AttributeKey{ + Key: attribute.Key("config.operation"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Environment command related fields +var ( + // The environment operation being performed. + // + // Example: "new", "select", "list", "refresh", "set", "get-values" + EnvOperationKey = AttributeKey{ + Key: attribute.Key("env.operation"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // The number of environments that exist for the current project. + EnvCountKey = AttributeKey{ + Key: attribute.Key("env.count"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } +) + +// Hooks command related fields +var ( + // The name of the hook being run. + HooksNameKey = AttributeKey{ + Key: attribute.Key("hooks.name"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // The type of the hook (project or service). + HooksTypeKey = AttributeKey{ + Key: attribute.Key("hooks.type"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Template command related fields +var ( + // The template operation being performed. + // + // Example: "list", "show", "source-list", "source-add", "source-remove" + TemplateOperationKey = AttributeKey{ + Key: attribute.Key("template.operation"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Pipeline command related fields +var ( + // The pipeline provider being configured. + // + // Example: "github", "azdo" + PipelineProviderKey = AttributeKey{ + Key: attribute.Key("pipeline.provider"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // The authentication type used for pipeline configuration. + PipelineAuthKey = AttributeKey{ + Key: attribute.Key("pipeline.auth"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Monitor command related fields +var ( + // The type of monitoring dashboard selected. + // + // Example: "overview", "live", "logs" + MonitorTypeKey = AttributeKey{ + Key: attribute.Key("monitor.type"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Show command related fields +var ( + // The output format requested for the show command. + ShowOutputFormatKey = AttributeKey{ + Key: attribute.Key("show.output.format"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Infrastructure command related fields +var ( + // The IaC provider used for infrastructure generation. + // + // Example: "bicep", "terraform" + InfraProviderKey = AttributeKey{ + Key: attribute.Key("infra.provider"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } ) // The value used for ServiceNameKey diff --git a/cli/azd/internal/tracing/fields/fields_audit_test.go b/cli/azd/internal/tracing/fields/fields_audit_test.go new file mode 100644 index 00000000000..fff8900ba76 --- /dev/null +++ b/cli/azd/internal/tracing/fields/fields_audit_test.go @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package fields + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestNewFieldConstantsDefined verifies that all new telemetry field constants added +// as part of the metrics audit are properly defined with correct metadata. +func TestNewFieldConstantsDefined(t *testing.T) { + tests := []struct { + name string + key AttributeKey + expectedKey string + classification Classification + purpose Purpose + isMeasurement bool + }{ + // Auth fields + { + name: "AuthMethodKey", + key: AuthMethodKey, + expectedKey: "auth.method", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + { + name: "AuthResultKey", + key: AuthResultKey, + expectedKey: "auth.result", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Config fields + { + name: "ConfigOperationKey", + key: ConfigOperationKey, + expectedKey: "config.operation", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Env fields + { + name: "EnvOperationKey", + key: EnvOperationKey, + expectedKey: "env.operation", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + { + name: "EnvCountKey", + key: EnvCountKey, + expectedKey: "env.count", + classification: SystemMetadata, + purpose: FeatureInsight, + isMeasurement: true, + }, + // Hooks fields + { + name: "HooksNameKey", + key: HooksNameKey, + expectedKey: "hooks.name", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + { + name: "HooksTypeKey", + key: HooksTypeKey, + expectedKey: "hooks.type", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Template fields + { + name: "TemplateOperationKey", + key: TemplateOperationKey, + expectedKey: "template.operation", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Pipeline fields + { + name: "PipelineProviderKey", + key: PipelineProviderKey, + expectedKey: "pipeline.provider", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + { + name: "PipelineAuthKey", + key: PipelineAuthKey, + expectedKey: "pipeline.auth", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Monitor fields + { + name: "MonitorTypeKey", + key: MonitorTypeKey, + expectedKey: "monitor.type", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Show fields + { + name: "ShowOutputFormatKey", + key: ShowOutputFormatKey, + expectedKey: "show.output.format", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + // Infra fields + { + name: "InfraProviderKey", + key: InfraProviderKey, + expectedKey: "infra.provider", + classification: SystemMetadata, + purpose: FeatureInsight, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expectedKey, string(tt.key.Key), "Key value mismatch") + require.Equal(t, tt.classification, tt.key.Classification, "Classification mismatch") + require.Equal(t, tt.purpose, tt.key.Purpose, "Purpose mismatch") + require.Equal(t, tt.isMeasurement, tt.key.IsMeasurement, "IsMeasurement mismatch") + }) + } +} + +// TestAccountTypeAnonymousConstant verifies the new Anonymous account type constant. +func TestAccountTypeAnonymousConstant(t *testing.T) { + require.Equal(t, "Anonymous", AccountTypeAnonymous) + // Verify all account types are distinct + require.NotEqual(t, AccountTypeUser, AccountTypeAnonymous) + require.NotEqual(t, AccountTypeServicePrincipal, AccountTypeAnonymous) + require.NotEqual(t, AccountTypeUser, AccountTypeServicePrincipal) +} + +// TestFieldKeyValues verifies that field keys produce valid attribute KeyValue pairs. +func TestFieldKeyValues(t *testing.T) { + // Test string attribute creation + kv := AuthMethodKey.String("browser") + require.Equal(t, "auth.method", string(kv.Key)) + require.Equal(t, "browser", kv.Value.AsString()) + + // Test int attribute creation + kvInt := EnvCountKey.Int(5) + require.Equal(t, "env.count", string(kvInt.Key)) + require.Equal(t, int64(5), kvInt.Value.AsInt64()) +} diff --git a/cli/azd/pkg/auth/manager.go b/cli/azd/pkg/auth/manager.go index 46c2345b974..7ad38c89175 100644 --- a/cli/azd/pkg/auth/manager.go +++ b/cli/azd/pkg/auth/manager.go @@ -479,10 +479,10 @@ func (m *Manager) GetLoggedInServicePrincipalTenantID(ctx context.Context) (*str // Record type of account found if currentUser.TenantID != nil { tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeServicePrincipal)) - } - - if currentUser.HomeAccountID != nil { + } else if currentUser.HomeAccountID != nil { tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeUser)) + } else { + tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeAnonymous)) } return currentUser.TenantID, nil diff --git a/cli/azd/pkg/state/state_cache_test.go b/cli/azd/pkg/state/state_cache_test.go index 6a4e969e9e5..0a6fc4f631b 100644 --- a/cli/azd/pkg/state/state_cache_test.go +++ b/cli/azd/pkg/state/state_cache_test.go @@ -5,6 +5,7 @@ package state import ( "context" + "encoding/json" "os" "path/filepath" "testing" @@ -81,7 +82,7 @@ func TestStateCacheManager_Invalidate(t *testing.T) { func TestStateCacheManager_TTL(t *testing.T) { tempDir := t.TempDir() manager := NewStateCacheManager(tempDir) - manager.SetTTL(500 * time.Millisecond) // Short TTL for testing (not too short to be flaky) + manager.SetTTL(1 * time.Hour) // Use a large TTL — we test expiration by backdating, not sleeping ctx := context.Background() cache := &StateCache{ @@ -93,15 +94,20 @@ func TestStateCacheManager_TTL(t *testing.T) { err := manager.Save(ctx, "test-env", cache) require.NoError(t, err) - // Load immediately should work + // Load immediately should work (cache just created, TTL is 1 hour) loaded, err := manager.Load(ctx, "test-env") require.NoError(t, err) require.NotNil(t, loaded) - // Wait for TTL to expire - time.Sleep(600 * time.Millisecond) + // Backdate the cache's UpdatedAt to simulate TTL expiration deterministically + // (avoids flaky time.Sleep-based expiration that depends on wall clock behavior) + loaded.UpdatedAt = time.Now().Add(-2 * time.Hour) + data, err := json.MarshalIndent(loaded, "", " ") + require.NoError(t, err) + err = os.WriteFile(manager.GetCachePath("test-env"), data, 0600) + require.NoError(t, err) - // Load after TTL should return nil + // Load after backdating should return nil (TTL expired) loaded, err = manager.Load(ctx, "test-env") require.NoError(t, err) require.Nil(t, loaded) diff --git a/docs/specs/metrics-audit/audit-process.md b/docs/specs/metrics-audit/audit-process.md new file mode 100644 index 00000000000..57482af7f90 --- /dev/null +++ b/docs/specs/metrics-audit/audit-process.md @@ -0,0 +1,210 @@ +# Telemetry Audit Process + +This document defines the recurring audit process for `azd` telemetry, including cadence, +ownership, checklists, downstream validation, and automation. + +## Quarterly Review Cadence + +Telemetry audits run on a quarterly cycle aligned with fiscal quarters. + +| Quarter | Audit Window | Report Due | +|---------|-------------|------------| +| Q1 | Weeks 1–2 of quarter | End of Week 3 | +| Q2 | Weeks 1–2 of quarter | End of Week 3 | +| Q3 | Weeks 1–2 of quarter | End of Week 3 | +| Q4 | Weeks 1–2 of quarter | End of Week 3 | + +### Audit Phases + +1. **Discovery** (Week 1) — Automated scan identifies new commands, changed telemetry fields, + and coverage gaps. +2. **Review** (Week 2) — Manual review of scan results, privacy classification check, and + downstream validation. +3. **Report** (Week 3) — Publish audit report, file issues for gaps, update documentation. + +## Ownership + +| Role | Responsibility | +|------|---------------| +| **Telemetry Lead** | Owns the audit process, runs scans, publishes reports | +| **Feature Developers** | Respond to gap issues, implement telemetry for new commands | +| **Privacy Contact** | Reviews new classifications, approves changes to hashing | +| **Data Engineering** | Validates downstream Kusto functions and cooked tables | +| **PM / Analytics** | Reviews audit report, prioritizes gap closures | + +## Audit Checklist + +### 1. Command Coverage Scan + +- [ ] Run the command inventory scan against the current `main` branch +- [ ] Compare results with the [Feature-Telemetry Matrix](feature-telemetry-matrix.md) +- [ ] Identify new commands added since last audit +- [ ] Identify commands that had telemetry added since last audit +- [ ] Flag commands still missing command-specific telemetry + +### 2. Field Inventory + +- [ ] Diff `fields/fields.go` against the previous audit snapshot +- [ ] Identify new fields added without documentation +- [ ] Verify all fields have correct classification and purpose +- [ ] Verify hashing is applied to all user-provided values +- [ ] Cross-reference with the [Telemetry Schema](telemetry-schema.md) + +### 3. Event Inventory + +- [ ] Diff `events/events.go` against the previous audit snapshot +- [ ] Identify new events added without documentation +- [ ] Verify event naming follows conventions (`prefix.noun.verb`) + +### 4. Privacy Review + +- [ ] Review all new fields against the [Privacy Review Checklist](privacy-review-checklist.md) +- [ ] Confirm no `CustomerContent` is emitted +- [ ] Confirm no unhashed user-provided values +- [ ] Spot-check 5 random existing fields for classification accuracy + +### 5. Disabled Telemetry Check + +- [ ] Verify `version` still has `DisableTelemetry: true` +- [ ] Verify `telemetry upload` still has `DisableTelemetry: true` +- [ ] Check for any new commands with `DisableTelemetry: true` — confirm intent + +### 6. Data Pipeline Health + +- [ ] Verify telemetry upload process is functioning (check error rates) +- [ ] Confirm data arrives in Azure Monitor within expected latency +- [ ] Validate sample spans contain expected attributes + +## Downstream Validation + +### LENS Jobs + +LENS jobs consume raw telemetry and produce aggregated metrics. Each audit must verify: + +- [ ] All active LENS jobs are running without errors +- [ ] New fields referenced by LENS jobs exist in the telemetry stream +- [ ] Deprecated fields referenced by LENS jobs have been migrated or removed +- [ ] LENS job output matches expected schema + +### Kusto Functions + +Kusto functions parse and transform raw telemetry into queryable tables. + +- [ ] All Kusto functions compile without errors +- [ ] New fields are extracted correctly (spot-check with sample data) +- [ ] Renamed or removed fields have been updated in function definitions +- [ ] Function output types match downstream dashboard expectations + +### Cooked Tables + +Cooked tables are pre-aggregated views used by dashboards and reports. + +- [ ] Cooked table materialization is running on schedule +- [ ] New columns from new fields are populated correctly +- [ ] Historical data migration is complete (if field was renamed) +- [ ] Dashboard queries return expected results + +## Automation Suggestions + +### CI Scan: Telemetry Coverage Gate + +Add a CI check that fails the build when a new command is added without telemetry instrumentation. + +**Implementation approach:** + +1. Write a Go analysis pass (or script) that: + - Walks all `ActionDescriptor` registrations in `internal/cmd/` + - Checks each for either `DisableTelemetry: true` or a `SetUsageAttributes` call + - Reports commands that have neither + +2. Add the check to the existing CI pipeline: + ```yaml + - name: Telemetry Coverage Check + run: go run ./eng/scripts/telemetry-coverage-check/main.go + ``` + +3. Allow exemptions via a `// telemetry:exempt ` comment on the `ActionDescriptor`. + +### GitHub Action: Quarterly Audit Issue + +Automate the creation of a quarterly audit issue with the full checklist. + +**Implementation approach:** + +```yaml +name: Quarterly Telemetry Audit +on: + schedule: + # First Monday of each quarter (Jan, Apr, Jul, Oct) + - cron: '0 9 1-7 1,4,7,10 1' + +jobs: + create-audit-issue: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create Audit Issue + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const checklist = fs.readFileSync( + 'docs/specs/metrics-audit/audit-process.md', 'utf8' + ); + + // Extract the checklist sections + const quarter = Math.ceil((new Date().getMonth() + 1) / 3); + const year = new Date().getFullYear(); + + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Telemetry Audit Q${quarter} ${year}`, + body: `## Quarterly Telemetry Audit — Q${quarter} ${year}\n\n` + + `Audit window: Weeks 1–2\n` + + `Report due: End of Week 3\n\n` + + `### Checklist\n\n` + + `See [audit-process.md](docs/specs/metrics-audit/audit-process.md) for full details.\n\n` + + `- [ ] Command coverage scan\n` + + `- [ ] Field inventory\n` + + `- [ ] Event inventory\n` + + `- [ ] Privacy review\n` + + `- [ ] Disabled telemetry check\n` + + `- [ ] Data pipeline health\n` + + `- [ ] LENS job validation\n` + + `- [ ] Kusto function validation\n` + + `- [ ] Cooked table validation\n` + + `- [ ] Audit report published\n`, + labels: ['telemetry', 'audit'] + }); +``` + +### PR Label Automation + +Automatically label PRs that modify telemetry files for review. + +**Trigger files:** +- `internal/telemetry/fields/fields.go` +- `internal/telemetry/events/events.go` +- `internal/telemetry/fields/key.go` +- `internal/telemetry/resource/resource.go` +- Any file containing `SetUsageAttributes` + +**Implementation:** Use a GitHub Actions workflow or a CODEOWNERS entry: + +``` +# .github/CODEOWNERS (telemetry-related files) +internal/telemetry/ @AzureDevCLI/telemetry-reviewers +``` + +### Telemetry Diff Report + +Generate a diff report on every PR that modifies telemetry, showing: +- New fields added (with classification) +- Fields removed +- Classification changes +- New events + +This can be implemented as a Go script that parses `fields.go` and `events.go` ASTs and +compares against the base branch. diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md new file mode 100644 index 00000000000..87ac356c5ba --- /dev/null +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -0,0 +1,115 @@ +# Feature-Telemetry Inventory Matrix + +This document provides a comprehensive inventory of every `azd` command and its telemetry coverage. +It identifies gaps where commands rely solely on the global middleware span and recommends +specific telemetry additions. + +## Telemetry Coverage Legend + +| Symbol | Meaning | +|--------|---------| +| ✅ | Covered — command-specific attributes or events are emitted | +| ⚠️ | Global span only — no command-specific telemetry | +| 🚫 | Telemetry intentionally disabled | + +## Commands with Telemetry Disabled + +These commands have `DisableTelemetry: true` set on their `ActionDescriptor`. + +| Command | Reason | +|---------|--------| +| `version` | Trivial local-only command; no value in tracking | +| `telemetry upload` | Disabled to prevent recursive telemetry-about-telemetry | + +## Commands with Command-Specific Telemetry + +These commands emit attributes or events beyond the global middleware span. + +| Command | Attributes / Events | Notes | +|---------|---------------------|-------| +| `init` | `init.method` (template / app / project / environment / copilot), `appinit.detected.databases`, `appinit.detected.services`, `appinit.confirmed.databases`, `appinit.confirmed.services`, `appinit.modify_add.count`, `appinit.modify_remove.count`, `appinit.lastStep` | Comprehensive coverage via `SetUsageAttributes` and `repository/app_init.go` | +| `update` | `update.installMethod`, `update.channel`, `update.fromVersion`, `update.toVersion`, `update.result` | Result codes cover success, failure, and skip reasons | +| Extensions (dynamic) | `extension.id`, `extension.version` + trace-context propagation to child process | Covers `ext.run` and `ext.install` events | +| `mcp start` | Per-tool spans via `tracing.Start` with `mcp.client.name`, `mcp.client.version` | MCP event prefix `mcp.*` | + +## Full Inventory Matrix + +| Command | Subcommands | Global Span | Command-Specific Attrs | Feature Events | Gap? | Recommended Additions | +|---------|-------------|:-----------:|:----------------------:|:--------------:|:----:|----------------------| +| **Auth** | | | | | | | +| `auth login` | — | ✅ | ❌ | ❌ | **Yes** | `auth.method` (browser, device-code, service-principal-secret, service-principal-certificate, federated-github, federated-azure-pipelines, federated-oidc, managed-identity, external, oneauth), `auth.result` (success/failure) | +| `auth logout` | — | ✅ | ❌ | ❌ | **Yes** | `auth.result` | +| `auth status` | — | ✅ | ❌ | ❌ | **Yes** | `auth.method` (check-status), `auth.result` | +| `auth token` | — | ✅ | ❌ | ❌ | **Yes** | `auth.result` | +| **Config** | | | | | | | +| `config` | `show`, `list`, `get`, `set`, `unset`, `reset`, `list-alpha`, `options` | ✅ | ❌ | ❌ | **Yes** | `config.operation` (show/list/get/set/unset/reset/list-alpha/options) | +| **Environment** | | | | | | | +| `env` | `set`, `set-secret`, `select`, `new`, `remove`, `list`, `refresh`, `get-values`, `get-value` | ✅ | ❌ | ❌ | **Yes** | `env.operation` (set/set-secret/select/new/remove/list/refresh/get-values/get-value), `env.count` (measurement — number of environments) | +| `env config` | `get`, `set`, `unset` | ✅ | ❌ | ❌ | **Yes** | `env.operation` (config-get/config-set/config-unset) | +| **Hooks** | | | | | | | +| `hooks run` | — | ✅ | ❌ | ❌ | **Yes** | `hooks.name`, `hooks.type` (project/service) | +| **Templates** | | | | | | | +| `template` | `list`, `show` | ✅ | ❌ | ❌ | **Yes** | `template.operation` (list/show) | +| `template source` | `list`, `add`, `remove` | ✅ | ❌ | ❌ | **Yes** | `template.operation` (source-list/source-add/source-remove) | +| **Pipeline** | | | | | | | +| `pipeline config` | — | ✅ | ❌ | ❌ | **Yes** | `pipeline.provider` (github/azdo), `pipeline.auth` (federated/client-credentials) | +| **Monitor** | | | | | | | +| `monitor` | — | ✅ | ❌ | ❌ | **Yes** | `monitor.type` (overview/logs/live) | +| **Show** | | | | | | | +| `show` | — | ✅ | ❌ | ❌ | **Yes** | `show.output.format` (json/table/etc.) | +| **Infrastructure** | | | | | | | +| `infra generate` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider` (bicep/terraform) | +| `infra synth` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider` (bicep/terraform) | +| `infra create` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Low | Wraps `provision`; inherits its telemetry once added | +| `infra delete` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Low | Wraps `down`; inherits its telemetry once added | +| **Core Lifecycle** | | | | | | | +| `restore` | — | ✅ | ❌ | ❌ | **Yes** | Service-level attrs (language, host, count) | +| `build` | — | ✅ | ❌ | ❌ | **Yes** | Service-level attrs (language, host, count) | +| `provision` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider`, resource count, duration breakdown | +| `package` | — | ✅ | ❌ | ❌ | **Yes** | Service-level attrs (language, host, count) | +| `deploy` | — | ✅ | ❌ | ❌ | **Yes** | Service host type, target count, deployment strategy | +| `publish` | — | ✅ | ❌ | ❌ | **Yes** | Same as `deploy` (alias behavior) | +| `up` | — | ✅ | ❌ | ❌ | **Yes** | Orchestration attrs: which phases ran, total service count | +| `down` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider`, resource count, purge flag | +| **Add** | | | | | | | +| `add` | — | ✅ | ❌ | ❌ | **Yes** | Component type added, source (template/manual) | +| **Completion** | | | | | | | +| `completion` | `bash`, `zsh`, `fish`, `powershell`, `fig` | ✅ | ❌ | ❌ | Low | Shell type — low priority, minimal analytical value | +| **VS Server** | | | | | | | +| `vs-server` | — | ✅ | ❌ | ❌ | Low | Long-running RPC; covered by `vsrpc.*` events | +| **Copilot Consent** | | | | | | | +| `copilot consent` | `list`, `revoke`, `grant` | ✅ | ❌ | ❌ | **Yes** | Consent operation type, scope | +| **Extension Management** | | | | | | | +| `extension` | `list`, `show`, `install`, `uninstall`, `upgrade` | ✅ | ❌ | ❌ | **Yes** | `extension.id`, `extension.version`, operation type | +| `extension source` | `list`, `add`, `remove`, `validate` | ✅ | ❌ | ❌ | **Yes** | Source operation type | +| **Init** | | | | | | | +| `init` | — | ✅ | ✅ | ✅ | No | — Already covered | +| **Update** | | | | | | | +| `update` | — | ✅ | ✅ | ✅ | No | — Already covered | +| **MCP** | | | | | | | +| `mcp start` | — | ✅ | ✅ | ✅ | No | — Already covered | +| **Disabled** | | | | | | | +| `version` | — | 🚫 | — | — | No | Intentionally disabled | +| `telemetry upload` | — | 🚫 | — | — | No | Intentionally disabled | + +## Gap Summary + +| Priority | Count | Commands | +|----------|-------|----------| +| **High** | 8 | `auth login/logout/status/token`, `provision`, `deploy`, `up`, `down` | +| **Medium** | 14 | `config *`, `env *`, `pipeline config`, `hooks run`, `template *`, `monitor`, `show`, `infra generate/synth`, `restore`, `build`, `package`, `add` | +| **Low** | 6 | `completion *`, `vs-server`, `infra create/delete` (deprecated), `copilot consent *`, `extension *` management | + +## Implementation Priority + +1. **Phase 1 — Auth & Core Lifecycle**: `auth login`, `provision`, `deploy`, `up`, `down` + — These are the highest-traffic commands with the most analytical value. + +2. **Phase 2 — Config, Env, Pipeline**: `config *`, `env *`, `pipeline config`, `hooks run` + — Understanding user configuration patterns and environment workflows. + +3. **Phase 3 — Templates & Infrastructure**: `template *`, `monitor`, `show`, `infra generate/synth` + — Template discovery and infrastructure generation insights. + +4. **Phase 4 — Remaining**: `restore`, `build`, `package`, `add`, `completion`, extension management + — Lower traffic or lower analytical value. diff --git a/docs/specs/metrics-audit/privacy-review-checklist.md b/docs/specs/metrics-audit/privacy-review-checklist.md new file mode 100644 index 00000000000..b6cc43c0d55 --- /dev/null +++ b/docs/specs/metrics-audit/privacy-review-checklist.md @@ -0,0 +1,199 @@ +# Privacy Review Checklist + +This document defines when a privacy review is required for telemetry changes in `azd`, +the data classification framework, hashing requirements, and a PR checklist template. + +## When to Trigger a Privacy Review + +A privacy review **must** be triggered when any of the following conditions are met: + +1. **New telemetry field** — Any new attribute key added to `fields/fields.go` or emitted + via `SetUsageAttributes` / `tracing.SetSpanAttributes`. + +2. **New event** — Any new event constant added to `events/events.go` or new span name + introduced via `tracing.Start`. + +3. **Classification change** — Any change to an existing field's `Classification` or `Purpose`. + +4. **New data source** — Telemetry that captures data from a source not previously instrumented + (e.g., a new Azure service response, user input, file system content). + +5. **Hashing removal or weakening** — Any change that removes `StringHashed` / `StringSliceHashed` + from a field that was previously hashed. + +6. **Cross-boundary data flow** — Telemetry that propagates trace context to external processes + (e.g., extension child processes) or receives context from external sources. + +7. **Measurement → String conversion** — Changing a field from a numeric measurement to a + string value (strings have higher re-identification risk). + +A privacy review is **not** required for: + +- Bug fixes to existing telemetry (e.g., fixing a typo in an attribute name). +- Removing telemetry fields entirely. +- Adding new values to an existing enum field (e.g., a new `auth.method` value) — unless + the new value captures data from a new source. + +## Raw Telemetry Data Shape Changes + +> "Any time any of the incoming raw data changes, your team needs to review and understand +> what needs to change to keep calculating things correctly" — AngelosP + +When the shape of raw telemetry data changes, ALL downstream consumers must be reviewed. +This is a **standalone mandatory checklist item** that applies whenever any of the following +occur: + +- [ ] **Field renamed** — A telemetry attribute key is renamed (e.g., `auth.type` → `auth.method`). + Review all Kusto functions, cooked table queries, LENS jobs, and dashboards that reference + the old key name. +- [ ] **Field type changed** — A field changes from string to int, or from single-value to array, + etc. Review all downstream parsers, `extend`/`project` statements in KQL, and any schema + validations. +- [ ] **Allowed values changed** — An enum field gains, removes, or renames values (e.g., + `auth.method` adding `"logout"`). Review all `case`/`iff`/`countif` expressions in Kusto + that filter or bucket by the old value set. +- [ ] **Field removed or deprecated** — A field is no longer emitted. Review all queries that + reference it and add null-handling or migration logic. +- [ ] **Measurement semantics changed** — Units change (seconds → milliseconds), counting + methodology changes, or aggregation expectations change. Review all KPI calculations, + percentile computations, and alerting thresholds. +- [ ] **Hashing algorithm changed** — Hash function or salt changes break join-ability with + historical data. Review all queries that correlate hashed fields across time ranges. + +**Action required**: Before merging any PR that changes raw telemetry data shape, the author +must verify that all downstream Kusto functions and KPI calculations still compute correctly +with the new shape. This includes cooked tables, LENS jobs, dashboards, and alerts. + +## Data Classifications + +All telemetry fields must be assigned exactly one classification from the table below. +Classifications are defined in `internal/telemetry/fields/fields.go`. + +| Classification | Description | Examples | Retention | +|----------------|-------------|----------|-----------| +| **PublicPersonalData** | Data the user has intentionally made public | GitHub username | Standard | +| **SystemMetadata** | Non-personal metadata about the system or environment | OS type, Go version, feature flags | Standard | +| **CallstackOrException** | Stack traces, panic details, error frames | `error.frame` | Reduced | +| **CustomerContent** | Content created by the user | File contents, messages | Highest restriction — avoid in telemetry | +| **EndUserPseudonymizedInformation** | User identifiers that have been pseudonymized | Hashed MAC address (`machine.id`), SQM User ID | Standard | +| **OrganizationalIdentifiableInformation** | Identifiers scoped to an organization | Azure subscription ID, tenant ID | Standard | + +### Classification Decision Tree + +``` +Is the data created by the user (file content, messages)? + └─ Yes → CustomerContent (do NOT emit in telemetry) + └─ No → + Can the data identify a specific person? + └─ Yes → + Is it already public? + └─ Yes → PublicPersonalData + └─ No → + Can it be hashed? + └─ Yes → Hash it → EndUserPseudonymizedInformation + └─ No → Do NOT emit — escalate to privacy team + └─ No → + Can it identify an organization? + └─ Yes → OrganizationalIdentifiableInformation + └─ No → + Is it a stack trace or exception detail? + └─ Yes → CallstackOrException + └─ No → SystemMetadata +``` + +## Hashing Requirements + +Any field that could identify a user, project, or environment **must** be hashed before emission. + +### Hash Functions + +All hashing functions are in `internal/telemetry/fields/key.go`. + +| Function | Signature | Behavior | +|----------|-----------|----------| +| `CaseInsensitiveHash` | `func CaseInsensitiveHash(value string) string` | Lowercases the input, then computes SHA-256. Returns hex-encoded digest. | +| `StringHashed` | `func StringHashed(key, value string) attribute.KeyValue` | Creates an OTel `attribute.KeyValue` with the value replaced by its case-insensitive SHA-256 hash. | +| `StringSliceHashed` | `func StringSliceHashed(key string, values []string) attribute.KeyValue` | Creates an OTel `attribute.KeyValue` where each element in the slice is independently hashed. | + +### Fields That Must Be Hashed + +| Field | Hash Function | Reason | +|-------|---------------|--------| +| `project.template.id` | `StringHashed` | Template IDs may contain repo URLs or user-chosen names | +| `project.template.version` | `StringHashed` | Version strings may be user-defined | +| `project.name` | `StringHashed` | Project names are user-chosen | +| `env.name` | `StringHashed` | Environment names may contain identifying information | + +### When to Hash New Fields + +A new field **must** be hashed if any of the following are true: + +- The value is user-provided (typed by the user or read from a user-authored file). +- The value could contain a project name, repository URL, or path. +- The value could be used to correlate across users or organizations. + +A new field should **not** be hashed if: + +- The value is from a fixed enum (e.g., `auth.method` = `"browser"`). +- The value is a count or duration (measurements). +- The value is system-generated metadata (e.g., OS type). + +## Data Catalog Classification Process + +When adding a new telemetry field: + +1. **Define the field** in `internal/telemetry/fields/fields.go` using the `NewKey` pattern. +2. **Assign classification** — use the decision tree above to determine the correct classification. +3. **Assign purpose** — select one or more from: `FeatureInsight`, `BusinessInsight`, `PerformanceAndHealth`. +4. **Determine hashing** — apply hashing rules above. +5. **Register in Data Catalog** — update the [Telemetry Schema](telemetry-schema.md) with: + - OTel key name + - Classification + - Purpose + - Whether it is hashed + - Whether it is a measurement + - Allowed values (if enum) +6. **Update LENS/Kusto** — if the field will be queried downstream, coordinate with the + data engineering team to update Kusto functions and cooked tables. + +## PR Checklist Template for Telemetry Changes + +Copy this checklist into your PR description when making telemetry changes. + +```markdown +## Telemetry Change Checklist + +### New Fields +- [ ] Field defined in `fields/fields.go` with correct classification and purpose +- [ ] Field documented in `docs/specs/metrics-audit/telemetry-schema.md` +- [ ] Hashing applied where required (user-provided values, names, paths) +- [ ] Measurement fields use correct OTel type (Int64, Float64) +- [ ] Enum values documented with allowed value set + +### New Events +- [ ] Event constant defined in `events/events.go` +- [ ] Event documented in `docs/specs/metrics-audit/telemetry-schema.md` +- [ ] Event follows naming convention (`prefix.noun.verb`) + +### Privacy +- [ ] Classification assigned using decision tree +- [ ] No `CustomerContent` emitted in telemetry +- [ ] No unhashed user-provided values +- [ ] No PII in string attributes (names, emails, paths) +- [ ] Privacy review triggered (if required per triggers above) + +### Testing +- [ ] Unit test verifies attributes are set on the span +- [ ] Integration test confirms end-to-end emission (if applicable) +- [ ] Verified field appears correctly in local telemetry output + +### Downstream +- [ ] LENS job updated (if field is queried in dashboards) +- [ ] Kusto function updated (if field is used in cooked tables) +- [ ] Dashboard updated (if field powers a new metric) + +### Documentation +- [ ] Feature-telemetry matrix updated (if gap is being closed) +- [ ] Telemetry schema updated with new field/event +- [ ] This checklist is complete +``` diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md new file mode 100644 index 00000000000..235d5f03291 --- /dev/null +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -0,0 +1,304 @@ +# Telemetry Schema Reference + +This document is the authoritative reference for all telemetry events, fields, classifications, +and data pipeline details in the Azure Developer CLI (`azd`). + +## Events + +Events are defined in `internal/telemetry/events/events.go`. Each event is emitted as an +OpenTelemetry span name or event name. + +| Constant | Value | Description | +|----------|-------|-------------| +| `CommandEventPrefix` | `cmd.` | Prefix for all command events (via `GetCommandEventName`) | +| `VsRpcEventPrefix` | `vsrpc.` | Prefix for VS Code JSON-RPC events | +| `McpEventPrefix` | `mcp.` | Prefix for MCP tool invocation events | +| `PackBuildEvent` | `tools.pack.build` | Cloud Native Buildpacks build event | +| `AgentTroubleshootEvent` | `agent.troubleshoot` | Agent troubleshooting event | +| `ExtensionRunEvent` | `ext.run` | Extension execution event | +| `ExtensionInstallEvent` | `ext.install` | Extension install/upgrade event | +| `CopilotInitializeEvent` | `copilot.initialize` | Copilot initialization event | +| `CopilotSessionEvent` | `copilot.session` | Copilot session lifecycle event | + +## Fields + +Fields are defined in `internal/telemetry/fields/fields.go`. Each field has a classification +and purpose that governs how it may be stored, queried, and retained. + +### Application-Level (Resource Attributes) + +These are set once at process startup via `resource.New()` and attached to every span. + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Service name | `service.name` | — | — | Always `"azd"` | +| Service version | `service.version` | — | — | Build version string | +| OS type | `os.type` | — | — | e.g. `linux`, `windows`, `darwin` | +| OS version | `os.version` | SystemMetadata | PerformanceAndHealth | Kernel / build version | +| Host architecture | `host.arch` | SystemMetadata | PerformanceAndHealth | e.g. `amd64`, `arm64` | +| Runtime version | `process.runtime.version` | SystemMetadata | PerformanceAndHealth | Go version | +| Machine ID | `machine.id` | EndUserPseudonymizedInformation | BusinessInsight | MAC address hash | +| Dev Device ID | `machine.devdeviceid` | EndUserPseudonymizedInformation | BusinessInsight | SQM User ID | +| Execution environment | `execution.environment` | SystemMetadata | BusinessInsight | CI system detection | +| Installer | `service.installer` | SystemMetadata | FeatureInsight | How azd was installed | + +### Experimentation + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Assignment context | `exp.assignmentContext` | SystemMetadata | FeatureInsight | + +### Identity and Account Context + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Object ID | `user_AuthenticatedId` | — | — | From Application Insights contracts | +| Tenant ID | `ad.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | +| Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"` or `"Service Principal"` | +| Subscription ID | `ad.subscription.id` | OrganizationalIdentifiableInformation | PerformanceAndHealth | Azure subscription | + +### Project Context (azure.yaml) + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Template ID | `project.template.id` | SystemMetadata | FeatureInsight | **Hashed** | +| Template version | `project.template.version` | SystemMetadata | FeatureInsight | **Hashed** | +| Project name | `project.name` | SystemMetadata | FeatureInsight | **Hashed** | +| Service hosts | `project.service.hosts` | SystemMetadata | FeatureInsight | List of host types | +| Service targets | `project.service.targets` | SystemMetadata | FeatureInsight | List of deploy targets | +| Service languages | `project.service.languages` | SystemMetadata | FeatureInsight | List of languages | +| Service language | `project.service.language` | SystemMetadata | PerformanceAndHealth | Single service language | +| Platform type | `platform.type` | SystemMetadata | FeatureInsight | e.g. `aca`, `aks` | + +### Config and Environment + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Feature flags | `config.features` | SystemMetadata | FeatureInsight | Active feature flags | +| Environment name | `env.name` | SystemMetadata | FeatureInsight | **Hashed** | + +### Command Entry-Point + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Flags | `cmd.flags` | SystemMetadata | FeatureInsight | Which flags were passed | +| Argument count | `cmd.args.count` | SystemMetadata | FeatureInsight | **Measurement** | +| Entry point | `cmd.entry` | SystemMetadata | FeatureInsight | How the command was invoked | + +### Error Attributes + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Error category | `error.category` | SystemMetadata | PerformanceAndHealth | +| Error code | `error.code` | SystemMetadata | PerformanceAndHealth | +| Error type | `error.type` | SystemMetadata | PerformanceAndHealth | +| Inner error | `error.inner` | SystemMetadata | PerformanceAndHealth | +| Error frame | `error.frame` | SystemMetadata | PerformanceAndHealth | + +Error classification is handled by `MapError` in `internal/cmd/errors.go`, which categorizes +errors into: update errors, auth errors, service (Azure) errors, deployment errors, extension +errors, tool errors, sentinel errors, and network errors. Each receives an `error.code`, +`error.category`, and contextual attributes. + +### Service Attributes + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Service host | `service.host` | SystemMetadata | PerformanceAndHealth | | +| Service name | `service.name` | SystemMetadata | PerformanceAndHealth | | +| Status code | `service.statusCode` | SystemMetadata | PerformanceAndHealth | **Measurement** | +| Method | `service.method` | SystemMetadata | PerformanceAndHealth | | +| Error code | `service.errorCode` | SystemMetadata | PerformanceAndHealth | **Measurement** | +| Correlation ID | `service.correlationId` | SystemMetadata | PerformanceAndHealth | | + +### Tool Attributes + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Tool name | `tool.name` | SystemMetadata | FeatureInsight | +| Tool exit code | `tool.exitCode` | SystemMetadata | PerformanceAndHealth | + +### Performance + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Interaction time | `perf.interact_time` | SystemMetadata | PerformanceAndHealth | **Measurement** — time to first user prompt | + +### Pack (Buildpacks) + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Builder image | `pack.builder.image` | SystemMetadata | FeatureInsight | +| Builder tag | `pack.builder.tag` | SystemMetadata | FeatureInsight | + +### MCP + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Client name | `mcp.client.name` | SystemMetadata | FeatureInsight | +| Client version | `mcp.client.version` | SystemMetadata | FeatureInsight | + +### Init + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Init method | `init.method` | SystemMetadata | FeatureInsight | template/app/project/environment/copilot | +| Detected databases | `appinit.detected.databases` | SystemMetadata | FeatureInsight | | +| Detected services | `appinit.detected.services` | SystemMetadata | FeatureInsight | | +| Confirmed databases | `appinit.confirmed.databases` | SystemMetadata | FeatureInsight | | +| Confirmed services | `appinit.confirmed.services` | SystemMetadata | FeatureInsight | | +| Modify add count | `appinit.modify_add.count` | SystemMetadata | FeatureInsight | **Measurement** | +| Modify remove count | `appinit.modify_remove.count` | SystemMetadata | FeatureInsight | **Measurement** | +| Last step | `appinit.lastStep` | SystemMetadata | FeatureInsight | | + +### Remote Build + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Remote build count | `container.remoteBuild.count` | SystemMetadata | FeatureInsight | **Measurement** | + +### JSON-RPC + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| RPC method | `rpc.method` | SystemMetadata | FeatureInsight | | +| Request ID | `rpc.jsonrpc.request_id` | SystemMetadata | PerformanceAndHealth | | +| Error code | `rpc.jsonrpc.error_code` | SystemMetadata | PerformanceAndHealth | **Measurement** | + +### Agent + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Fix attempts | `agent.fix.attempts` | SystemMetadata | PerformanceAndHealth | + +### Extensions + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Extension ID | `extension.id` | SystemMetadata | FeatureInsight | +| Extension version | `extension.version` | SystemMetadata | FeatureInsight | +| Extension installed | `extension.installed` | SystemMetadata | FeatureInsight | + +### Update + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Update channel | `update.channel` | SystemMetadata | FeatureInsight | +| Install method | `update.installMethod` | SystemMetadata | FeatureInsight | +| From version | `update.fromVersion` | SystemMetadata | FeatureInsight | +| To version | `update.toVersion` | SystemMetadata | FeatureInsight | +| Update result | `update.result` | SystemMetadata | FeatureInsight | + +### Copilot Session + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Session ID | `copilot.session.id` | SystemMetadata | FeatureInsight | | +| Is new session | `copilot.session.isNew` | SystemMetadata | FeatureInsight | | +| Message count | `copilot.session.messageCount` | SystemMetadata | FeatureInsight | **Measurement** | + +### Copilot Init + +| Field | OTel Key | Classification | Purpose | +|-------|----------|----------------|---------| +| Is first run | `copilot.init.isFirstRun` | SystemMetadata | FeatureInsight | +| Reasoning effort | `copilot.init.reasoningEffort` | SystemMetadata | FeatureInsight | +| Model | `copilot.init.model` | SystemMetadata | FeatureInsight | +| Consent scope | `copilot.init.consentScope` | SystemMetadata | FeatureInsight | + +### Copilot Message + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Mode | `copilot.mode` | SystemMetadata | FeatureInsight | | +| Model | `copilot.message.model` | SystemMetadata | FeatureInsight | | +| Input tokens | `copilot.message.inputTokens` | SystemMetadata | PerformanceAndHealth | **Measurement** | +| Output tokens | `copilot.message.outputTokens` | SystemMetadata | PerformanceAndHealth | **Measurement** | +| Billing rate | `copilot.message.billingRate` | SystemMetadata | BusinessInsight | **Measurement** | +| Premium requests | `copilot.message.premiumRequests` | SystemMetadata | BusinessInsight | **Measurement** | +| Duration (ms) | `copilot.message.durationMs` | SystemMetadata | PerformanceAndHealth | **Measurement** | + +### Copilot Consent + +| Field | OTel Key | Classification | Purpose | Notes | +|-------|----------|----------------|---------|-------| +| Approved count | `copilot.consent.approvedCount` | SystemMetadata | FeatureInsight | **Measurement** | +| Denied count | `copilot.consent.deniedCount` | SystemMetadata | FeatureInsight | **Measurement** | + +## New Fields (Added by This Audit) + +The following fields are being introduced to close telemetry gaps identified in the +[Feature-Telemetry Matrix](feature-telemetry-matrix.md). + +| Field | OTel Key | Classification | Purpose | Values | +|-------|----------|----------------|---------|--------| +| Auth method | `auth.method` | SystemMetadata | FeatureInsight | `browser`, `device-code`, `service-principal-secret`, `service-principal-certificate`, `federated-github`, `federated-azure-pipelines`, `federated-oidc`, `managed-identity`, `external`, `oneauth`, `check-status`, `logout` | +| Auth result | `auth.result` | SystemMetadata | FeatureInsight | `success`, `failure`, `logged-in`, `not-logged-in` | +| Config operation | `config.operation` | SystemMetadata | FeatureInsight | show, list, get, set, unset, reset, list-alpha, options | +| Env operation | `env.operation` | SystemMetadata | FeatureInsight | set, set-secret, select, new, remove, list, refresh, get-values, get-value, config-get, config-set, config-unset | +| Env count | `env.count` | SystemMetadata | FeatureInsight | **Measurement** — number of environments | +| Hooks name | `hooks.name` | SystemMetadata | FeatureInsight | Hook script name | +| Hooks type | `hooks.type` | SystemMetadata | FeatureInsight | `project`, `service` | +| Template operation | `template.operation` | SystemMetadata | FeatureInsight | list, show, source-list, source-add, source-remove | +| Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | `github`, `azdo` | +| Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | `federated`, `client-credentials` | +| Monitor type | `monitor.type` | SystemMetadata | FeatureInsight | `overview`, `logs`, `live` | +| Show output format | `show.output.format` | SystemMetadata | FeatureInsight | json, table, etc. | +| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | `bicep`, `terraform` | + +## Data Classifications + +Classifications are defined in `internal/telemetry/fields/fields.go` and control how data +is stored, retained, and who may access it. + +| Classification | Description | +|----------------|-------------| +| `PublicPersonalData` | Data the user has made public (e.g. GitHub username) | +| `SystemMetadata` | Non-personal system/environment metadata | +| `CallstackOrException` | Stack traces and exception details | +| `CustomerContent` | User-created content (files, messages) — highest sensitivity | +| `EndUserPseudonymizedInformation` | Pseudonymized user identifiers (hashed MACs, device IDs) | +| `OrganizationalIdentifiableInformation` | Organization-level identifiers (subscription IDs, tenant IDs) | + +## Purposes + +Each field is tagged with one or more purposes that govern its permitted use. + +| Purpose | Description | +|---------|-------------| +| `FeatureInsight` | Understanding feature adoption and usage patterns | +| `BusinessInsight` | Business metrics (active users, organizations, growth) | +| `PerformanceAndHealth` | Performance monitoring, error rates, reliability | + +## Hashing + +Sensitive values are hashed before emission using functions in `internal/telemetry/fields/key.go`. + +| Function | Behavior | +|----------|----------| +| `CaseInsensitiveHash(value)` | Lowercases, then SHA-256 hashes | +| `StringHashed(key, value)` | Creates an OTel attribute with a case-insensitive SHA-256 hash | +| `StringSliceHashed(key, values)` | Hashes each element in a string slice independently | + +Fields that are hashed: `project.template.id`, `project.template.version`, `project.name`, `env.name`. + +## Data Pipeline + +``` +┌──────────────┐ ┌─────────────────────┐ ┌──────────────┐ ┌──────────────────┐ +│ OTel Spans │───▶│ App Insights │───▶│ Disk Queue │───▶│ Azure Monitor / │ +│ (in-process)│ │ Exporter (custom) │ │ (~/.azd/) │ │ Kusto │ +└──────────────┘ └─────────────────────┘ └──────────────┘ └──────────────────┘ + │ + ▼ + ┌──────────────┐ + │ telemetry │ + │ upload cmd │ + └──────────────┘ +``` + +1. **Instrumentation**: Commands create OTel spans with attributes via `tracing.Start` and `SetUsageAttributes`. +2. **Export**: A custom Application Insights exporter converts spans to App Insights envelopes. +3. **Queue**: Envelopes are written to disk under `~/.azd/telemetry/`. +4. **Upload**: The `azd telemetry upload` command (run as a background process) reads the queue and sends data to Azure Monitor. +5. **Analysis**: Data flows into Kusto tables for dashboarding and analysis via LENS jobs and cooked tables. From b6f2d124d9363273466e329dffd7a6a92c71c69d Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 05:35:51 -0700 Subject: [PATCH 02/14] fix: address PR review feedback (threads 1-8) - Rename test: TestTelemetryFieldConstants with clarified allowlist approach - pipeline.go: skip SetUsageAttributes for empty provider/auth values - docs: fix internal/telemetry/ -> cli/azd/internal/tracing/ paths - docs: add Anonymous to ad.account.type allowed values - docs: add missing legend symbol in feature-telemetry-matrix.md - docs: pick CODEOWNERS over GitHub Actions for telemetry PR labeling - docs: add opt-out rate estimation section with @AngelosP question - cspell: add metrics-audit word list for docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .vscode/cspell.misc.yaml | 14 +++++++ cli/azd/cmd/pipeline.go | 12 +++--- cli/azd/cmd/telemetry_coverage_test.go | 15 ++++--- docs/specs/metrics-audit/audit-process.md | 39 +++++++++++++++---- .../metrics-audit/feature-telemetry-matrix.md | 1 + .../metrics-audit/privacy-review-checklist.md | 8 ++-- docs/specs/metrics-audit/telemetry-schema.md | 8 ++-- 7 files changed, 71 insertions(+), 26 deletions(-) diff --git a/.vscode/cspell.misc.yaml b/.vscode/cspell.misc.yaml index 66a501e4eb8..062bcf9cf2a 100644 --- a/.vscode/cspell.misc.yaml +++ b/.vscode/cspell.misc.yaml @@ -20,6 +20,20 @@ overrides: words: - MSRC - msrc + - filename: ./docs/specs/metrics-audit/** + words: + - vsrpc + - Buildpacks + - devdeviceid + - appinit + - oneauth + - dashboarding + - Pseudonymized + - pseudonymized + - unhashed + - countif + - Angelos + - Entra - filename: ./README.md words: - VSIX diff --git a/cli/azd/cmd/pipeline.go b/cli/azd/cmd/pipeline.go index 40d04990596..73c7478cba5 100644 --- a/cli/azd/cmd/pipeline.go +++ b/cli/azd/cmd/pipeline.go @@ -164,11 +164,13 @@ func newPipelineConfigAction( // Run implements action interface func (p *pipelineConfigAction) Run(ctx context.Context) (*actions.ActionResult, error) { - // Track pipeline provider and auth type - tracing.SetUsageAttributes( - fields.PipelineProviderKey.String(p.flags.PipelineProvider), - fields.PipelineAuthKey.String(p.flags.PipelineAuthTypeName), - ) + // Track pipeline provider and auth type (only when explicitly specified) + if p.flags.PipelineProvider != "" { + tracing.SetUsageAttributes(fields.PipelineProviderKey.String(p.flags.PipelineProvider)) + } + if p.flags.PipelineAuthTypeName != "" { + tracing.SetUsageAttributes(fields.PipelineAuthKey.String(p.flags.PipelineAuthTypeName)) + } infra, err := p.importManager.ProjectInfrastructure(ctx, p.projectConfig) if err != nil { diff --git a/cli/azd/cmd/telemetry_coverage_test.go b/cli/azd/cmd/telemetry_coverage_test.go index d0370993e2e..1daf5a3c286 100644 --- a/cli/azd/cmd/telemetry_coverage_test.go +++ b/cli/azd/cmd/telemetry_coverage_test.go @@ -10,13 +10,16 @@ import ( "github.com/stretchr/testify/require" ) -// TestTelemetryFieldsForGapCommands verifies that all new telemetry field constants -// added for previously-uninstrumented commands are properly defined and produce valid -// attribute key-value pairs. +// TestTelemetryFieldConstants verifies that all telemetry field constants added for +// command-specific instrumentation are properly defined and produce valid attribute +// key-value pairs. This is a contract test: if a field constant is removed or renamed, +// this test will fail, catching regressions in the telemetry schema. // -// This serves as a CI telemetry coverage check: if a field constant is removed or renamed, -// this test will fail, catching regressions in telemetry instrumentation. -func TestTelemetryFieldsForGapCommands(t *testing.T) { +// NOTE: This test validates field definitions, not command-level instrumentation. +// Command-level coverage is enforced via the documented allowlist in +// TestCommandTelemetryCoverageAllowlist (below) and the feature-telemetry-matrix.md. +// Full AST-based scanning of SetUsageAttributes calls is a future enhancement. +func TestTelemetryFieldConstants(t *testing.T) { // Auth command telemetry fields t.Run("AuthFields", func(t *testing.T) { kv := fields.AuthMethodKey.String("browser") diff --git a/docs/specs/metrics-audit/audit-process.md b/docs/specs/metrics-audit/audit-process.md index 57482af7f90..dd7c1e6924e 100644 --- a/docs/specs/metrics-audit/audit-process.md +++ b/docs/specs/metrics-audit/audit-process.md @@ -69,7 +69,29 @@ Telemetry audits run on a quarterly cycle aligned with fiscal quarters. - [ ] Verify `telemetry upload` still has `DisableTelemetry: true` - [ ] Check for any new commands with `DisableTelemetry: true` — confirm intent -### 6. Data Pipeline Health +### 6. Opt-Out Rate Estimation + +When `AZURE_DEV_COLLECT_TELEMETRY=no`, the entire telemetry pipeline is disabled — no +spans are created and no data is sent. This means **opted-out users are invisible** in +telemetry data and we cannot directly measure the opt-out rate. + +**Estimation approach** (indirect): + +- [ ] Compare total install/download counts (from package manager stats, GitHub releases, + winget/brew/apt download logs) against distinct active telemetry users in the same period +- [ ] Estimate: `opt-out rate ≈ 1 − (active telemetry users / total installs)` +- [ ] Track this ratio over time to detect trends + +> **⚠️ Open question for @AngelosP / Privacy team**: Should azd send a single anonymous +> opt-out counter signal (containing zero identifying information — no machine ID, no IP, +> just an increment) when the user has `AZURE_DEV_COLLECT_TELEMETRY=no`? This is a gray +> area: GDPR Article 7(3) requires stopping processing on consent withdrawal, but a +> zero-identifier counter may not constitute "personal data." The .NET SDK installer does +> send a telemetry entry on successful installation even before the user sets the opt-out +> variable. A decision from the privacy team would clarify whether this approach is +> acceptable for azd. + +### 7. Data Pipeline Health - [ ] Verify telemetry upload process is functioning (check error rates) - [ ] Confirm data arrives in Azure Monitor within expected latency @@ -185,19 +207,22 @@ jobs: Automatically label PRs that modify telemetry files for review. **Trigger files:** -- `internal/telemetry/fields/fields.go` -- `internal/telemetry/events/events.go` -- `internal/telemetry/fields/key.go` -- `internal/telemetry/resource/resource.go` +- `cli/azd/internal/tracing/fields/fields.go` +- `cli/azd/internal/tracing/events/events.go` +- `cli/azd/internal/tracing/fields/key.go` +- `cli/azd/internal/tracing/resource/resource.go` - Any file containing `SetUsageAttributes` -**Implementation:** Use a GitHub Actions workflow or a CODEOWNERS entry: +**Implementation:** Use a CODEOWNERS entry to require telemetry team review: ``` # .github/CODEOWNERS (telemetry-related files) -internal/telemetry/ @AzureDevCLI/telemetry-reviewers +cli/azd/internal/tracing/ @AzureDevCLI/telemetry-reviewers ``` +This is preferred over a separate GitHub Actions workflow because it integrates directly +with the existing PR review flow and requires no additional CI configuration. + ### Telemetry Diff Report Generate a diff report on every PR that modifies telemetry, showing: diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index 87ac356c5ba..a023335e8f5 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -10,6 +10,7 @@ specific telemetry additions. |--------|---------| | ✅ | Covered — command-specific attributes or events are emitted | | ⚠️ | Global span only — no command-specific telemetry | +| ❌ | Gap identified — needs instrumentation | | 🚫 | Telemetry intentionally disabled | ## Commands with Telemetry Disabled diff --git a/docs/specs/metrics-audit/privacy-review-checklist.md b/docs/specs/metrics-audit/privacy-review-checklist.md index b6cc43c0d55..469f4bf908b 100644 --- a/docs/specs/metrics-audit/privacy-review-checklist.md +++ b/docs/specs/metrics-audit/privacy-review-checklist.md @@ -7,10 +7,10 @@ the data classification framework, hashing requirements, and a PR checklist temp A privacy review **must** be triggered when any of the following conditions are met: -1. **New telemetry field** — Any new attribute key added to `fields/fields.go` or emitted +1. **New telemetry field** — Any new attribute key added to `cli/azd/internal/tracing/fields/fields.go` or emitted via `SetUsageAttributes` / `tracing.SetSpanAttributes`. -2. **New event** — Any new event constant added to `events/events.go` or new span name +2. **New event** — Any new event constant added to `cli/azd/internal/tracing/events/events.go` or new span name introduced via `tracing.Start`. 3. **Classification change** — Any change to an existing field's `Classification` or `Purpose`. @@ -67,7 +67,7 @@ with the new shape. This includes cooked tables, LENS jobs, dashboards, and aler ## Data Classifications All telemetry fields must be assigned exactly one classification from the table below. -Classifications are defined in `internal/telemetry/fields/fields.go`. +Classifications are defined in `cli/azd/internal/tracing/fields/fields.go`. | Classification | Description | Examples | Retention | |----------------|-------------|----------|-----------| @@ -107,7 +107,7 @@ Any field that could identify a user, project, or environment **must** be hashed ### Hash Functions -All hashing functions are in `internal/telemetry/fields/key.go`. +All hashing functions are in `cli/azd/internal/tracing/fields/key.go`. | Function | Signature | Behavior | |----------|-----------|----------| diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 235d5f03291..0719a5a55b0 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -5,7 +5,7 @@ and data pipeline details in the Azure Developer CLI (`azd`). ## Events -Events are defined in `internal/telemetry/events/events.go`. Each event is emitted as an +Events are defined in `cli/azd/internal/tracing/events/events.go`. Each event is emitted as an OpenTelemetry span name or event name. | Constant | Value | Description | @@ -22,7 +22,7 @@ OpenTelemetry span name or event name. ## Fields -Fields are defined in `internal/telemetry/fields/fields.go`. Each field has a classification +Fields are defined in `cli/azd/internal/tracing/fields/fields.go`. Each field has a classification and purpose that governs how it may be stored, queried, and retained. ### Application-Level (Resource Attributes) @@ -54,7 +54,7 @@ These are set once at process startup via `resource.New()` and attached to every |-------|----------|----------------|---------|-------| | Object ID | `user_AuthenticatedId` | — | — | From Application Insights contracts | | Tenant ID | `ad.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | -| Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"` or `"Service Principal"` | +| Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"`, `"Service Principal"`, or `"Anonymous"` | | Subscription ID | `ad.subscription.id` | OrganizationalIdentifiableInformation | PerformanceAndHealth | Azure subscription | ### Project Context (azure.yaml) @@ -272,7 +272,7 @@ Each field is tagged with one or more purposes that govern its permitted use. ## Hashing -Sensitive values are hashed before emission using functions in `internal/telemetry/fields/key.go`. +Sensitive values are hashed before emission using functions in `cli/azd/internal/tracing/fields/key.go`. | Function | Behavior | |----------|----------| From 253127de3599a0af3e6f172d6061ec5680fda16e Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 05:51:03 -0700 Subject: [PATCH 03/14] fix: hash hook names since extensions can define arbitrary names Extensions register custom hooks via WithProjectEventHandler/WithServiceEventHandler with arbitrary string names that are not validated against a fixed set. Hash the hook name to prevent potential PII leakage in telemetry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/hooks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/cmd/hooks.go b/cli/azd/cmd/hooks.go index 443654eebb2..38a2d1d6f55 100644 --- a/cli/azd/cmd/hooks.go +++ b/cli/azd/cmd/hooks.go @@ -125,7 +125,7 @@ func (hra *hooksRunAction) Run(ctx context.Context) (*actions.ActionResult, erro hookType = "service" } tracing.SetUsageAttributes( - fields.HooksNameKey.String(hookName), + fields.StringHashed(fields.HooksNameKey, hookName), fields.HooksTypeKey.String(hookType), ) From 67b6a525f61d62e4294745184c8c633d08ed0811 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 06:02:19 -0700 Subject: [PATCH 04/14] fix: emit default values for pipeline/infra telemetry, revert hook hashing - pipeline.provider: emit 'auto' when user doesn't specify --provider - pipeline.auth.type: emit 'auto' when user doesn't specify --auth-type - infra.provider: emit 'auto' when provider not set in project config - hooks.name: revert to raw string (not hashed) for telemetry readability - audit-process.md: add telemetry validation pipeline section - telemetry-schema.md: document 'auto' as valid value for provider fields Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/hooks.go | 2 +- cli/azd/cmd/infra_generate.go | 9 +++- cli/azd/cmd/pipeline.go | 7 ++- docs/specs/metrics-audit/audit-process.md | 53 ++++++++++++++++++++ docs/specs/metrics-audit/telemetry-schema.md | 6 +-- 5 files changed, 70 insertions(+), 7 deletions(-) diff --git a/cli/azd/cmd/hooks.go b/cli/azd/cmd/hooks.go index 38a2d1d6f55..443654eebb2 100644 --- a/cli/azd/cmd/hooks.go +++ b/cli/azd/cmd/hooks.go @@ -125,7 +125,7 @@ func (hra *hooksRunAction) Run(ctx context.Context) (*actions.ActionResult, erro hookType = "service" } tracing.SetUsageAttributes( - fields.StringHashed(fields.HooksNameKey, hookName), + fields.HooksNameKey.String(hookName), fields.HooksTypeKey.String(hookType), ) diff --git a/cli/azd/cmd/infra_generate.go b/cli/azd/cmd/infra_generate.go index 7dc6825e94e..9a8bdb7778f 100644 --- a/cli/azd/cmd/infra_generate.go +++ b/cli/azd/cmd/infra_generate.go @@ -88,8 +88,13 @@ func newInfraGenerateAction( func (a *infraGenerateAction) Run(ctx context.Context) (*actions.ActionResult, error) { // Track infra provider from project configuration - if a.projectConfig != nil && a.projectConfig.Infra.Provider != "" { - tracing.SetUsageAttributes(fields.InfraProviderKey.String(string(a.projectConfig.Infra.Provider))) + // Emit "auto" when provider is empty, so we know auto-detection was used. + if a.projectConfig != nil { + provider := string(a.projectConfig.Infra.Provider) + if provider == "" { + provider = "auto" + } + tracing.SetUsageAttributes(fields.InfraProviderKey.String(provider)) } if a.calledAs == "synth" { diff --git a/cli/azd/cmd/pipeline.go b/cli/azd/cmd/pipeline.go index 73c7478cba5..9a46825d9ec 100644 --- a/cli/azd/cmd/pipeline.go +++ b/cli/azd/cmd/pipeline.go @@ -164,12 +164,17 @@ func newPipelineConfigAction( // Run implements action interface func (p *pipelineConfigAction) Run(ctx context.Context) (*actions.ActionResult, error) { - // Track pipeline provider and auth type (only when explicitly specified) + // Track pipeline provider and auth type + // Emit "auto" when the user didn't specify, so we know auto-detection was used. if p.flags.PipelineProvider != "" { tracing.SetUsageAttributes(fields.PipelineProviderKey.String(p.flags.PipelineProvider)) + } else { + tracing.SetUsageAttributes(fields.PipelineProviderKey.String("auto")) } if p.flags.PipelineAuthTypeName != "" { tracing.SetUsageAttributes(fields.PipelineAuthKey.String(p.flags.PipelineAuthTypeName)) + } else { + tracing.SetUsageAttributes(fields.PipelineAuthKey.String("auto")) } infra, err := p.importManager.ProjectInfrastructure(ctx, p.projectConfig) diff --git a/docs/specs/metrics-audit/audit-process.md b/docs/specs/metrics-audit/audit-process.md index dd7c1e6924e..1e604ef2ac4 100644 --- a/docs/specs/metrics-audit/audit-process.md +++ b/docs/specs/metrics-audit/audit-process.md @@ -223,6 +223,59 @@ cli/azd/internal/tracing/ @AzureDevCLI/telemetry-reviewers This is preferred over a separate GitHub Actions workflow because it integrates directly with the existing PR review flow and requires no additional CI configuration. +## Telemetry Validation Pipeline + +### 1. Local Validation + +Use `--trace-log-file ` to dump all telemetry spans to a JSON file, then inspect for +expected attributes. + +```bash +azd pipeline config --trace-log-file telemetry-dump.json +# Then inspect telemetry-dump.json for pipeline.provider, pipeline.auth.type fields +``` + +```bash +azd infra synth --trace-log-file telemetry-dump.json +# Inspect for infra.provider field +``` + +This flag is available on all azd commands and writes the full span tree (with all attributes) +to the specified file. Use `jq` or similar tools to filter for specific keys. + +### 2. Functional Tests + +The repo has existing functional telemetry tests at +`cli/azd/test/functional/telemetry_test.go` that run real commands and validate trace +attributes. New telemetry fields should be covered here. + +When adding a new field, add a test case that: +1. Runs the command that emits the field. +2. Reads the trace output. +3. Asserts the expected attribute key and value are present. + +### 3. PR Builds + +Azure Pipelines publishes PR-specific builds via `eng/pipelines/release-cli.yml`. Install a +PR build with: + +```bash +azd version install pr/ +``` + +Then manually test commands and inspect `--trace-log-file` output to verify the new telemetry +attributes are present with expected values. + +### 4. Pre-Production Checklist + +Before merging telemetry changes: + +- [ ] Unit tests pass (`go test ./cmd/... ./internal/tracing/...`) +- [ ] Functional telemetry tests pass +- [ ] Local `--trace-log-file` validation for each new field +- [ ] PR build smoke test with real Azure subscription +- [ ] Dev telemetry endpoint receives expected attributes (non-prod builds auto-target dev App Insights) + ### Telemetry Diff Report Generate a diff report on every PR that modifies telemetry, showing: diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 0719a5a55b0..f77fa7e49b4 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -240,11 +240,11 @@ The following fields are being introduced to close telemetry gaps identified in | Hooks name | `hooks.name` | SystemMetadata | FeatureInsight | Hook script name | | Hooks type | `hooks.type` | SystemMetadata | FeatureInsight | `project`, `service` | | Template operation | `template.operation` | SystemMetadata | FeatureInsight | list, show, source-list, source-add, source-remove | -| Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | `github`, `azdo` | -| Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | `federated`, `client-credentials` | +| Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | `github`, `azdo`, `auto` (auto-detected) | +| Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | `federated`, `client-credentials`, `auto` (auto-detected) | | Monitor type | `monitor.type` | SystemMetadata | FeatureInsight | `overview`, `logs`, `live` | | Show output format | `show.output.format` | SystemMetadata | FeatureInsight | json, table, etc. | -| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | `bicep`, `terraform` | +| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | `bicep`, `terraform`, `auto` (auto-detected from files) | ## Data Classifications From 93d9cfb55e9fa29fa48b2be35b1e1d94082eb0ab Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 08:02:32 -0700 Subject: [PATCH 05/14] fix: add CODEOWNERS to cspell word list Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .vscode/cspell.misc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/cspell.misc.yaml b/.vscode/cspell.misc.yaml index 062bcf9cf2a..45e912c422e 100644 --- a/.vscode/cspell.misc.yaml +++ b/.vscode/cspell.misc.yaml @@ -34,6 +34,7 @@ overrides: - countif - Angelos - Entra + - CODEOWNERS - filename: ./README.md words: - VSIX From 61e9d377008eb3a199e6bba0a443daf58f15a9ad Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 11:24:33 -0700 Subject: [PATCH 06/14] refactor: remove redundant telemetry attributes per review feedback Remove attributes that duplicate data already captured by command span names (config.operation, env.operation, template.operation), OTel span status (auth.result), or cmd.flags flag names (monitor.type). Remove show.output.format (should be global, tracked as follow-up). Remove dead code Anonymous fallback in manager.go. 8 unique attributes remain: auth.method, auth.tenant.id.hashed, env.count, hooks.name, hooks.type, infra.provider, pipeline.provider, pipeline.auth.type. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/auth_login.go | 2 - cli/azd/cmd/auth_logout.go | 3 - cli/azd/cmd/auth_status.go | 6 - cli/azd/cmd/config.go | 9 - cli/azd/cmd/env.go | 11 -- cli/azd/cmd/env_remove.go | 3 - cli/azd/cmd/monitor.go | 11 -- cli/azd/cmd/telemetry_coverage_test.go | 158 +++++------------ cli/azd/cmd/templates.go | 7 - cli/azd/internal/cmd/show/show.go | 3 - cli/azd/internal/tracing/fields/fields.go | 64 ------- .../tracing/fields/fields_audit_test.go | 55 ------ cli/azd/pkg/auth/manager.go | 2 - .../metrics-audit/feature-telemetry-matrix.md | 166 +++++++++--------- docs/specs/metrics-audit/telemetry-schema.md | 10 +- 15 files changed, 136 insertions(+), 374 deletions(-) diff --git a/cli/azd/cmd/auth_login.go b/cli/azd/cmd/auth_login.go index 7e25f88fa02..435009ccb07 100644 --- a/cli/azd/cmd/auth_login.go +++ b/cli/azd/cmd/auth_login.go @@ -362,10 +362,8 @@ func (la *loginAction) Run(ctx context.Context) (*actions.ActionResult, error) { } if err := la.login(ctx); err != nil { - tracing.SetUsageAttributes(fields.AuthResultKey.String("failure")) return nil, err } - tracing.SetUsageAttributes(fields.AuthResultKey.String("success")) if _, err := la.verifyLoggedIn(ctx); err != nil { return nil, err diff --git a/cli/azd/cmd/auth_logout.go b/cli/azd/cmd/auth_logout.go index d3b1825a3b8..9482479d7ea 100644 --- a/cli/azd/cmd/auth_logout.go +++ b/cli/azd/cmd/auth_logout.go @@ -70,16 +70,13 @@ func (la *logoutAction) Run(ctx context.Context) (*actions.ActionResult, error) err := la.authManager.Logout(ctx) if err != nil { - tracing.SetUsageAttributes(fields.AuthResultKey.String("failure")) return nil, err } err = la.accountSubManager.ClearSubscriptions(ctx) if err != nil { - tracing.SetUsageAttributes(fields.AuthResultKey.String("failure")) return nil, err } - tracing.SetUsageAttributes(fields.AuthResultKey.String("success")) return nil, nil } diff --git a/cli/azd/cmd/auth_status.go b/cli/azd/cmd/auth_status.go index 1c035dffdd8..927a8008c43 100644 --- a/cli/azd/cmd/auth_status.go +++ b/cli/azd/cmd/auth_status.go @@ -14,8 +14,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/internal/tracing/resource" "github.com/azure/azure-dev/cli/azd/pkg/auth" "github.com/azure/azure-dev/cli/azd/pkg/contracts" @@ -73,7 +71,6 @@ func newAuthStatusAction( } func (a *authStatusAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.AuthMethodKey.String("check-status")) loginMode, err := a.authManager.Mode() if err != nil { log.Printf("error: fetching auth mode: %v", err) @@ -97,16 +94,13 @@ func (a *authStatusAction) Run(ctx context.Context) (*actions.ActionResult, erro res := contracts.StatusResult{} if err != nil { res.Status = contracts.AuthStatusUnauthenticated - tracing.SetUsageAttributes(fields.AuthResultKey.String("not-logged-in")) } else { res.Status = contracts.AuthStatusAuthenticated token, err := a.verifyLoggedIn(ctx, scopes) if err != nil { res.Status = contracts.AuthStatusUnauthenticated - tracing.SetUsageAttributes(fields.AuthResultKey.String("not-logged-in")) log.Printf("error: verifying logged in status: %v", err) } else { - tracing.SetUsageAttributes(fields.AuthResultKey.String("logged-in")) if token != nil { expiresOn := contracts.RFC3339Time(token.ExpiresOn) res.ExpiresOn = &expiresOn diff --git a/cli/azd/cmd/config.go b/cli/azd/cmd/config.go index 6bc085592f9..e26d222417e 100644 --- a/cli/azd/cmd/config.go +++ b/cli/azd/cmd/config.go @@ -17,8 +17,6 @@ import ( "github.com/MakeNowJust/heredoc/v2" "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -209,7 +207,6 @@ func newConfigShowAction( // Executes the `azd config show` action func (a *configShowAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("show")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -279,7 +276,6 @@ func newConfigGetAction( // Executes the `azd config get ` action func (a *configGetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("get")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -321,7 +317,6 @@ func newConfigSetAction(configManager config.UserConfigManager, args []string) a // Executes the `azd config set ` action func (a *configSetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("set")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -354,7 +349,6 @@ func newConfigUnsetAction(configManager config.UserConfigManager, args []string) // Executes the `azd config unset ` action func (a *configUnsetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("unset")) azdConfig, err := a.configManager.Load() if err != nil { return nil, err @@ -405,7 +399,6 @@ func newConfigResetAction( // Executes the `azd config reset` action func (a *configResetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("reset")) a.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Reset configuration (azd config reset)", }) @@ -480,7 +473,6 @@ type configListAlphaAction struct { } func (a *configListAlphaAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("list-alpha")) features, err := a.alphaFeaturesManager.ListFeatures() if err != nil { return nil, err @@ -577,7 +569,6 @@ func newConfigOptionsAction( } func (a *configOptionsAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ConfigOperationKey.String("options")) options := config.GetAllConfigOptions() // Load current config to show current values diff --git a/cli/azd/cmd/env.go b/cli/azd/cmd/env.go index ffeb90a6a49..5f299e8bb91 100644 --- a/cli/azd/cmd/env.go +++ b/cli/azd/cmd/env.go @@ -215,7 +215,6 @@ func newEnvSetAction( } func (e *envSetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("set")) // To track case conflicts dotEnv := e.env.Dotenv() keyValues := make(map[string]string) @@ -367,7 +366,6 @@ type envSetSecretAction struct { } func (e *envSetSecretAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("set-secret")) if len(e.args) < 1 { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrNoArgsProvided, @@ -788,7 +786,6 @@ func newEnvSelectAction( } func (e *envSelectAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("select")) var environmentName string // If no argument provided, prompt the user to select an environment @@ -873,7 +870,6 @@ func newEnvListAction( } func (e *envListAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("list")) envs, err := e.envManager.List(ctx) if err != nil { @@ -975,7 +971,6 @@ func newEnvNewAction( } func (en *envNewAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("new")) environmentName := "" if len(en.args) >= 1 { environmentName = en.args[0] @@ -1150,7 +1145,6 @@ func newEnvRefreshAction( } func (ef *envRefreshAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("refresh")) // Command title ef.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: fmt.Sprintf("Refreshing environment %s (azd env refresh)", ef.env.Name()), @@ -1311,7 +1305,6 @@ func newEnvGetValuesAction( } func (eg *envGetValuesAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("get-values")) name, err := eg.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err @@ -1404,7 +1397,6 @@ func newEnvGetValueAction( } func (eg *envGetValueAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("get-value")) if len(eg.args) < 1 { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrNoKeyNameProvided, @@ -1509,7 +1501,6 @@ func newEnvConfigGetAction( } func (a *envConfigGetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("config-get")) name, err := a.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err @@ -1608,7 +1599,6 @@ func newEnvConfigSetAction( } func (a *envConfigSetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("config-set")) name, err := a.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err @@ -1709,7 +1699,6 @@ func newEnvConfigUnsetAction( } func (a *envConfigUnsetAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("config-unset")) name, err := a.azdCtx.GetDefaultEnvironmentName() if err != nil { return nil, err diff --git a/cli/azd/cmd/env_remove.go b/cli/azd/cmd/env_remove.go index b999ba94a61..05d8e67c16d 100644 --- a/cli/azd/cmd/env_remove.go +++ b/cli/azd/cmd/env_remove.go @@ -12,8 +12,6 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/environment" "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -113,7 +111,6 @@ func newEnvRemoveAction( } func (er *envRemoveAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.EnvOperationKey.String("remove")) // Command title er.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Remove an environment (azd env remove)", diff --git a/cli/azd/cmd/monitor.go b/cli/azd/cmd/monitor.go index 7b9a04eea71..a1eea96ff58 100644 --- a/cli/azd/cmd/monitor.go +++ b/cli/azd/cmd/monitor.go @@ -9,8 +9,6 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/apphost" @@ -102,15 +100,6 @@ func (m *monitorAction) Run(ctx context.Context) (*actions.ActionResult, error) m.flags.monitorOverview = true } - // Track which monitor type was selected - monitorType := "overview" - if m.flags.monitorLive { - monitorType = "live" - } else if m.flags.monitorLogs { - monitorType = "logs" - } - tracing.SetUsageAttributes(fields.MonitorTypeKey.String(monitorType)) - if m.env.GetSubscriptionId() == "" { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrInfraNotProvisioned, diff --git a/cli/azd/cmd/telemetry_coverage_test.go b/cli/azd/cmd/telemetry_coverage_test.go index 1daf5a3c286..6240e59e28e 100644 --- a/cli/azd/cmd/telemetry_coverage_test.go +++ b/cli/azd/cmd/telemetry_coverage_test.go @@ -32,7 +32,7 @@ func TestTelemetryFieldConstants(t *testing.T) { "service-principal-certificate", "federated-github", "federated-azure-pipelines", "federated-oidc", "managed-identity", "external", "oneauth", - "check-status", "logout", + "logout", } for _, method := range authMethods { kv := fields.AuthMethodKey.String(method) @@ -40,39 +40,8 @@ func TestTelemetryFieldConstants(t *testing.T) { } }) - // Auth result telemetry fields - t.Run("AuthResultFields", func(t *testing.T) { - authResults := []string{"success", "failure", "logged-in", "not-logged-in"} - for _, result := range authResults { - kv := fields.AuthResultKey.String(result) - require.Equal(t, "auth.result", string(kv.Key)) - require.Equal(t, result, kv.Value.AsString()) - } - }) - - // Config command telemetry fields - t.Run("ConfigFields", func(t *testing.T) { - operations := []string{"show", "get", "set", "unset", "reset", "list-alpha", "options"} - for _, op := range operations { - kv := fields.ConfigOperationKey.String(op) - require.Equal(t, "config.operation", string(kv.Key)) - require.Equal(t, op, kv.Value.AsString()) - } - }) - // Env command telemetry fields t.Run("EnvFields", func(t *testing.T) { - operations := []string{ - "set", "set-secret", "select", "new", "remove", - "list", "refresh", "get-values", "get-value", - "config-get", "config-set", "config-unset", - } - for _, op := range operations { - kv := fields.EnvOperationKey.String(op) - require.Equal(t, "env.operation", string(kv.Key)) - require.Equal(t, op, kv.Value.AsString()) - } - // Env count is a measurement kvCount := fields.EnvCountKey.Int(3) require.Equal(t, "env.count", string(kvCount.Key)) @@ -88,15 +57,6 @@ func TestTelemetryFieldConstants(t *testing.T) { require.Equal(t, "hooks.type", string(kvType.Key)) }) - // Template command telemetry fields - t.Run("TemplateFields", func(t *testing.T) { - operations := []string{"list", "show", "source-list", "source-add", "source-remove"} - for _, op := range operations { - kv := fields.TemplateOperationKey.String(op) - require.Equal(t, "template.operation", string(kv.Key)) - } - }) - // Pipeline command telemetry fields t.Run("PipelineFields", func(t *testing.T) { kv := fields.PipelineProviderKey.String("github") @@ -106,22 +66,6 @@ func TestTelemetryFieldConstants(t *testing.T) { require.Equal(t, "pipeline.auth", string(kvAuth.Key)) }) - // Monitor command telemetry fields - t.Run("MonitorFields", func(t *testing.T) { - types := []string{"overview", "live", "logs"} - for _, monitorType := range types { - kv := fields.MonitorTypeKey.String(monitorType) - require.Equal(t, "monitor.type", string(kv.Key)) - require.Equal(t, monitorType, kv.Value.AsString()) - } - }) - - // Show command telemetry fields - t.Run("ShowFields", func(t *testing.T) { - kv := fields.ShowOutputFormatKey.String("json") - require.Equal(t, "show.output.format", string(kv.Key)) - }) - // Infra command telemetry fields t.Run("InfraFields", func(t *testing.T) { providers := []string{"bicep", "terraform"} @@ -131,14 +75,6 @@ func TestTelemetryFieldConstants(t *testing.T) { require.Equal(t, provider, kv.Value.AsString()) } }) - - // AccountType Anonymous constant - t.Run("AccountTypeAnonymous", func(t *testing.T) { - require.Equal(t, "Anonymous", fields.AccountTypeAnonymous) - kv := fields.AccountTypeKey.String(fields.AccountTypeAnonymous) - require.Equal(t, "ad.account.type", string(kv.Key)) - require.Equal(t, "Anonymous", kv.Value.AsString()) - }) } // TestCommandTelemetryCoverage ensures every user-facing command is explicitly categorized @@ -160,58 +96,58 @@ func TestCommandTelemetryCoverage(t *testing.T) { // When adding a command here, ensure the command's action sets at least one // command-specific attribute (e.g., auth.method, config.operation, env.operation). commandsWithSpecificTelemetry := []string{ - "auth login", // auth.method, auth.result - "auth logout", // auth.method (logout), auth.result - "auth status", // auth.method (check-status), auth.result - "build", // (via hooks middleware) - "config get", // config.operation - "config list", // config.operation - "config reset", // config.operation - "config set", // config.operation - "config show", // config.operation - "config unset", // config.operation - "deploy", // infra.provider, service attributes (via hooks middleware) - "down", // infra.provider (via hooks middleware) - "env get-value", // env.operation - "env get-values", // env.operation - "env list", // env.operation, env.count - "env new", // env.operation - "env refresh", // env.operation - "env select", // env.operation - "env set", // env.operation - "env set-secret", // env.operation - "hooks run", // hooks.name, hooks.type - "init", // init.method, appinit.* fields - "monitor", // monitor.type - "package", // (via hooks middleware) - "pipeline config", // pipeline.provider, pipeline.auth - "provision", // infra.provider (via hooks middleware) - "restore", // (via hooks middleware) - "show", // show.output.format - "template list", // template.operation - "template show", // template.operation - "template source add", // template.operation - "template source list", // template.operation - "template source remove", // template.operation - "up", // infra.provider (via hooks middleware, composes provision+deploy) - "update", // update.* fields + "auth login", // auth.method + "auth logout", // auth.method (logout) + "build", // (via hooks middleware) + "deploy", // infra.provider, service attributes (via hooks middleware) + "down", // infra.provider (via hooks middleware) + "env list", // env.count + "hooks run", // hooks.name, hooks.type + "init", // init.method, appinit.* fields + "package", // (via hooks middleware) + "pipeline config", // pipeline.provider, pipeline.auth + "provision", // infra.provider (via hooks middleware) + "restore", // (via hooks middleware) + "up", // infra.provider (via hooks middleware, composes provision+deploy) + "update", // update.* fields } // Commands that rely ONLY on global middleware telemetry (command name, flags, // duration, errors) and do NOT emit command-specific attributes. Each entry // includes a justification for why command-specific telemetry is not needed. commandsWithOnlyGlobalTelemetry := []string{ - "completion", // Shell completion script generation — no meaningful usage signal - "config list-alpha", // Simple list of alpha features — no operational variance - "copilot", // Copilot session telemetry handled by copilot.* fields at session level - "env config get", // Thin wrapper — low cardinality, global telemetry sufficient - "env config set", // Thin wrapper — low cardinality, global telemetry sufficient - "env config unset", // Thin wrapper — low cardinality, global telemetry sufficient - "env remove", // Destructive but simple — global telemetry captures usage - "mcp", // MCP tool telemetry handled by mcp.* fields at invocation level - "telemetry", // Meta-command for telemetry itself — avoid recursion - "version", // Telemetry explicitly disabled (DisableTelemetry: true) - "vs-server", // JSON-RPC server — telemetry handled by rpc.* fields per call + "auth status", // Global telemetry sufficient — auth check is simple pass/fail + "completion", // Shell completion script generation — no meaningful usage signal + "config get", // Global telemetry sufficient — low cardinality + "config list", // Global telemetry sufficient — low cardinality + "config list-alpha", // Simple list of alpha features — no operational variance + "config reset", // Global telemetry sufficient — low cardinality + "config set", // Global telemetry sufficient — low cardinality + "config show", // Global telemetry sufficient — low cardinality + "config unset", // Global telemetry sufficient — low cardinality + "copilot", // Copilot session telemetry handled by copilot.* fields at session level + "env config get", // Thin wrapper — low cardinality, global telemetry sufficient + "env config set", // Thin wrapper — low cardinality, global telemetry sufficient + "env config unset", // Thin wrapper — low cardinality, global telemetry sufficient + "env get-value", // Global telemetry sufficient — command name captures operation + "env get-values", // Global telemetry sufficient — command name captures operation + "env new", // Global telemetry sufficient — command name captures operation + "env refresh", // Global telemetry sufficient — command name captures operation + "env remove", // Destructive but simple — global telemetry captures usage + "env select", // Global telemetry sufficient — command name captures operation + "env set", // Global telemetry sufficient — command name captures operation + "env set-secret", // Global telemetry sufficient — command name captures operation + "mcp", // MCP tool telemetry handled by mcp.* fields at invocation level + "monitor", // Global telemetry sufficient — command name captures usage + "show", // Global telemetry sufficient — output format not analytically useful + "telemetry", // Meta-command for telemetry itself — avoid recursion + "template list", // Global telemetry sufficient — command name captures operation + "template show", // Global telemetry sufficient — command name captures operation + "template source add", // Global telemetry sufficient — command name captures operation + "template source list", // Global telemetry sufficient — command name captures operation + "template source remove", // Global telemetry sufficient — command name captures operation + "version", // Telemetry explicitly disabled (DisableTelemetry: true) + "vs-server", // JSON-RPC server — telemetry handled by rpc.* fields per call } // Build lookup maps diff --git a/cli/azd/cmd/templates.go b/cli/azd/cmd/templates.go index f81e3eb225f..5a728a5549e 100644 --- a/cli/azd/cmd/templates.go +++ b/cli/azd/cmd/templates.go @@ -12,8 +12,6 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/input" "github.com/azure/azure-dev/cli/azd/pkg/output" "github.com/azure/azure-dev/cli/azd/pkg/output/ux" @@ -105,7 +103,6 @@ func newTemplateListAction( } func (tl *templateListAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.TemplateOperationKey.String("list")) options := &templates.ListOptions{ Source: tl.flags.source, Tags: tl.flags.tags, @@ -170,7 +167,6 @@ func newTemplateShowAction( } func (a *templateShowAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.TemplateOperationKey.String("show")) matchingTemplate, err := a.templateManager.GetTemplate(ctx, a.path) if err != nil { @@ -325,7 +321,6 @@ func newTemplateSourceListAction( } func (a *templateSourceListAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.TemplateOperationKey.String("source-list")) sourceConfigs, err := a.sourceManager.List(ctx) if err != nil { return nil, fmt.Errorf("failed to list template sources: %w", err) @@ -412,7 +407,6 @@ func newTemplateSourceAddAction( } func (a *templateSourceAddAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.TemplateOperationKey.String("source-add")) a.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Add template source (azd template source add)", }) @@ -510,7 +504,6 @@ func newTemplateSourceRemoveAction( } func (a *templateSourceRemoveAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.TemplateOperationKey.String("source-remove")) a.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: "Remove template source (azd template source remove)", }) diff --git a/cli/azd/internal/cmd/show/show.go b/cli/azd/internal/cmd/show/show.go index d24d54176fb..e9b7d75ca88 100644 --- a/cli/azd/internal/cmd/show/show.go +++ b/cli/azd/internal/cmd/show/show.go @@ -21,8 +21,6 @@ import ( "github.com/azure/azure-dev/cli/azd/cmd/actions" "github.com/azure/azure-dev/cli/azd/internal" "github.com/azure/azure-dev/cli/azd/internal/cmd" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/azapi" @@ -141,7 +139,6 @@ func NewShowAction( } func (s *showAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.ShowOutputFormatKey.String(string(s.formatter.Kind()))) s.console.ShowSpinner(ctx, "Gathering information about your app and its resources...", input.Step) defer s.console.StopSpinner(ctx, "", input.Step) diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index d185b9134b8..5a73962ccf8 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -312,8 +312,6 @@ const ( AccountTypeUser = "User" // A service principal, typically an application. AccountTypeServicePrincipal = "Service Principal" - // An anonymous (unauthenticated) user. - AccountTypeAnonymous = "Anonymous" ) // Auth command related fields @@ -326,38 +324,10 @@ var ( Classification: SystemMetadata, Purpose: FeatureInsight, } - // The result of the auth operation. - // - // Example: "success", "failure" - AuthResultKey = AttributeKey{ - Key: attribute.Key("auth.result"), - Classification: SystemMetadata, - Purpose: FeatureInsight, - } -) - -// Config command related fields -var ( - // The config operation being performed. - // - // Example: "show", "get", "set", "unset", "reset", "list-alpha", "options" - ConfigOperationKey = AttributeKey{ - Key: attribute.Key("config.operation"), - Classification: SystemMetadata, - Purpose: FeatureInsight, - } ) // Environment command related fields var ( - // The environment operation being performed. - // - // Example: "new", "select", "list", "refresh", "set", "get-values" - EnvOperationKey = AttributeKey{ - Key: attribute.Key("env.operation"), - Classification: SystemMetadata, - Purpose: FeatureInsight, - } // The number of environments that exist for the current project. EnvCountKey = AttributeKey{ Key: attribute.Key("env.count"), @@ -383,18 +353,6 @@ var ( } ) -// Template command related fields -var ( - // The template operation being performed. - // - // Example: "list", "show", "source-list", "source-add", "source-remove" - TemplateOperationKey = AttributeKey{ - Key: attribute.Key("template.operation"), - Classification: SystemMetadata, - Purpose: FeatureInsight, - } -) - // Pipeline command related fields var ( // The pipeline provider being configured. @@ -413,28 +371,6 @@ var ( } ) -// Monitor command related fields -var ( - // The type of monitoring dashboard selected. - // - // Example: "overview", "live", "logs" - MonitorTypeKey = AttributeKey{ - Key: attribute.Key("monitor.type"), - Classification: SystemMetadata, - Purpose: FeatureInsight, - } -) - -// Show command related fields -var ( - // The output format requested for the show command. - ShowOutputFormatKey = AttributeKey{ - Key: attribute.Key("show.output.format"), - Classification: SystemMetadata, - Purpose: FeatureInsight, - } -) - // Infrastructure command related fields var ( // The IaC provider used for infrastructure generation. diff --git a/cli/azd/internal/tracing/fields/fields_audit_test.go b/cli/azd/internal/tracing/fields/fields_audit_test.go index fff8900ba76..2eeae30606e 100644 --- a/cli/azd/internal/tracing/fields/fields_audit_test.go +++ b/cli/azd/internal/tracing/fields/fields_audit_test.go @@ -28,29 +28,7 @@ func TestNewFieldConstantsDefined(t *testing.T) { classification: SystemMetadata, purpose: FeatureInsight, }, - { - name: "AuthResultKey", - key: AuthResultKey, - expectedKey: "auth.result", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - // Config fields - { - name: "ConfigOperationKey", - key: ConfigOperationKey, - expectedKey: "config.operation", - classification: SystemMetadata, - purpose: FeatureInsight, - }, // Env fields - { - name: "EnvOperationKey", - key: EnvOperationKey, - expectedKey: "env.operation", - classification: SystemMetadata, - purpose: FeatureInsight, - }, { name: "EnvCountKey", key: EnvCountKey, @@ -74,14 +52,6 @@ func TestNewFieldConstantsDefined(t *testing.T) { classification: SystemMetadata, purpose: FeatureInsight, }, - // Template fields - { - name: "TemplateOperationKey", - key: TemplateOperationKey, - expectedKey: "template.operation", - classification: SystemMetadata, - purpose: FeatureInsight, - }, // Pipeline fields { name: "PipelineProviderKey", @@ -97,22 +67,6 @@ func TestNewFieldConstantsDefined(t *testing.T) { classification: SystemMetadata, purpose: FeatureInsight, }, - // Monitor fields - { - name: "MonitorTypeKey", - key: MonitorTypeKey, - expectedKey: "monitor.type", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - // Show fields - { - name: "ShowOutputFormatKey", - key: ShowOutputFormatKey, - expectedKey: "show.output.format", - classification: SystemMetadata, - purpose: FeatureInsight, - }, // Infra fields { name: "InfraProviderKey", @@ -133,15 +87,6 @@ func TestNewFieldConstantsDefined(t *testing.T) { } } -// TestAccountTypeAnonymousConstant verifies the new Anonymous account type constant. -func TestAccountTypeAnonymousConstant(t *testing.T) { - require.Equal(t, "Anonymous", AccountTypeAnonymous) - // Verify all account types are distinct - require.NotEqual(t, AccountTypeUser, AccountTypeAnonymous) - require.NotEqual(t, AccountTypeServicePrincipal, AccountTypeAnonymous) - require.NotEqual(t, AccountTypeUser, AccountTypeServicePrincipal) -} - // TestFieldKeyValues verifies that field keys produce valid attribute KeyValue pairs. func TestFieldKeyValues(t *testing.T) { // Test string attribute creation diff --git a/cli/azd/pkg/auth/manager.go b/cli/azd/pkg/auth/manager.go index 7ad38c89175..1bebb0313b3 100644 --- a/cli/azd/pkg/auth/manager.go +++ b/cli/azd/pkg/auth/manager.go @@ -481,8 +481,6 @@ func (m *Manager) GetLoggedInServicePrincipalTenantID(ctx context.Context) (*str tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeServicePrincipal)) } else if currentUser.HomeAccountID != nil { tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeUser)) - } else { - tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeAnonymous)) } return currentUser.TenantID, nil diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index a023335e8f5..c6e1eb1e927 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -35,82 +35,90 @@ These commands emit attributes or events beyond the global middleware span. ## Full Inventory Matrix -| Command | Subcommands | Global Span | Command-Specific Attrs | Feature Events | Gap? | Recommended Additions | -|---------|-------------|:-----------:|:----------------------:|:--------------:|:----:|----------------------| -| **Auth** | | | | | | | -| `auth login` | — | ✅ | ❌ | ❌ | **Yes** | `auth.method` (browser, device-code, service-principal-secret, service-principal-certificate, federated-github, federated-azure-pipelines, federated-oidc, managed-identity, external, oneauth), `auth.result` (success/failure) | -| `auth logout` | — | ✅ | ❌ | ❌ | **Yes** | `auth.result` | -| `auth status` | — | ✅ | ❌ | ❌ | **Yes** | `auth.method` (check-status), `auth.result` | -| `auth token` | — | ✅ | ❌ | ❌ | **Yes** | `auth.result` | -| **Config** | | | | | | | -| `config` | `show`, `list`, `get`, `set`, `unset`, `reset`, `list-alpha`, `options` | ✅ | ❌ | ❌ | **Yes** | `config.operation` (show/list/get/set/unset/reset/list-alpha/options) | -| **Environment** | | | | | | | -| `env` | `set`, `set-secret`, `select`, `new`, `remove`, `list`, `refresh`, `get-values`, `get-value` | ✅ | ❌ | ❌ | **Yes** | `env.operation` (set/set-secret/select/new/remove/list/refresh/get-values/get-value), `env.count` (measurement — number of environments) | -| `env config` | `get`, `set`, `unset` | ✅ | ❌ | ❌ | **Yes** | `env.operation` (config-get/config-set/config-unset) | -| **Hooks** | | | | | | | -| `hooks run` | — | ✅ | ❌ | ❌ | **Yes** | `hooks.name`, `hooks.type` (project/service) | -| **Templates** | | | | | | | -| `template` | `list`, `show` | ✅ | ❌ | ❌ | **Yes** | `template.operation` (list/show) | -| `template source` | `list`, `add`, `remove` | ✅ | ❌ | ❌ | **Yes** | `template.operation` (source-list/source-add/source-remove) | -| **Pipeline** | | | | | | | -| `pipeline config` | — | ✅ | ❌ | ❌ | **Yes** | `pipeline.provider` (github/azdo), `pipeline.auth` (federated/client-credentials) | -| **Monitor** | | | | | | | -| `monitor` | — | ✅ | ❌ | ❌ | **Yes** | `monitor.type` (overview/logs/live) | -| **Show** | | | | | | | -| `show` | — | ✅ | ❌ | ❌ | **Yes** | `show.output.format` (json/table/etc.) | -| **Infrastructure** | | | | | | | -| `infra generate` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider` (bicep/terraform) | -| `infra synth` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider` (bicep/terraform) | -| `infra create` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Low | Wraps `provision`; inherits its telemetry once added | -| `infra delete` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Low | Wraps `down`; inherits its telemetry once added | -| **Core Lifecycle** | | | | | | | -| `restore` | — | ✅ | ❌ | ❌ | **Yes** | Service-level attrs (language, host, count) | -| `build` | — | ✅ | ❌ | ❌ | **Yes** | Service-level attrs (language, host, count) | -| `provision` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider`, resource count, duration breakdown | -| `package` | — | ✅ | ❌ | ❌ | **Yes** | Service-level attrs (language, host, count) | -| `deploy` | — | ✅ | ❌ | ❌ | **Yes** | Service host type, target count, deployment strategy | -| `publish` | — | ✅ | ❌ | ❌ | **Yes** | Same as `deploy` (alias behavior) | -| `up` | — | ✅ | ❌ | ❌ | **Yes** | Orchestration attrs: which phases ran, total service count | -| `down` | — | ✅ | ❌ | ❌ | **Yes** | `infra.provider`, resource count, purge flag | -| **Add** | | | | | | | -| `add` | — | ✅ | ❌ | ❌ | **Yes** | Component type added, source (template/manual) | -| **Completion** | | | | | | | -| `completion` | `bash`, `zsh`, `fish`, `powershell`, `fig` | ✅ | ❌ | ❌ | Low | Shell type — low priority, minimal analytical value | -| **VS Server** | | | | | | | -| `vs-server` | — | ✅ | ❌ | ❌ | Low | Long-running RPC; covered by `vsrpc.*` events | -| **Copilot Consent** | | | | | | | -| `copilot consent` | `list`, `revoke`, `grant` | ✅ | ❌ | ❌ | **Yes** | Consent operation type, scope | -| **Extension Management** | | | | | | | -| `extension` | `list`, `show`, `install`, `uninstall`, `upgrade` | ✅ | ❌ | ❌ | **Yes** | `extension.id`, `extension.version`, operation type | -| `extension source` | `list`, `add`, `remove`, `validate` | ✅ | ❌ | ❌ | **Yes** | Source operation type | -| **Init** | | | | | | | -| `init` | — | ✅ | ✅ | ✅ | No | — Already covered | -| **Update** | | | | | | | -| `update` | — | ✅ | ✅ | ✅ | No | — Already covered | -| **MCP** | | | | | | | -| `mcp start` | — | ✅ | ✅ | ✅ | No | — Already covered | -| **Disabled** | | | | | | | -| `version` | — | 🚫 | — | — | No | Intentionally disabled | -| `telemetry upload` | — | 🚫 | — | — | No | Intentionally disabled | - -## Gap Summary - -| Priority | Count | Commands | -|----------|-------|----------| -| **High** | 8 | `auth login/logout/status/token`, `provision`, `deploy`, `up`, `down` | -| **Medium** | 14 | `config *`, `env *`, `pipeline config`, `hooks run`, `template *`, `monitor`, `show`, `infra generate/synth`, `restore`, `build`, `package`, `add` | -| **Low** | 6 | `completion *`, `vs-server`, `infra create/delete` (deprecated), `copilot consent *`, `extension *` management | - -## Implementation Priority - -1. **Phase 1 — Auth & Core Lifecycle**: `auth login`, `provision`, `deploy`, `up`, `down` - — These are the highest-traffic commands with the most analytical value. - -2. **Phase 2 — Config, Env, Pipeline**: `config *`, `env *`, `pipeline config`, `hooks run` - — Understanding user configuration patterns and environment workflows. - -3. **Phase 3 — Templates & Infrastructure**: `template *`, `monitor`, `show`, `infra generate/synth` - — Template discovery and infrastructure generation insights. - -4. **Phase 4 — Remaining**: `restore`, `build`, `package`, `add`, `completion`, extension management - — Lower traffic or lower analytical value. +| Command | Subcommands | Global Span | Command-Specific Attrs | Feature Events | Notes | +|---------|-------------|:-----------:|:----------------------:|:--------------:|-------| +| **Auth** | | | | | | +| `auth login` | — | ✅ | ✅ | ❌ | `auth.method` (browser, device-code, service-principal-secret, etc.) | +| `auth logout` | — | ✅ | ✅ | ❌ | `auth.method` (logout) | +| `auth status` | — | ✅ | ❌ | ❌ | Global telemetry sufficient — simple pass/fail check | +| `auth token` | — | ✅ | ❌ | ❌ | Global telemetry sufficient | +| **Config** | | | | | | +| `config` | `show`, `list`, `get`, `set`, `unset`, `reset`, `list-alpha`, `options` | ✅ | ❌ | ❌ | Redundant — command name in global span captures operation | +| **Environment** | | | | | | +| `env` | `set`, `set-secret`, `select`, `new`, `remove`, `refresh`, `get-values`, `get-value` | ✅ | ❌ | ❌ | Redundant — command name in global span captures operation | +| `env list` | — | ✅ | ✅ | ❌ | `env.count` (measurement — number of environments) | +| `env config` | `get`, `set`, `unset` | ✅ | ❌ | ❌ | Thin wrappers — global telemetry sufficient | +| **Hooks** | | | | | | +| `hooks run` | — | ✅ | ✅ | ❌ | `hooks.name`, `hooks.type` (project/service) | +| **Templates** | | | | | | +| `template` | `list`, `show` | ✅ | ❌ | ❌ | Redundant — command name in global span captures operation | +| `template source` | `list`, `add`, `remove` | ✅ | ❌ | ❌ | Redundant — command name in global span captures operation | +| **Pipeline** | | | | | | +| `pipeline config` | — | ✅ | ✅ | ❌ | `pipeline.provider` (github/azdo), `pipeline.auth` (federated/client-credentials) | +| **Monitor** | | | | | | +| `monitor` | — | ✅ | ❌ | ❌ | Redundant — command name in global span is sufficient | +| **Show** | | | | | | +| `show` | — | ✅ | ❌ | ❌ | Redundant — output format not analytically useful | +| **Infrastructure** | | | | | | +| `infra generate` | — | ✅ | ✅ | ❌ | `infra.provider` (bicep/terraform) | +| `infra synth` | — | ✅ | ✅ | ❌ | `infra.provider` (bicep/terraform) | +| `infra create` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Wraps `provision`; inherits its telemetry | +| `infra delete` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Wraps `down`; inherits its telemetry | +| **Core Lifecycle** | | | | | | +| `restore` | — | ✅ | ❌ | ❌ | Via hooks middleware | +| `build` | — | ✅ | ❌ | ❌ | Via hooks middleware | +| `provision` | — | ✅ | ❌ | ❌ | `infra.provider` set via hooks middleware | +| `package` | — | ✅ | ❌ | ❌ | Via hooks middleware | +| `deploy` | — | ✅ | ❌ | ❌ | `infra.provider`, service attributes via hooks middleware | +| `publish` | — | ✅ | ❌ | ❌ | Same as `deploy` (alias behavior) | +| `up` | — | ✅ | ❌ | ❌ | `infra.provider` via hooks middleware (composes provision+deploy) | +| `down` | — | ✅ | ❌ | ❌ | `infra.provider` via hooks middleware | +| **Add** | | | | | | +| `add` | — | ✅ | ❌ | ❌ | Low priority | +| **Completion** | | | | | | +| `completion` | `bash`, `zsh`, `fish`, `powershell`, `fig` | ✅ | ❌ | ❌ | Low priority — minimal analytical value | +| **VS Server** | | | | | | +| `vs-server` | — | ✅ | ❌ | ❌ | Long-running RPC; covered by `vsrpc.*` events | +| **Copilot Consent** | | | | | | +| `copilot consent` | `list`, `revoke`, `grant` | ✅ | ❌ | ❌ | Low priority | +| **Extension Management** | | | | | | +| `extension` | `list`, `show`, `install`, `uninstall`, `upgrade` | ✅ | ❌ | ❌ | Covered by `extension.*` fields | +| `extension source` | `list`, `add`, `remove`, `validate` | ✅ | ❌ | ❌ | Low priority | +| **Init** | | | | | | +| `init` | — | ✅ | ✅ | ✅ | Comprehensive coverage via `appinit.*` fields | +| **Update** | | | | | | +| `update` | — | ✅ | ✅ | ✅ | Covered by `update.*` fields | +| **MCP** | | | | | | +| `mcp start` | — | ✅ | ✅ | ✅ | Per-tool spans via `mcp.*` | +| **Disabled** | | | | | | +| `version` | — | 🚫 | — | — | Intentionally disabled | +| `telemetry upload` | — | 🚫 | — | — | Intentionally disabled | + +## Retained Fields Summary + +After the redundancy audit (per PR review feedback from @weikanglim), the following +command-specific telemetry fields provide analytical value beyond the command name: + +| Field | OTel Key | Commands | Justification | +|-------|----------|----------|---------------| +| Auth method | `auth.method` | `auth login`, `auth logout` | Distinguishes authentication flow type (browser, device-code, SP, federated, etc.) | +| Env count | `env.count` | `env list` | Measurement — number of environments is a quantitative metric | +| Hooks name | `hooks.name` | `hooks run` | Identifies which hook script ran | +| Hooks type | `hooks.type` | `hooks run` | Distinguishes project vs service hooks | +| Pipeline provider | `pipeline.provider` | `pipeline config` | Distinguishes GitHub vs Azure DevOps | +| Pipeline auth | `pipeline.auth` | `pipeline config` | Distinguishes federated vs client-credentials | +| Infra provider | `infra.provider` | `infra generate`, `infra synth` | Distinguishes Bicep vs Terraform | + +### Removed Fields (Redundant with Command Name) + +The following fields were removed because the command name in the global span already +captures the operation type, making the attribute redundant: + +| Removed Field | Reason | +|---------------|--------| +| `auth.result` | Success/failure already captured by span status | +| `config.operation` | Each config subcommand has its own command name | +| `env.operation` | Each env subcommand has its own command name | +| `template.operation` | Each template subcommand has its own command name | +| `monitor.type` | Single command — no distinguishing value | +| `show.output.format` | Output format not analytically useful | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index f77fa7e49b4..0b88f5aa9a2 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -54,7 +54,7 @@ These are set once at process startup via `resource.New()` and attached to every |-------|----------|----------------|---------|-------| | Object ID | `user_AuthenticatedId` | — | — | From Application Insights contracts | | Tenant ID | `ad.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | -| Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"`, `"Service Principal"`, or `"Anonymous"` | +| Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"` or `"Service Principal"` | | Subscription ID | `ad.subscription.id` | OrganizationalIdentifiableInformation | PerformanceAndHealth | Azure subscription | ### Project Context (azure.yaml) @@ -232,18 +232,12 @@ The following fields are being introduced to close telemetry gaps identified in | Field | OTel Key | Classification | Purpose | Values | |-------|----------|----------------|---------|--------| -| Auth method | `auth.method` | SystemMetadata | FeatureInsight | `browser`, `device-code`, `service-principal-secret`, `service-principal-certificate`, `federated-github`, `federated-azure-pipelines`, `federated-oidc`, `managed-identity`, `external`, `oneauth`, `check-status`, `logout` | -| Auth result | `auth.result` | SystemMetadata | FeatureInsight | `success`, `failure`, `logged-in`, `not-logged-in` | -| Config operation | `config.operation` | SystemMetadata | FeatureInsight | show, list, get, set, unset, reset, list-alpha, options | -| Env operation | `env.operation` | SystemMetadata | FeatureInsight | set, set-secret, select, new, remove, list, refresh, get-values, get-value, config-get, config-set, config-unset | +| Auth method | `auth.method` | SystemMetadata | FeatureInsight | `browser`, `device-code`, `service-principal-secret`, `service-principal-certificate`, `federated-github`, `federated-azure-pipelines`, `federated-oidc`, `managed-identity`, `external`, `oneauth`, `logout` | | Env count | `env.count` | SystemMetadata | FeatureInsight | **Measurement** — number of environments | | Hooks name | `hooks.name` | SystemMetadata | FeatureInsight | Hook script name | | Hooks type | `hooks.type` | SystemMetadata | FeatureInsight | `project`, `service` | -| Template operation | `template.operation` | SystemMetadata | FeatureInsight | list, show, source-list, source-add, source-remove | | Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | `github`, `azdo`, `auto` (auto-detected) | | Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | `federated`, `client-credentials`, `auto` (auto-detected) | -| Monitor type | `monitor.type` | SystemMetadata | FeatureInsight | `overview`, `logs`, `live` | -| Show output format | `show.output.format` | SystemMetadata | FeatureInsight | json, table, etc. | | Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | `bicep`, `terraform`, `auto` (auto-detected from files) | ## Data Classifications From 490374dbbc599fd45d2cd8513d528cd19716276e Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 11:37:13 -0700 Subject: [PATCH 07/14] fix: gofmt formatting in telemetry coverage test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/telemetry_coverage_test.go | 62 +++++++++++++------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/cli/azd/cmd/telemetry_coverage_test.go b/cli/azd/cmd/telemetry_coverage_test.go index 6240e59e28e..ce1d1c61496 100644 --- a/cli/azd/cmd/telemetry_coverage_test.go +++ b/cli/azd/cmd/telemetry_coverage_test.go @@ -116,38 +116,38 @@ func TestCommandTelemetryCoverage(t *testing.T) { // duration, errors) and do NOT emit command-specific attributes. Each entry // includes a justification for why command-specific telemetry is not needed. commandsWithOnlyGlobalTelemetry := []string{ - "auth status", // Global telemetry sufficient — auth check is simple pass/fail - "completion", // Shell completion script generation — no meaningful usage signal - "config get", // Global telemetry sufficient — low cardinality - "config list", // Global telemetry sufficient — low cardinality - "config list-alpha", // Simple list of alpha features — no operational variance - "config reset", // Global telemetry sufficient — low cardinality - "config set", // Global telemetry sufficient — low cardinality - "config show", // Global telemetry sufficient — low cardinality - "config unset", // Global telemetry sufficient — low cardinality - "copilot", // Copilot session telemetry handled by copilot.* fields at session level - "env config get", // Thin wrapper — low cardinality, global telemetry sufficient - "env config set", // Thin wrapper — low cardinality, global telemetry sufficient - "env config unset", // Thin wrapper — low cardinality, global telemetry sufficient - "env get-value", // Global telemetry sufficient — command name captures operation - "env get-values", // Global telemetry sufficient — command name captures operation - "env new", // Global telemetry sufficient — command name captures operation - "env refresh", // Global telemetry sufficient — command name captures operation - "env remove", // Destructive but simple — global telemetry captures usage - "env select", // Global telemetry sufficient — command name captures operation - "env set", // Global telemetry sufficient — command name captures operation - "env set-secret", // Global telemetry sufficient — command name captures operation - "mcp", // MCP tool telemetry handled by mcp.* fields at invocation level - "monitor", // Global telemetry sufficient — command name captures usage - "show", // Global telemetry sufficient — output format not analytically useful - "telemetry", // Meta-command for telemetry itself — avoid recursion - "template list", // Global telemetry sufficient — command name captures operation - "template show", // Global telemetry sufficient — command name captures operation - "template source add", // Global telemetry sufficient — command name captures operation - "template source list", // Global telemetry sufficient — command name captures operation + "auth status", // Global telemetry sufficient — auth check is simple pass/fail + "completion", // Shell completion script generation — no meaningful usage signal + "config get", // Global telemetry sufficient — low cardinality + "config list", // Global telemetry sufficient — low cardinality + "config list-alpha", // Simple list of alpha features — no operational variance + "config reset", // Global telemetry sufficient — low cardinality + "config set", // Global telemetry sufficient — low cardinality + "config show", // Global telemetry sufficient — low cardinality + "config unset", // Global telemetry sufficient — low cardinality + "copilot", // Copilot session telemetry handled by copilot.* fields at session level + "env config get", // Thin wrapper — low cardinality, global telemetry sufficient + "env config set", // Thin wrapper — low cardinality, global telemetry sufficient + "env config unset", // Thin wrapper — low cardinality, global telemetry sufficient + "env get-value", // Global telemetry sufficient — command name captures operation + "env get-values", // Global telemetry sufficient — command name captures operation + "env new", // Global telemetry sufficient — command name captures operation + "env refresh", // Global telemetry sufficient — command name captures operation + "env remove", // Destructive but simple — global telemetry captures usage + "env select", // Global telemetry sufficient — command name captures operation + "env set", // Global telemetry sufficient — command name captures operation + "env set-secret", // Global telemetry sufficient — command name captures operation + "mcp", // MCP tool telemetry handled by mcp.* fields at invocation level + "monitor", // Global telemetry sufficient — command name captures usage + "show", // Global telemetry sufficient — output format not analytically useful + "telemetry", // Meta-command for telemetry itself — avoid recursion + "template list", // Global telemetry sufficient — command name captures operation + "template show", // Global telemetry sufficient — command name captures operation + "template source add", // Global telemetry sufficient — command name captures operation + "template source list", // Global telemetry sufficient — command name captures operation "template source remove", // Global telemetry sufficient — command name captures operation - "version", // Telemetry explicitly disabled (DisableTelemetry: true) - "vs-server", // JSON-RPC server — telemetry handled by rpc.* fields per call + "version", // Telemetry explicitly disabled (DisableTelemetry: true) + "vs-server", // JSON-RPC server — telemetry handled by rpc.* fields per call } // Build lookup maps From 193ebf07cb648041f30eae9e8e8c21764c334625 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 11:52:26 -0700 Subject: [PATCH 08/14] fix: add weikanglim to cspell word list Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .vscode/cspell.misc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/cspell.misc.yaml b/.vscode/cspell.misc.yaml index 45e912c422e..9b6242d0f58 100644 --- a/.vscode/cspell.misc.yaml +++ b/.vscode/cspell.misc.yaml @@ -35,6 +35,7 @@ overrides: - Angelos - Entra - CODEOWNERS + - weikanglim - filename: ./README.md words: - VSIX From e4f0425a628c3911cb34414c3d49e8898e7502b7 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:52:59 -0700 Subject: [PATCH 09/14] fix: log resolved pipeline provider instead of 'auto' sentinel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use CiProviderName() to log the actual resolved provider name after auto-detection instead of the 'auto' placeholder. For auth type, only log when explicitly specified — cmd.flags absence indicates auto-detection. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/pipeline.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/cli/azd/cmd/pipeline.go b/cli/azd/cmd/pipeline.go index 9a46825d9ec..5c2560ef30f 100644 --- a/cli/azd/cmd/pipeline.go +++ b/cli/azd/cmd/pipeline.go @@ -164,19 +164,6 @@ func newPipelineConfigAction( // Run implements action interface func (p *pipelineConfigAction) Run(ctx context.Context) (*actions.ActionResult, error) { - // Track pipeline provider and auth type - // Emit "auto" when the user didn't specify, so we know auto-detection was used. - if p.flags.PipelineProvider != "" { - tracing.SetUsageAttributes(fields.PipelineProviderKey.String(p.flags.PipelineProvider)) - } else { - tracing.SetUsageAttributes(fields.PipelineProviderKey.String("auto")) - } - if p.flags.PipelineAuthTypeName != "" { - tracing.SetUsageAttributes(fields.PipelineAuthKey.String(p.flags.PipelineAuthTypeName)) - } else { - tracing.SetUsageAttributes(fields.PipelineAuthKey.String("auto")) - } - infra, err := p.importManager.ProjectInfrastructure(ctx, p.projectConfig) if err != nil { return nil, err @@ -185,6 +172,13 @@ func (p *pipelineConfigAction) Run(ctx context.Context) (*actions.ActionResult, // Command title pipelineProviderName := p.manager.CiProviderName() + + // Track the resolved pipeline provider (after CiProviderName resolves auto-detection). + // cmd.flags already indicates whether --provider was explicitly set by the user. + tracing.SetUsageAttributes(fields.PipelineProviderKey.String(pipelineProviderName)) + if p.flags.PipelineAuthTypeName != "" { + tracing.SetUsageAttributes(fields.PipelineAuthKey.String(p.flags.PipelineAuthTypeName)) + } p.console.MessageUxItem(ctx, &ux.MessageTitle{ Title: fmt.Sprintf("Configure your %s pipeline", pipelineProviderName), }) From 5441e88cadf355f987e1c945e86345dd0d50caf7 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Wed, 25 Mar 2026 15:36:57 -0700 Subject: [PATCH 10/14] fix: address Wei pass-2 review feedback - Remove 'logout' as auth.method value (not an auth method) - Unhash tenant ID (infrastructure GUID, not PII) - Revert unrelated cosmetic change in auth_status.go - Revert unrelated if/else logic change in manager.go - Remove redundant TestFieldKeyValues test - Rename tenant key from ad.tenant.id to auth.tenant.id Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/auth_login.go | 4 ++-- cli/azd/cmd/auth_logout.go | 4 ---- cli/azd/cmd/auth_status.go | 8 +++----- cli/azd/cmd/telemetry_coverage_test.go | 3 +-- cli/azd/internal/tracing/fields/fields.go | 2 +- .../internal/tracing/fields/fields_audit_test.go | 13 ------------- cli/azd/pkg/auth/manager.go | 4 +++- 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/cli/azd/cmd/auth_login.go b/cli/azd/cmd/auth_login.go index 435009ccb07..66459ea7012 100644 --- a/cli/azd/cmd/auth_login.go +++ b/cli/azd/cmd/auth_login.go @@ -455,9 +455,9 @@ func runningOnCodespacesBrowser(ctx context.Context, commandRunner exec.CommandR } func (la *loginAction) login(ctx context.Context) error { - // Track hashed tenant ID if provided (before resolving from env vars) + // Track tenant ID if provided (before resolving from env vars) if la.flags.tenantID != "" { - tracing.SetUsageAttributes(fields.StringHashed(fields.TenantIdKey, la.flags.tenantID)) + tracing.SetUsageAttributes(fields.TenantIdKey.String(la.flags.tenantID)) } if la.flags.federatedTokenProvider == azurePipelinesProvider { diff --git a/cli/azd/cmd/auth_logout.go b/cli/azd/cmd/auth_logout.go index 9482479d7ea..4cb7b874886 100644 --- a/cli/azd/cmd/auth_logout.go +++ b/cli/azd/cmd/auth_logout.go @@ -9,8 +9,6 @@ import ( "io" "github.com/azure/azure-dev/cli/azd/cmd/actions" - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/auth" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -56,8 +54,6 @@ func newLogoutAction( } func (la *logoutAction) Run(ctx context.Context) (*actions.ActionResult, error) { - tracing.SetUsageAttributes(fields.AuthMethodKey.String("logout")) - if la.annotations[loginCmdParentAnnotation] == "" { fmt.Fprintln( la.console.Handles().Stderr, diff --git a/cli/azd/cmd/auth_status.go b/cli/azd/cmd/auth_status.go index 927a8008c43..b27dacc334a 100644 --- a/cli/azd/cmd/auth_status.go +++ b/cli/azd/cmd/auth_status.go @@ -100,11 +100,9 @@ func (a *authStatusAction) Run(ctx context.Context) (*actions.ActionResult, erro if err != nil { res.Status = contracts.AuthStatusUnauthenticated log.Printf("error: verifying logged in status: %v", err) - } else { - if token != nil { - expiresOn := contracts.RFC3339Time(token.ExpiresOn) - res.ExpiresOn = &expiresOn - } + } else if token != nil { + expiresOn := contracts.RFC3339Time(token.ExpiresOn) + res.ExpiresOn = &expiresOn } switch details.LoginType { diff --git a/cli/azd/cmd/telemetry_coverage_test.go b/cli/azd/cmd/telemetry_coverage_test.go index ce1d1c61496..87b303b71e4 100644 --- a/cli/azd/cmd/telemetry_coverage_test.go +++ b/cli/azd/cmd/telemetry_coverage_test.go @@ -32,7 +32,6 @@ func TestTelemetryFieldConstants(t *testing.T) { "service-principal-certificate", "federated-github", "federated-azure-pipelines", "federated-oidc", "managed-identity", "external", "oneauth", - "logout", } for _, method := range authMethods { kv := fields.AuthMethodKey.String(method) @@ -97,7 +96,6 @@ func TestCommandTelemetryCoverage(t *testing.T) { // command-specific attribute (e.g., auth.method, config.operation, env.operation). commandsWithSpecificTelemetry := []string{ "auth login", // auth.method - "auth logout", // auth.method (logout) "build", // (via hooks middleware) "deploy", // infra.provider, service attributes (via hooks middleware) "down", // infra.provider (via hooks middleware) @@ -116,6 +114,7 @@ func TestCommandTelemetryCoverage(t *testing.T) { // duration, errors) and do NOT emit command-specific attributes. Each entry // includes a justification for why command-specific telemetry is not needed. commandsWithOnlyGlobalTelemetry := []string{ + "auth logout", // No command-specific telemetry — logout is a simple operation "auth status", // Global telemetry sufficient — auth check is simple pass/fail "completion", // Shell completion script generation — no meaningful usage signal "config get", // Global telemetry sufficient — low cardinality diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index 5a73962ccf8..c76d5062146 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -139,7 +139,7 @@ var ( ObjectIdKey = attribute.Key(contracts.UserAuthUserId) // user_AuthenticatedId // Tenant ID of the principal. TenantIdKey = AttributeKey{ - Key: attribute.Key("ad.tenant.id"), + Key: attribute.Key("auth.tenant.id"), Classification: SystemMetadata, Purpose: BusinessInsight, } diff --git a/cli/azd/internal/tracing/fields/fields_audit_test.go b/cli/azd/internal/tracing/fields/fields_audit_test.go index 2eeae30606e..70c813277d0 100644 --- a/cli/azd/internal/tracing/fields/fields_audit_test.go +++ b/cli/azd/internal/tracing/fields/fields_audit_test.go @@ -86,16 +86,3 @@ func TestNewFieldConstantsDefined(t *testing.T) { }) } } - -// TestFieldKeyValues verifies that field keys produce valid attribute KeyValue pairs. -func TestFieldKeyValues(t *testing.T) { - // Test string attribute creation - kv := AuthMethodKey.String("browser") - require.Equal(t, "auth.method", string(kv.Key)) - require.Equal(t, "browser", kv.Value.AsString()) - - // Test int attribute creation - kvInt := EnvCountKey.Int(5) - require.Equal(t, "env.count", string(kvInt.Key)) - require.Equal(t, int64(5), kvInt.Value.AsInt64()) -} diff --git a/cli/azd/pkg/auth/manager.go b/cli/azd/pkg/auth/manager.go index 1bebb0313b3..46c2345b974 100644 --- a/cli/azd/pkg/auth/manager.go +++ b/cli/azd/pkg/auth/manager.go @@ -479,7 +479,9 @@ func (m *Manager) GetLoggedInServicePrincipalTenantID(ctx context.Context) (*str // Record type of account found if currentUser.TenantID != nil { tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeServicePrincipal)) - } else if currentUser.HomeAccountID != nil { + } + + if currentUser.HomeAccountID != nil { tracing.SetGlobalAttributes(fields.AccountTypeKey.String(fields.AccountTypeUser)) } From 6f208474d0e30580be509ee062ef36dbbe6db92f Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 26 Mar 2026 08:27:30 -0700 Subject: [PATCH 11/14] ci: re-trigger pipeline (Mac build flake) From 16a06daf5732efe628d296bb98792234b6296e30 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 26 Mar 2026 09:37:45 -0700 Subject: [PATCH 12/14] fix: address spboyer review - schema doc mismatch, missing infra generate, undocumented check-status - Update telemetry-schema.md: ad.tenant.id -> auth.tenant.id to match code - Add check-status to auth.method allowed values, remove stale logout - Add missing 'infra generate' to commandsWithSpecificTelemetry manifest Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/telemetry_coverage_test.go | 1 + docs/specs/metrics-audit/telemetry-schema.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/azd/cmd/telemetry_coverage_test.go b/cli/azd/cmd/telemetry_coverage_test.go index 87b303b71e4..3aaef855bcf 100644 --- a/cli/azd/cmd/telemetry_coverage_test.go +++ b/cli/azd/cmd/telemetry_coverage_test.go @@ -101,6 +101,7 @@ func TestCommandTelemetryCoverage(t *testing.T) { "down", // infra.provider (via hooks middleware) "env list", // env.count "hooks run", // hooks.name, hooks.type + "infra generate", // infra.provider "init", // init.method, appinit.* fields "package", // (via hooks middleware) "pipeline config", // pipeline.provider, pipeline.auth diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 0b88f5aa9a2..4bfb48e7049 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -53,7 +53,7 @@ These are set once at process startup via `resource.New()` and attached to every | Field | OTel Key | Classification | Purpose | Notes | |-------|----------|----------------|---------|-------| | Object ID | `user_AuthenticatedId` | — | — | From Application Insights contracts | -| Tenant ID | `ad.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | +| Tenant ID | `auth.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | | Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"` or `"Service Principal"` | | Subscription ID | `ad.subscription.id` | OrganizationalIdentifiableInformation | PerformanceAndHealth | Azure subscription | @@ -232,7 +232,7 @@ The following fields are being introduced to close telemetry gaps identified in | Field | OTel Key | Classification | Purpose | Values | |-------|----------|----------------|---------|--------| -| Auth method | `auth.method` | SystemMetadata | FeatureInsight | `browser`, `device-code`, `service-principal-secret`, `service-principal-certificate`, `federated-github`, `federated-azure-pipelines`, `federated-oidc`, `managed-identity`, `external`, `oneauth`, `logout` | +| Auth method | `auth.method` | SystemMetadata | FeatureInsight | `browser`, `device-code`, `service-principal-secret`, `service-principal-certificate`, `federated-github`, `federated-azure-pipelines`, `federated-oidc`, `managed-identity`, `external`, `oneauth`, `check-status` | | Env count | `env.count` | SystemMetadata | FeatureInsight | **Measurement** — number of environments | | Hooks name | `hooks.name` | SystemMetadata | FeatureInsight | Hook script name | | Hooks type | `hooks.type` | SystemMetadata | FeatureInsight | `project`, `service` | From 6e941d879839a0c00ed913a756849c77a232a558 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 26 Mar 2026 10:10:25 -0700 Subject: [PATCH 13/14] fix: validate hook names before logging to telemetry Known built-in hook names (pre/post build, deploy, etc.) are logged raw. Unknown/extension-defined hook names are hashed via SHA-256 to avoid logging arbitrary user input as customer content. Addresses review feedback from @weikanglim on hook name telemetry. Tracks extension hook telemetry gap in issue #7326. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/hooks.go | 30 +++++++++++++++++++- docs/specs/metrics-audit/telemetry-schema.md | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cli/azd/cmd/hooks.go b/cli/azd/cmd/hooks.go index 443654eebb2..1cc582a05f1 100644 --- a/cli/azd/cmd/hooks.go +++ b/cli/azd/cmd/hooks.go @@ -117,6 +117,28 @@ const ( hookContextService hookContextType = "service" ) +// knownHookNames is the set of built-in azd hook names. +// Extension-defined hooks are not included here; they are hashed in telemetry. +// See https://github.com/Azure/azure-dev/issues/7348 for tracking. +var knownHookNames = map[string]bool{ + "prebuild": true, + "postbuild": true, + "predeploy": true, + "postdeploy": true, + "predown": true, + "postdown": true, + "prepackage": true, + "postpackage": true, + "preprovision": true, + "postprovision": true, + "prepublish": true, + "postpublish": true, + "prerestore": true, + "postrestore": true, + "preup": true, + "postup": true, +} + func (hra *hooksRunAction) Run(ctx context.Context) (*actions.ActionResult, error) { hookName := hra.args[0] @@ -124,8 +146,14 @@ func (hra *hooksRunAction) Run(ctx context.Context) (*actions.ActionResult, erro if hra.flags.service != "" { hookType = "service" } + + // Log known hook names raw; hash unknown names to avoid logging arbitrary user input. + hookNameAttr := fields.StringHashed(fields.HooksNameKey, hookName) + if knownHookNames[hookName] { + hookNameAttr = fields.HooksNameKey.String(hookName) + } tracing.SetUsageAttributes( - fields.HooksNameKey.String(hookName), + hookNameAttr, fields.HooksTypeKey.String(hookType), ) diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 4bfb48e7049..f993db3de82 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -234,7 +234,7 @@ The following fields are being introduced to close telemetry gaps identified in |-------|----------|----------------|---------|--------| | Auth method | `auth.method` | SystemMetadata | FeatureInsight | `browser`, `device-code`, `service-principal-secret`, `service-principal-certificate`, `federated-github`, `federated-azure-pipelines`, `federated-oidc`, `managed-identity`, `external`, `oneauth`, `check-status` | | Env count | `env.count` | SystemMetadata | FeatureInsight | **Measurement** — number of environments | -| Hooks name | `hooks.name` | SystemMetadata | FeatureInsight | Hook script name | +| Hooks name | `hooks.name` | SystemMetadata | FeatureInsight | Built-in hook name (raw) or SHA-256 hash for extension/custom hooks. Known values: `prebuild`, `postbuild`, `predeploy`, `postdeploy`, `predown`, `postdown`, `prepackage`, `postpackage`, `preprovision`, `postprovision`, `prepublish`, `postpublish`, `prerestore`, `postrestore`, `preup`, `postup` | | Hooks type | `hooks.type` | SystemMetadata | FeatureInsight | `project`, `service` | | Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | `github`, `azdo`, `auto` (auto-detected) | | Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | `federated`, `client-credentials`, `auto` (auto-detected) | From a845ab7dbf6ea851e08ea576a70bac8f298e2e5f Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 26 Mar 2026 10:23:50 -0700 Subject: [PATCH 14/14] fix: remove audit test and revert TenantIdKey to ad.tenant.id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove fields_audit_test.go per Wei's feedback — test duplicated field constants without clear value; snapshot testing would be preferred. - Revert TenantIdKey from auth.tenant.id back to ad.tenant.id to avoid data contract change on existing context-level field. - Update telemetry-schema.md to match. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/internal/tracing/fields/fields.go | 2 +- .../tracing/fields/fields_audit_test.go | 88 ------------------- docs/specs/metrics-audit/telemetry-schema.md | 2 +- 3 files changed, 2 insertions(+), 90 deletions(-) delete mode 100644 cli/azd/internal/tracing/fields/fields_audit_test.go diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index c76d5062146..5a73962ccf8 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -139,7 +139,7 @@ var ( ObjectIdKey = attribute.Key(contracts.UserAuthUserId) // user_AuthenticatedId // Tenant ID of the principal. TenantIdKey = AttributeKey{ - Key: attribute.Key("auth.tenant.id"), + Key: attribute.Key("ad.tenant.id"), Classification: SystemMetadata, Purpose: BusinessInsight, } diff --git a/cli/azd/internal/tracing/fields/fields_audit_test.go b/cli/azd/internal/tracing/fields/fields_audit_test.go deleted file mode 100644 index 70c813277d0..00000000000 --- a/cli/azd/internal/tracing/fields/fields_audit_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package fields - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -// TestNewFieldConstantsDefined verifies that all new telemetry field constants added -// as part of the metrics audit are properly defined with correct metadata. -func TestNewFieldConstantsDefined(t *testing.T) { - tests := []struct { - name string - key AttributeKey - expectedKey string - classification Classification - purpose Purpose - isMeasurement bool - }{ - // Auth fields - { - name: "AuthMethodKey", - key: AuthMethodKey, - expectedKey: "auth.method", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - // Env fields - { - name: "EnvCountKey", - key: EnvCountKey, - expectedKey: "env.count", - classification: SystemMetadata, - purpose: FeatureInsight, - isMeasurement: true, - }, - // Hooks fields - { - name: "HooksNameKey", - key: HooksNameKey, - expectedKey: "hooks.name", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - { - name: "HooksTypeKey", - key: HooksTypeKey, - expectedKey: "hooks.type", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - // Pipeline fields - { - name: "PipelineProviderKey", - key: PipelineProviderKey, - expectedKey: "pipeline.provider", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - { - name: "PipelineAuthKey", - key: PipelineAuthKey, - expectedKey: "pipeline.auth", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - // Infra fields - { - name: "InfraProviderKey", - key: InfraProviderKey, - expectedKey: "infra.provider", - classification: SystemMetadata, - purpose: FeatureInsight, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.expectedKey, string(tt.key.Key), "Key value mismatch") - require.Equal(t, tt.classification, tt.key.Classification, "Classification mismatch") - require.Equal(t, tt.purpose, tt.key.Purpose, "Purpose mismatch") - require.Equal(t, tt.isMeasurement, tt.key.IsMeasurement, "IsMeasurement mismatch") - }) - } -} diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index f993db3de82..2d0e22e6a63 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -53,7 +53,7 @@ These are set once at process startup via `resource.New()` and attached to every | Field | OTel Key | Classification | Purpose | Notes | |-------|----------|----------------|---------|-------| | Object ID | `user_AuthenticatedId` | — | — | From Application Insights contracts | -| Tenant ID | `auth.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | +| Tenant ID | `ad.tenant.id` | SystemMetadata | BusinessInsight | Entra ID tenant | | Account type | `ad.account.type` | SystemMetadata | BusinessInsight | `"User"` or `"Service Principal"` | | Subscription ID | `ad.subscription.id` | OrganizationalIdentifiableInformation | PerformanceAndHealth | Azure subscription |