Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cli/azd/cmd/telemetry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,13 @@ func TestTelemetryFieldConstants(t *testing.T) {
}
})

t.Run("HostedAgentFields", func(t *testing.T) {
t.Parallel()
kv := fields.AgentDeploymentModeKey.String("code")
require.Equal(t, "agent.deploy.mode", string(kv.Key))
require.Equal(t, "code", kv.Value.AsString())
})
Comment on lines +86 to +91

// Tool command telemetry fields
t.Run("ToolFields", func(t *testing.T) {
t.Parallel()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ import (

// Reference implementation

const (
agentDeploymentModeArtifactMetadataKey = "azure.ai.agents.deploymentMode"
agentDeploymentModeCode = "code"
agentDeploymentModeContainer = "container"
agentDeploymentModeImage = "byo_image"
)

// displayableProtocolEntry defines a protocol that produces user-visible invocation endpoints.
type displayableProtocolEntry struct {
Protocol agent_api.AgentProtocol
Expand Down Expand Up @@ -526,8 +533,9 @@ func (p *AgentServiceTargetProvider) Package(
Location: zipPath,
LocationKind: azdext.LocationKind_LOCATION_KIND_LOCAL,
Metadata: map[string]string{
"type": "code-zip",
"sha256": sha256Hex,
"type": "code-zip",
"sha256": sha256Hex,
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeCode,
},
},
},
Expand Down Expand Up @@ -600,6 +608,8 @@ func (p *AgentServiceTargetProvider) Package(
newArtifacts = append(newArtifacts, packageResponse.Result.Artifacts...)
}

setAgentDeploymentMode(newArtifacts, agentDeploymentModeContainer)

return &azdext.ServicePackageResult{
Artifacts: newArtifacts,
}, nil
Expand Down Expand Up @@ -651,11 +661,22 @@ func (p *AgentServiceTargetProvider) Publish(
return nil, classifyContainerPublishError(err)
}

setAgentDeploymentMode(publishResponse.Result.Artifacts, agentDeploymentModeContainer)

return &azdext.ServicePublishResult{
Artifacts: publishResponse.Result.Artifacts,
}, nil
}

func setAgentDeploymentMode(artifacts []*azdext.Artifact, mode string) {
for _, artifact := range artifacts {
if artifact.Metadata == nil {
artifact.Metadata = map[string]string{}
}
artifact.Metadata[agentDeploymentModeArtifactMetadataKey] = mode
}
}

func classifyContainerPublishError(err error) error {
if isPrivateACRNetworkAccessError(err) {
return exterrors.Dependency(
Expand Down Expand Up @@ -916,7 +937,8 @@ func preBuiltImageArtifact(imageURL string) *azdext.Artifact {
Location: imageURL,
LocationKind: azdext.LocationKind_LOCATION_KIND_REMOTE,
Metadata: map[string]string{
preBuiltImageArtifactSourceKey: preBuiltImageArtifactSource,
preBuiltImageArtifactSourceKey: preBuiltImageArtifactSource,
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeImage,
},
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1148,6 +1148,11 @@ func TestPackage_SkipsWhenPreBuiltImageChosen(t *testing.T) {
require.Equal(t, imageURL, result.Artifacts[0].Location)
require.Equal(t, azdext.LocationKind_LOCATION_KIND_REMOTE, result.Artifacts[0].LocationKind)
require.Equal(t, preBuiltImageArtifactSource, result.Artifacts[0].Metadata[preBuiltImageArtifactSourceKey])
require.Equal(
t,
agentDeploymentModeImage,
result.Artifacts[0].Metadata[agentDeploymentModeArtifactMetadataKey],
)
require.Contains(t, progressMessages, "Using pre-built container image, skipping package")
}

Expand Down Expand Up @@ -1180,6 +1185,11 @@ func TestPackage_BuildsWhenUserChoseDockerfile(t *testing.T) {
require.Equal(t, int32(1), promptStub.selectCalls.Load())
require.Equal(t, int32(1), containerStub.buildCalls.Load())
require.Equal(t, int32(1), containerStub.packageCalls.Load())
require.Equal(
t,
agentDeploymentModeContainer,
result.Artifacts[0].Metadata[agentDeploymentModeArtifactMetadataKey],
)
}

func TestPublish_SkipsWhenPreBuiltImageChosen(t *testing.T) {
Expand All @@ -1205,6 +1215,11 @@ func TestPublish_SkipsWhenPreBuiltImageChosen(t *testing.T) {
require.NotNil(t, result)
require.Len(t, result.Artifacts, 1)
require.Equal(t, imageURL, result.Artifacts[0].Location)
require.Equal(
t,
agentDeploymentModeImage,
result.Artifacts[0].Metadata[agentDeploymentModeArtifactMetadataKey],
)
require.Contains(t, progressMessages, "Using pre-built container image, skipping publish")
}

Expand Down Expand Up @@ -1239,6 +1254,11 @@ func TestPublish_PublishesWhenPackageBuiltFromDockerfile(t *testing.T) {
require.NotNil(t, result)
require.NotEmpty(t, result.Artifacts, "expected published container artifacts")
require.Equal(t, int32(1), containerStub.publishCalls.Load())
require.Equal(
t,
agentDeploymentModeContainer,
result.Artifacts[0].Metadata[agentDeploymentModeArtifactMetadataKey],
)
}

func TestPublish_PrivateACRNetworkAccessGuidance(t *testing.T) {
Expand Down
6 changes: 6 additions & 0 deletions cli/azd/internal/tracing/fields/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,12 @@ var (
Classification: SystemMetadata,
Purpose: PerformanceAndHealth,
}
// The deployment mode selected for a hosted agent.
AgentDeploymentModeKey = AttributeKey{
Key: attribute.Key("agent.deploy.mode"),
Classification: SystemMetadata,
Purpose: FeatureInsight,
}
)

// Platform related attributes for integrations like devcenter / ADE
Expand Down
66 changes: 66 additions & 0 deletions cli/azd/pkg/project/service_target_external.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"log"

"github.com/azure/azure-dev/cli/azd/internal/mapper"
"github.com/azure/azure-dev/cli/azd/internal/tracing"
"github.com/azure/azure-dev/cli/azd/internal/tracing/fields"
"github.com/azure/azure-dev/cli/azd/pkg/async"
"github.com/azure/azure-dev/cli/azd/pkg/azdext"
"github.com/azure/azure-dev/cli/azd/pkg/environment"
Expand Down Expand Up @@ -125,6 +127,8 @@ func (est *ExternalServiceTarget) Publish(
return nil, fmt.Errorf("failed to convert publish result: %w", err)
}

recordHostedAgentDeploymentMode(ctx, est.extension, result.Artifacts, serviceContext.Package)

return result, nil
}

Expand Down Expand Up @@ -204,6 +208,8 @@ func (est *ExternalServiceTarget) Package(
return nil, err
}

recordHostedAgentDeploymentMode(ctx, est.extension, convertedResult.Artifacts, serviceContext.Package)

return convertedResult, nil
}

Expand Down Expand Up @@ -412,3 +418,63 @@ func createProgressFunc(progress *async.Progress[ServiceProgress]) func(string)
}
}
}

const (
hostedAgentExtensionID = "azure.ai.agents"
agentDeploymentModeArtifactMetadataKey = "azure.ai.agents.deploymentMode"
agentDeploymentModeCode = "code"
agentDeploymentModeContainer = "container"
agentDeploymentModeImage = "byo_image"
)

func recordHostedAgentDeploymentMode(
ctx context.Context,
extension *extensions.Extension,
resultArtifacts ArtifactCollection,
packageArtifacts ArtifactCollection,
) {
if extension == nil || extension.Id != hostedAgentExtensionID {
return
}

mode, hasMetadata := hostedAgentDeploymentMode(resultArtifacts)
if !hasMetadata {
mode, _ = hostedAgentDeploymentMode(packageArtifacts)
}

if mode != "" {
tracing.SetAttributesInContext(ctx, fields.AgentDeploymentModeKey.String(mode))
}
}

func hostedAgentDeploymentMode(artifacts ArtifactCollection) (string, bool) {
hasModeMetadata := false
for _, artifact := range artifacts {
if artifact == nil {
continue
}

mode, ok := artifact.Metadata[agentDeploymentModeArtifactMetadataKey]
if !ok {
continue
}
hasModeMetadata = true

switch mode {
case agentDeploymentModeCode, agentDeploymentModeContainer, agentDeploymentModeImage:
return mode, true
}
}

if hasModeMetadata {
return "", true
}

for _, artifact := range artifacts {
if artifact != nil && artifact.Kind == ArtifactKindContainer {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This container-kind fallback records container for any unlabeled container artifact. Combined with the extension-side gap noted on service_target_agent.go, a user-provided image that reaches core as an unlabeled container artifact (for example the --from-package flow, which bypasses the extension's Package labeling) gets recorded as container rather than byo_image. Fixing only the extension's Publish branch won't fully close this, since this inference can't tell a built image from a provided one. If the from-package provenance is available upstream, carrying it through would let this fallback select byo_image.

return agentDeploymentModeContainer, true
}
}

return "", false
}
123 changes: 123 additions & 0 deletions cli/azd/pkg/project/service_target_external_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package project

import (
"context"
"testing"

"github.com/azure/azure-dev/cli/azd/internal/tracing/fields"
"github.com/azure/azure-dev/cli/azd/pkg/extensions"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)

func TestRecordHostedAgentDeploymentMode(t *testing.T) {
tests := []struct {
name string
extension *extensions.Extension
resultArtifacts ArtifactCollection
packageArtifacts ArtifactCollection
wantMode string
}{
{
name: "code",
extension: &extensions.Extension{Id: hostedAgentExtensionID},
resultArtifacts: ArtifactCollection{{
Metadata: map[string]string{
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeCode,
},
}},
wantMode: agentDeploymentModeCode,
},
{
name: "container",
extension: &extensions.Extension{Id: hostedAgentExtensionID},
resultArtifacts: ArtifactCollection{{
Metadata: map[string]string{
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeContainer,
},
}},
wantMode: agentDeploymentModeContainer,
},
{
name: "bring your own image",
extension: &extensions.Extension{Id: hostedAgentExtensionID},
resultArtifacts: ArtifactCollection{{
Metadata: map[string]string{
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeImage,
},
}},
wantMode: agentDeploymentModeImage,
},
{
name: "package artifact",
extension: &extensions.Extension{Id: hostedAgentExtensionID},
packageArtifacts: ArtifactCollection{{
Metadata: map[string]string{
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeImage,
},
}},
wantMode: agentDeploymentModeImage,
},
{
name: "unknown mode",
extension: &extensions.Extension{Id: hostedAgentExtensionID},
resultArtifacts: ArtifactCollection{{
Kind: ArtifactKindContainer,
Metadata: map[string]string{
agentDeploymentModeArtifactMetadataKey: "unknown",
},
}},
},
{
name: "unknown key",
extension: &extensions.Extension{Id: hostedAgentExtensionID},
resultArtifacts: ArtifactCollection{{
Metadata: map[string]string{
"unknown": agentDeploymentModeCode,
},
}},
},
{
name: "other extension",
extension: &extensions.Extension{Id: "other"},
resultArtifacts: ArtifactCollection{{
Metadata: map[string]string{
agentDeploymentModeArtifactMetadataKey: agentDeploymentModeCode,
},
}},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
recorder := tracetest.NewSpanRecorder()
provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
t.Cleanup(func() {
require.NoError(t, provider.Shutdown(context.WithoutCancel(t.Context())))
})

ctx, span := provider.Tracer("test").Start(t.Context(), "exegraph.step")
recordHostedAgentDeploymentMode(
ctx,
tt.extension,
tt.resultArtifacts,
tt.packageArtifacts,
)
span.End()

spans := recorder.Ended()
require.Len(t, spans, 1)

if tt.wantMode == "" {
require.Empty(t, spans[0].Attributes())
return
}

require.Contains(t, spans[0].Attributes(), fields.AgentDeploymentModeKey.String(tt.wantMode))
})
}
}
6 changes: 6 additions & 0 deletions docs/reference/telemetry-data.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,12 @@ These are set once at process startup and attached to **every** span.
| `project.service.language` | string | ❌ | Language of specific service being executed — see [Service Languages](#service-languages) |
| `platform.type` | string | ❌ | Platform integration (e.g., `aca`, `aks`) |

### Hosted Agent Fields

| Field Key | Type | Hashed? | Description |
|-----------|------|---------|-------------|
| `agent.deploy.mode` | string | ❌ | Selected hosted-agent deployment path: `code`, `container`, or `byo_image` |
Comment on lines +183 to +187

#### Service Targets

Valid values for `project.service.hosts` and `project.service.targets`:
Expand Down
3 changes: 2 additions & 1 deletion docs/specs/metrics-audit/feature-telemetry-matrix.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ These commands emit attributes or events beyond the global middleware span.
| `tool install` / `tool upgrade` / `tool uninstall` / `tool check` / `tool list` / `tool show` | `tool.id`, `tool.ids`, `tool.dry_run`, `tool.install.strategy`, `tool.install.success`, `tool.install.success_count`, `tool.install.failure_count`, `tool.install.failed_ids`, `tool.install.duration_ms`, `tool.upgrade.from_version`, `tool.upgrade.to_version`, `tool.check.updates_available` | Comprehensive coverage in `cli/azd/cmd/tool.go`; install/upgrade emit `tools.pack.build` spans for pack-based tools |
| `copilot` (agent) | `copilot.initialize` event (model + reasoning config), `copilot.session` event (session create/resume) | Emitted from `internal/agent/copilot_agent.go`; covers the experimental copilot agent surface |
| `provision` | `validation.provision` event (provision validation outcome + 6 fields), 8 `arm.*` events (subscription / resource-group deploy / stack-deploy / what-if / validate), `aks.postprovision.skip`, per-layer `provision.layer.*` counts (`count`, `max_parallel`, `safe_fallback_count`, `explicit_dependson_count`) when multi-layer infra is used | Telemetry added across `internal/cmd/provision_*.go` and the ARM deployment client |
| `deploy` / `publish` / `package` | `deploy.appservice.zip` event (zip-deploy outcome), `container.credentials` / `container.publish` / `container.remotebuild` events for container-based services | Per-service-target instrumentation; container events emitted from container-app and ACR push paths |
| `deploy` / `publish` / `package` | `deploy.appservice.zip` event (zip-deploy outcome), `container.credentials` / `container.publish` / `container.remotebuild` events for container-based services, `agent.deploy.mode` for hosted agents | Per-service-target instrumentation; hosted-agent mode distinguishes code, container, and prebuilt-image deployments |
| `hooks run` (and all hook-running commands) | `hooks.exec` event with `hooks.name` (hashed unless built-in lifecycle name), `hooks.type` (project / service / **layer**), `hooks.kind` (script runtime — `sh` / `pwsh` / `js` / `ts` / `python` / `dotnet`) | `hooks.type=layer` was added with multi-layer provision; pre/post is encoded in `hooks.name` (e.g., `prebuild` / `postbuild`); emitted from the hooks runner on every lifecycle command |

## Full Inventory Matrix
Expand Down Expand Up @@ -161,6 +161,7 @@ privacy review covers every emission point.
| **Multi-layer provision** | `provision` (when `infra.layers[]` is configured in `azure.yaml`) | (none — enriches the `provision` span) | `provision.layer.count`, `provision.layer.max_parallel`, `provision.layer.safe_fallback_count`, `provision.layer.explicit_dependson_count` | All four are integer measurements emitted from `internal/cmd/provision_graph.go`; no per-layer duration or outcome attribute is emitted |
| **Execution graph (scheduler)** | `up`, `provision`, `deploy`, `package`, `publish`, `down` | `exegraph.run`, `exegraph.step` | `exegraph.step.count`, `exegraph.max_concurrency`, `exegraph.error_policy`, `exegraph.step.name` (hashed), `exegraph.step.deps` (hashed slice), `exegraph.step.tags` (raw — hardcoded literals only), `exegraph.step.timeout_s` | Step names embed user-defined service / layer names from `azure.yaml`; both `name` and `deps` use `fields.StringHashed` / `fields.StringSliceHashed` |
| **Container lifecycle** | `package`, `deploy` (container service targets) | `container.credentials`, `container.publish`, `container.remotebuild` | `container.publish` sets `container.remotebuild` (bool) only; `container.credentials` and `container.remotebuild` set no attributes (span status carries success/failure and duration) | The hashed `pack.builder.image` / `pack.builder.tag` attributes are emitted on the separate `tools.pack.build` span, not the `container.*` spans |
| **Hosted agent deployment** | `package`, `deploy`, `publish` (`azure.ai.agent`) | (none — enriches the active span) | `agent.deploy.mode` (`code`, `container`, `byo_image`) | The mode is determined from the loaded agent definition and selected image path |
| **App Service deploy** | `deploy`, `publish` (App Service targets) | `deploy.appservice.zip` | `deploy.appservice.linux` (bool), `deploy.appservice.attempt` (retry attempt number) | Zip-deploy path only; outcome / duration are carried by the span status and span timing, not by dedicated attributes |
| **AKS service target** | `provision` (AKS preprovision/postprovision) | `aks.postprovision.skip` | Skip reason | Recorded when cluster is not yet available for context setup |
| **Agent troubleshoot middleware** | Triggered on command failure when troubleshooting is engaged | `agent.troubleshoot` | Error chain attributes, hashed error fields | Emitted from `cmd/middleware/error.go` |
Expand Down
Loading
Loading