Skip to content

fix(gh-copilot): update Copilot metrics endpoints & schemas to latest… #9002

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bujjibabukatta wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(gh-copilot): update Copilot metrics endpoints & schemas to latest… #9002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 59 additions & 7 deletions backend/plugins/gh-copilot/models/enterprise_metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,13 @@ type CopilotCodeMetrics struct {

// CopilotCliMetrics contains CLI usage breakdown metrics.
type CopilotCliMetrics struct {
CliSessionCount int `json:"cliSessionCount" gorm:"comment:Number of CLI sessions"`
CliRequestCount int `json:"cliRequestCount" gorm:"comment:Number of CLI requests"`
CliPromptCount int `json:"cliPromptCount" gorm:"comment:Number of CLI prompts"`
CliOutputTokenSum int `json:"cliOutputTokenSum" gorm:"comment:Total output tokens from CLI"`
CliPromptTokenSum int `json:"cliPromptTokenSum" gorm:"comment:Total prompt tokens from CLI"`
CliSessionCount int `json:"cliSessionCount" gorm:"comment:Number of CLI sessions"`
CliRequestCount int `json:"cliRequestCount" gorm:"comment:Number of CLI requests"`
CliPromptCount int `json:"cliPromptCount" gorm:"comment:Number of CLI prompts"`
CliOutputTokenSum int `json:"cliOutputTokenSum" gorm:"comment:Total output tokens from CLI"`
CliPromptTokenSum int `json:"cliPromptTokenSum" gorm:"comment:Total prompt tokens from CLI"`
CliLastKnownVersion string `json:"cliLastKnownVersion" gorm:"type:varchar(50);comment:Last known Copilot CLI version"`
CliAvgTokensPerRequest float64 `json:"cliAvgTokensPerRequest" gorm:"comment:Average tokens consumed per CLI request"`
}

// GhCopilotEnterpriseDailyMetrics captures daily enterprise-level aggregate Copilot metrics.
Expand All @@ -59,7 +61,12 @@ type GhCopilotEnterpriseDailyMetrics struct {
ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"`
Day time.Time `gorm:"primaryKey;type:date" json:"day"`

EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"`
EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"`
// OrganizationId identifies the owning org for org-level connections/rows.
// Previously always written as "" by ExtractOrgMetrics, making org-level
// rows in this shared table unidentifiable without a join back to the
// connection/scope config.
OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"`
DailyActiveUsers int `json:"dailyActiveUsers"`
WeeklyActiveUsers int `json:"weeklyActiveUsers"`
MonthlyActiveUsers int `json:"monthlyActiveUsers"`
Expand All @@ -69,6 +76,11 @@ type GhCopilotEnterpriseDailyMetrics struct {
// CLI active users
DailyActiveCliUsers int `json:"dailyActiveCliUsers" gorm:"comment:Daily active CLI users"`

// Copilot cloud agent (coding agent) active user counts
DailyActiveCopilotCloudAgentUsers int `json:"dailyActiveCopilotCloudAgentUsers" gorm:"comment:Daily active Copilot cloud/coding agent users"`
WeeklyActiveCopilotCloudAgentUsers int `json:"weeklyActiveCopilotCloudAgentUsers" gorm:"comment:Weekly active Copilot cloud/coding agent users"`
MonthlyActiveCopilotCloudAgentUsers int `json:"monthlyActiveCopilotCloudAgentUsers" gorm:"comment:Monthly active Copilot cloud/coding agent users"`

// Code review user counts
DailyActiveCopilotCodeReviewUsers int `json:"dailyActiveCopilotCodeReviewUsers"`
DailyPassiveCopilotCodeReviewUsers int `json:"dailyPassiveCopilotCodeReviewUsers"`
Expand Down Expand Up @@ -100,7 +112,11 @@ type GhCopilotEnterpriseDailyMetrics struct {
PRMedianMinToMergeCopilotReviewed float64 `json:"prMedianMinToMergeCopilotReviewed" gorm:"comment:Median min to merge Copilot-reviewed PRs"`
PRTotalCopilotSuggestions int `json:"prTotalCopilotSuggestions" gorm:"comment:Total Copilot review suggestions"`
PRTotalCopilotAppliedSuggestions int `json:"prTotalCopilotAppliedSuggestions" gorm:"comment:Total Copilot applied suggestions"`

// PRCopilotSuggestionsByCommentType is a JSON-encoded array of
// {comment_type, total_suggestions, total_applied_suggestions}, e.g. the
// split between "suggestion" and "explanation" style review comments.
PRCopilotSuggestionsByCommentType string `json:"prCopilotSuggestionsByCommentType" gorm:"type:text;comment:JSON breakdown of Copilot PR suggestions by comment type"`

CopilotActivityMetrics `mapstructure:",squash"`
CopilotCliMetrics `mapstructure:",squash"`
common.NoPKModel
Expand Down Expand Up @@ -187,3 +203,39 @@ type GhCopilotMetricsByModelFeature struct {
func (GhCopilotMetricsByModelFeature) TableName() string {
return "_tool_copilot_metrics_by_model_feature"
}

// GhCopilotMetricsByAiAdoptionPhase stores enterprise/org metrics broken down
// by AI adoption cohort (phase 0-3, see GitHub's totals_by_ai_adoption_phase).
// Unlike CopilotActivityMetrics elsewhere in this file, most of these fields
// are per-user averages within the phase rather than sums.
type GhCopilotMetricsByAiAdoptionPhase struct {
ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"`
ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"`
Day time.Time `gorm:"primaryKey;type:date" json:"day"`
Phase int `gorm:"primaryKey" json:"phase"`

PhaseVersion int `json:"phaseVersion" gorm:"comment:Adoption-phase classification version, starts at 1"`
EngagedUsers int `json:"engagedUsers" gorm:"comment:Users engaged in this phase (2-day-in-28 window)"`

AvgUserInitiatedInteractionCount float64 `json:"avgUserInitiatedInteractionCount"`
AvgCodeGenerationActivityCount float64 `json:"avgCodeGenerationActivityCount"`
AvgCodeAcceptanceActivityCount float64 `json:"avgCodeAcceptanceActivityCount"`
AvgLocAddedSum float64 `json:"avgLocAddedSum"`
AvgLocDeletedSum float64 `json:"avgLocDeletedSum"`

AvgPullRequestsCreated float64 `json:"avgPullRequestsCreated"`
AvgPullRequestsMerged float64 `json:"avgPullRequestsMerged"`
AvgPullRequestsReviewed float64 `json:"avgPullRequestsReviewed"`
AvgMedianMinutesToMerge float64 `json:"avgMedianMinutesToMerge"`

AvgPullRequestsMinutesToReview float64 `json:"avgPullRequestsMinutesToReview" gorm:"comment:Added 2026-07-07"`
AvgPullRequestsReviewCycles float64 `json:"avgPullRequestsReviewCycles" gorm:"comment:Added 2026-07-07"`

TotalPullRequestsMerged int `json:"totalPullRequestsMerged" gorm:"comment:True sum (not average), added 2026-06-26"`

common.NoPKModel
}

func (GhCopilotMetricsByAiAdoptionPhase) TableName() string {
return "_tool_copilot_metrics_by_ai_adoption_phase"
}
2 changes: 2 additions & 0 deletions backend/plugins/gh-copilot/models/migrationscripts/register.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,7 @@ func All() []plugin.MigrationScript {
new(addPRFieldsToEnterpriseMetrics),
new(addOrganizationIdToUserMetrics),
new(addCopilotMetricsGaps),
new(addCopilotMetricsGapsV2),
new(addCopilot28DayReports),
}
}
4 changes: 4 additions & 0 deletions backend/plugins/gh-copilot/models/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func GetTablesInfo() []dal.Tabler {
&GhCopilotMetricsByLanguageFeature{},
&GhCopilotMetricsByLanguageModel{},
&GhCopilotMetricsByModelFeature{},
&GhCopilotMetricsByAiAdoptionPhase{},
// User-level metrics (from enterprise user reports)
&GhCopilotUserDailyMetrics{},
&GhCopilotUserMetricsByIde{},
Expand All @@ -47,5 +48,8 @@ func GetTablesInfo() []dal.Tabler {
&GhCopilotSeat{},
// User-team mappings
&GhCopilotUserTeam{},
// 28-day rolling-window snapshots (separate cadence from the daily tables above)
&GhCopilotEnterprise28DayMetrics{},
&GhCopilotUser28DayMetrics{},
}
}
7 changes: 7 additions & 0 deletions backend/plugins/gh-copilot/models/user_metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ type GhCopilotUserDailyMetrics struct {
UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"`
UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"`
UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"`
UsedCopilotCodingAgent bool `json:"usedCopilotCodingAgent" gorm:"comment:Whether user used the Copilot coding agent"`
UsedCopilotCloudAgent bool `json:"usedCopilotCloudAgent" gorm:"comment:Whether user used the Copilot cloud agent"`

// AI adoption cohort classification (phase 0-3), added 2026-05-29.
AiAdoptionPhase int `json:"aiAdoptionPhase" gorm:"comment:0=no cohort,1=code-first,2=agent-first,3=multi-agent"`
AiAdoptionPhaseVersion int `json:"aiAdoptionPhaseVersion" gorm:"comment:Classification logic version, starts at 1"`


CopilotActivityMetrics `mapstructure:",squash"`
CopilotCliMetrics `mapstructure:",squash"`
Expand Down
117 changes: 106 additions & 11 deletions backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,24 @@ import (
// --- Enterprise report JSON structures ---

type enterpriseDayTotal struct {
Day string `json:"day"`
EnterpriseId string `json:"enterprise_id"`
Day string `json:"day"`
EnterpriseId string `json:"enterprise_id"`
// OrganizationId is only present on organization-level reports (org report
// shares this same flat schema). Needed so ExtractOrgMetrics can stamp the
// owning org onto rows instead of leaving them unidentifiable.
OrganizationId string `json:"organization_id"`
DailyActiveUsers int `json:"daily_active_users"`
WeeklyActiveUsers int `json:"weekly_active_users"`
MonthlyActiveUsers int `json:"monthly_active_users"`
MonthlyActiveChatUsers int `json:"monthly_active_chat_users"`
MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"`
DailyActiveCliUsers int `json:"daily_active_cli_users"`

// Copilot cloud agent (coding agent) active user counts — added in 2026-03-10.
DailyActiveCopilotCloudAgentUsers int `json:"daily_active_copilot_cloud_agent_users"`
WeeklyActiveCopilotCloudAgentUsers int `json:"weekly_active_copilot_cloud_agent_users"`
MonthlyActiveCopilotCloudAgentUsers int `json:"monthly_active_copilot_cloud_agent_users"`

// Code review user counts
DailyActiveCopilotCodeReviewUsers int `json:"daily_active_copilot_code_review_users"`
DailyPassiveCopilotCodeReviewUsers int `json:"daily_passive_copilot_code_review_users"`
Expand Down Expand Up @@ -69,6 +78,12 @@ type enterpriseDayTotal struct {
TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"`
TotalsByCli *totalsByCli `json:"totals_by_cli"`
PullRequests *pullRequestStats `json:"pull_requests"`

// TotalsByAiAdoptionPhase groups engagement/activity metrics by AI adoption
// cohort (phase 0-3). Added 2026-05-29, extended with review-cycle metrics
// 2026-07-07. Only present on enterprise/org reports, not user reports.
TotalsByAiAdoptionPhase []totalsByAiAdoptionPhase `json:"totals_by_ai_adoption_phase"`

}

type totalsByIde struct {
Expand Down Expand Up @@ -130,18 +145,58 @@ type pullRequestStats struct {
MedianMinToMergeCopilotReviewed float64 `json:"median_minutes_to_merge_copilot_reviewed"`
TotalCopilotSuggestions int `json:"total_copilot_suggestions"`
TotalCopilotAppliedSuggestions int `json:"total_copilot_applied_suggestions"`

// CopilotSuggestionsByCommentType breaks total_copilot_suggestions down by
// the kind of PR review comment Copilot generated (e.g. "suggestion", "explanation").
CopilotSuggestionsByCommentType []prSuggestionsByCommentType `json:"copilot_suggestions_by_comment_type"`
}

type prSuggestionsByCommentType struct {
CommentType string `json:"comment_type"`
TotalSuggestions int `json:"total_suggestions"`
TotalApplied int `json:"total_applied_suggestions"`
}

type totalsByCli struct {
SessionCount int `json:"session_count"`
RequestCount int `json:"request_count"`
PromptCount int `json:"prompt_count"`
TokenUsage *cliTokens `json:"token_usage"`
SessionCount int `json:"session_count"`
RequestCount int `json:"request_count"`
PromptCount int `json:"prompt_count"`
TokenUsage *cliTokens `json:"token_usage"`
LastKnownCliVersion string `json:"last_known_cli_version"`
}

type cliTokens struct {
OutputTokensSum int `json:"output_tokens_sum"`
PromptTokensSum int `json:"prompt_tokens_sum"`
OutputTokensSum int `json:"output_tokens_sum"`
PromptTokensSum int `json:"prompt_tokens_sum"`
AvgTokensPerRequest float64 `json:"avg_tokens_per_request"`
}

// totalsByAiAdoptionPhase is one entry of the totals_by_ai_adoption_phase
// breakdown on enterprise/org reports. Per-user fields are averages within the
// phase, not sums (per GitHub's docs) — the *_total fields are true sums.
type totalsByAiAdoptionPhase struct {
Phase int `json:"phase"`
Version int `json:"version"`

EngagedUsers int `json:"engaged_users"`

AvgUserInitiatedInteractionCount float64 `json:"avg_user_initiated_interaction_count"`
AvgCodeGenerationActivityCount float64 `json:"avg_code_generation_activity_count"`
AvgCodeAcceptanceActivityCount float64 `json:"avg_code_acceptance_activity_count"`
AvgLocAddedSum float64 `json:"avg_loc_added_sum"`
AvgLocDeletedSum float64 `json:"avg_loc_deleted_sum"`

AvgPullRequestsCreated float64 `json:"avg_pull_requests_created"`
AvgPullRequestsMerged float64 `json:"avg_pull_requests_merged"`
AvgPullRequestsReviewed float64 `json:"avg_pull_requests_reviewed"`
AvgMedianMinutesToMerge float64 `json:"avg_median_minutes_to_merge"`

// Added 2026-07-07.
AvgPullRequestsMinutesToReview float64 `json:"avg_pull_requests_minutes_to_review"`
AvgPullRequestsReviewCycles float64 `json:"avg_pull_requests_review_cycles"`

// Added 2026-06-26 — true total rather than a per-user average.
TotalPullRequestsMerged int `json:"total_pull_requests_merged"`
}

type totalsByModelFeature struct {
Expand Down Expand Up @@ -203,13 +258,18 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error {
ScopeId: data.Options.ScopeId,
Day: day,
EnterpriseId: dt.EnterpriseId,
OrganizationId: dt.OrganizationId,
DailyActiveUsers: dt.DailyActiveUsers,
WeeklyActiveUsers: dt.WeeklyActiveUsers,
MonthlyActiveUsers: dt.MonthlyActiveUsers,
MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers,
MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers,
DailyActiveCliUsers: dt.DailyActiveCliUsers,

DailyActiveCopilotCloudAgentUsers: dt.DailyActiveCopilotCloudAgentUsers,
WeeklyActiveCopilotCloudAgentUsers: dt.WeeklyActiveCopilotCloudAgentUsers,
MonthlyActiveCopilotCloudAgentUsers: dt.MonthlyActiveCopilotCloudAgentUsers,

DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers,
DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers,
WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers,
Expand All @@ -236,13 +296,15 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error {
}
if dt.TotalsByCli != nil {
dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{
CliSessionCount: dt.TotalsByCli.SessionCount,
CliRequestCount: dt.TotalsByCli.RequestCount,
CliPromptCount: dt.TotalsByCli.PromptCount,
CliSessionCount: dt.TotalsByCli.SessionCount,
CliRequestCount: dt.TotalsByCli.RequestCount,
CliPromptCount: dt.TotalsByCli.PromptCount,
CliLastKnownVersion: dt.TotalsByCli.LastKnownCliVersion,
}
if dt.TotalsByCli.TokenUsage != nil {
dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum
dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum
dailyMetrics.CopilotCliMetrics.CliAvgTokensPerRequest = dt.TotalsByCli.TokenUsage.AvgTokensPerRequest
}
}
if dt.PullRequests != nil {
Expand All @@ -260,8 +322,41 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error {
dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed
dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions
dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions

// Stored as a JSON blob rather than a normalized breakdown table for
// now — comment-type cardinality is small (suggestion/explanation)
// and doesn't warrant its own table + primary key the way IDE/feature
// breakdowns do. Revisit if GitHub adds more comment types.
if len(dt.PullRequests.CopilotSuggestionsByCommentType) > 0 {
if b, jsonErr := json.Marshal(dt.PullRequests.CopilotSuggestionsByCommentType); jsonErr == nil {
dailyMetrics.PRCopilotSuggestionsByCommentType = string(b)
}
}
}
results = append(results, dailyMetrics)
// By AI adoption phase
for _, phase := range dt.TotalsByAiAdoptionPhase {
results = append(results, &models.GhCopilotMetricsByAiAdoptionPhase{
ConnectionId: data.Options.ConnectionId,
ScopeId: data.Options.ScopeId,
Day: day,
Phase: phase.Phase,
PhaseVersion: phase.Version,
EngagedUsers: phase.EngagedUsers,
AvgUserInitiatedInteractionCount: phase.AvgUserInitiatedInteractionCount,
AvgCodeGenerationActivityCount: phase.AvgCodeGenerationActivityCount,
AvgCodeAcceptanceActivityCount: phase.AvgCodeAcceptanceActivityCount,
AvgLocAddedSum: phase.AvgLocAddedSum,
AvgLocDeletedSum: phase.AvgLocDeletedSum,
AvgPullRequestsCreated: phase.AvgPullRequestsCreated,
AvgPullRequestsMerged: phase.AvgPullRequestsMerged,
AvgPullRequestsReviewed: phase.AvgPullRequestsReviewed,
AvgMedianMinutesToMerge: phase.AvgMedianMinutesToMerge,
AvgPullRequestsMinutesToReview: phase.AvgPullRequestsMinutesToReview,
AvgPullRequestsReviewCycles: phase.AvgPullRequestsReviewCycles,
TotalPullRequestsMerged: phase.TotalPullRequestsMerged,
})
}

// By IDE
for _, ide := range dt.TotalsByIde {
Expand Down
Loading