From 9a60a56b141af9d0b681ab82fa19c0bd82aa45d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 04:01:28 +0000 Subject: [PATCH 1/3] Skip pricing injection for dynamic auto model aliases Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/smoke-copilot-auto.lock.yml | 1 - pkg/workflow/compiler_model_pricing.go | 8 ++++++++ pkg/workflow/compiler_model_pricing_test.go | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/smoke-copilot-auto.lock.yml b/.github/workflows/smoke-copilot-auto.lock.yml index 80f7930c2f1..9cac699d23d 100644 --- a/.github/workflows/smoke-copilot-auto.lock.yml +++ b/.github/workflows/smoke-copilot-auto.lock.yml @@ -149,7 +149,6 @@ jobs: GH_AW_INFO_FIREWALL_TYPE: "squid" GH_AW_INFO_FRONTMATTER_EMOJI: "🌸" GH_AW_COMPILED_STRICT: "true" - GH_AW_INFO_MODEL_COSTS: '{"providers":{"github-copilot":{"models":{"auto":{"cost":{"input":"8.5e-07","output":"1.55e-06"}}}}}}' GH_AW_INFO_FEATURES: '{"gh-aw-detection":false}' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: diff --git a/pkg/workflow/compiler_model_pricing.go b/pkg/workflow/compiler_model_pricing.go index d99dad5964b..9ae4db67b4d 100644 --- a/pkg/workflow/compiler_model_pricing.go +++ b/pkg/workflow/compiler_model_pricing.go @@ -111,9 +111,17 @@ func resolveProviderAndModelForPricing(workflowData *WorkflowData) (string, stri if provider == "" || model == "" { return "", "", false } + if isDynamicModelAliasForPricing(model) { + compilerModelPricingLog.Printf("Skipping external pricing lookup for dynamic model alias %q", model) + return "", "", false + } return provider, model, true } +func isDynamicModelAliasForPricing(model string) bool { + return strings.EqualFold(strings.TrimSpace(model), "auto") +} + // modelCostsHasPricingFor reports whether the ModelCosts overlay already contains a models // entry for the given provider/model. An empty provider matches any provider entry. func modelCostsHasPricingFor(modelCosts map[string]any, provider, model string) bool { diff --git a/pkg/workflow/compiler_model_pricing_test.go b/pkg/workflow/compiler_model_pricing_test.go index 3754ef07521..5e28de25cde 100644 --- a/pkg/workflow/compiler_model_pricing_test.go +++ b/pkg/workflow/compiler_model_pricing_test.go @@ -262,3 +262,22 @@ func TestResolveModelPricingIfMissing_SkipsMalformedQualifiedModel(t *testing.T) assert.Nil(t, c.resolveModelPricingIfMissing(nil, &WorkflowData{Model: "openai/", EngineConfig: &EngineConfig{}})) assert.False(t, called) } + +func TestResolveModelPricingIfMissing_SkipsDynamicAutoModelAlias(t *testing.T) { + c := &Compiler{} + called := false + c.SetModelPricingResolver(func(_ context.Context, _, _ string) (map[string]float64, bool) { + called = true + return map[string]float64{"input": 1e-06}, true + }) + + assert.Nil(t, c.resolveModelPricingIfMissing(nil, &WorkflowData{ + Model: "auto", + EngineConfig: &EngineConfig{ID: "copilot"}, + })) + assert.Nil(t, c.resolveModelPricingIfMissing(nil, &WorkflowData{ + Model: "github_models/auto", + EngineConfig: &EngineConfig{ID: "copilot"}, + })) + assert.False(t, called) +} From 8ef60744741a9e513b40b6c298e6382ea73b36f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 04:18:57 +0000 Subject: [PATCH 2/3] Remove compile-time models.dev pricing lookup Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- cmd/gh-aw/compile_flags_test.go | 8 -------- cmd/gh-aw/main.go | 3 --- docs/src/content/docs/setup/cli.md | 2 +- pkg/cli/cli_consistency_help_test.go | 4 ++-- pkg/cli/compile_compiler_setup.go | 8 -------- pkg/cli/compile_compiler_setup_test.go | 9 +-------- pkg/cli/compile_config.go | 1 - pkg/cli/model_costs.go | 19 ------------------- pkg/cli/model_costs_test.go | 9 --------- 9 files changed, 4 insertions(+), 59 deletions(-) diff --git a/cmd/gh-aw/compile_flags_test.go b/cmd/gh-aw/compile_flags_test.go index b846b513844..81998ebe3d9 100644 --- a/cmd/gh-aw/compile_flags_test.go +++ b/cmd/gh-aw/compile_flags_test.go @@ -21,14 +21,6 @@ func TestCompileCommandShortFlags(t *testing.T) { t.Fatalf("expected --logical-repo shorthand to be -l, got -%s", logicalRepoFlag.Shorthand) } - noModelsDevLookupFlag := compileCmd.Flags().Lookup("no-models-dev-lookup") - if noModelsDevLookupFlag == nil { - t.Fatal("expected --no-models-dev-lookup flag on compile command") - } - if noModelsDevLookupFlag.DefValue != "false" { - t.Fatalf("expected --no-models-dev-lookup default to be false, got %s", noModelsDevLookupFlag.DefValue) - } - grantFlag := compileCmd.Flags().Lookup("grant") if grantFlag == nil { t.Fatal("expected --grant flag on compile command") diff --git a/cmd/gh-aw/main.go b/cmd/gh-aw/main.go index 3683a5a0966..48688f0e911 100644 --- a/cmd/gh-aw/main.go +++ b/cmd/gh-aw/main.go @@ -336,7 +336,6 @@ Unlike ` + "`gh aw upgrade`" + `, ` + "`gh aw compile`" + ` only applies codemod staged, _ := cmd.Flags().GetBool("staged") approve, _ := cmd.Flags().GetBool("approve") validateImages, _ := cmd.Flags().GetBool("validate-images") - disableModelsDevLookup, _ := cmd.Flags().GetBool("no-models-dev-lookup") priorManifestFile, _ := cmd.Flags().GetString("prior-manifest-file") ghes, _ := cmd.Flags().GetBool("ghes") verbose, _ := cmd.Flags().GetBool("verbose") @@ -403,7 +402,6 @@ Unlike ` + "`gh aw upgrade`" + `, ` + "`gh aw compile`" + ` only applies codemod Staged: staged, Approve: approve, ValidateImages: validateImages, - DisableModelsDevLookup: disableModelsDevLookup, PriorManifestFile: priorManifestFile, GHESCompat: ghes, UseSamples: useSamples, @@ -767,7 +765,6 @@ Use "` + string(constants.CLIExtensionPrefix) + ` help all" to show help for all compileCmd.Flags().Bool("staged", false, "Force all safe-outputs into staged mode") compileCmd.Flags().Bool("approve", false, "Approve all safe update changes. When strict mode is active (the default), the compiler emits warnings for new restricted secrets or unapproved action additions/removals not present in the existing gh-aw-manifest. Use this flag to approve and skip safe update enforcement") compileCmd.Flags().Bool("validate-images", false, "Require Docker to be available for container image validation. Without this flag, container image validation is silently skipped when Docker is not installed or the daemon is not running") - compileCmd.Flags().Bool("no-models-dev-lookup", false, "Disable compile-time models.dev pricing lookup for models missing from the embedded catalog") compileCmd.Flags().String("prior-manifest-file", "", "Path to a JSON file containing pre-cached gh-aw-manifests (map[lockFile]*GHAWManifest); used by the MCP server to supply a tamper-proof manifest baseline captured at startup") compileCmd.Flags().Bool("ghes", false, "Enable GitHub Enterprise Server (GHES) compatibility mode. Artifact actions continue using latest non-v3 pins (v3 is deprecated). Overrides the aw.json ghes field") if err := compileCmd.Flags().MarkHidden("prior-manifest-file"); err != nil { diff --git a/docs/src/content/docs/setup/cli.md b/docs/src/content/docs/setup/cli.md index ae371bfdd75..d6c64a833db 100644 --- a/docs/src/content/docs/setup/cli.md +++ b/docs/src/content/docs/setup/cli.md @@ -331,7 +331,7 @@ If the repository root contains an [`aw.yml` manifest](/gh-aw/reference/aw-yml-p Unlike `gh aw upgrade`, `gh aw compile` does not run codemods unless you pass `--fix`. -**Options:** `--action-mode`, `--action-tag`, `--actionlint`, `--actions-repo`, `--allow-action-refs`, `--approve`, `--dependabot`, `--dir/-d`, `--engine/-e`, `--fail-fast`, `--fix`, `--force/-f`, `--force-refresh-action-pins`, `--gh-aw-ref`, `--ghes`, `--grant`, `--grype`, `--json/-j`, `--logical-repo/-l`, `--no-check-update`, `--no-emit`, `--no-models-dev-lookup`, `--poutine`, `--purge`, `--refresh-stop-time`, `--runner-guard`, `--schedule-seed`, `--show-all`, `--staged`, `--stats`, `--strict`, `--syft`, `--trial`, `--validate`, `--validate-images`, `--watch/-w`, `--yamllint`, `--zizmor` +**Options:** `--action-mode`, `--action-tag`, `--actionlint`, `--actions-repo`, `--allow-action-refs`, `--approve`, `--dependabot`, `--dir/-d`, `--engine/-e`, `--fail-fast`, `--fix`, `--force/-f`, `--force-refresh-action-pins`, `--gh-aw-ref`, `--ghes`, `--grant`, `--grype`, `--json/-j`, `--logical-repo/-l`, `--no-check-update`, `--no-emit`, `--poutine`, `--purge`, `--refresh-stop-time`, `--runner-guard`, `--schedule-seed`, `--show-all`, `--staged`, `--stats`, `--strict`, `--syft`, `--trial`, `--validate`, `--validate-images`, `--watch/-w`, `--yamllint`, `--zizmor` **`--gh-aw-ref` flag:** Convenience alias for `--action-mode release --action-tag `. Accepts a branch name, tag, or commit SHA targeting the `github/gh-aw` repository. Branch and tag names are resolved to their full commit SHA at compile time, so the baked-in reference is immutable and reproducible. Useful for E2E-testing workflows compiled against a specific gh-aw revision. diff --git a/pkg/cli/cli_consistency_help_test.go b/pkg/cli/cli_consistency_help_test.go index 8fdf8759d39..d379732d2b5 100644 --- a/pkg/cli/cli_consistency_help_test.go +++ b/pkg/cli/cli_consistency_help_test.go @@ -50,7 +50,7 @@ func TestUpdateDocsIncludeCoolDownOption(t *testing.T) { assert.Contains(t, updateSection, "`--cool-down`", "update docs options should include --cool-down") } -func TestCompileDocsIncludeNoModelsDevLookupOption(t *testing.T) { +func TestCompileDocsReflectCurrentOptions(t *testing.T) { _, currentFile, _, ok := runtime.Caller(0) require.True(t, ok, "should resolve current test file path") @@ -63,7 +63,7 @@ func TestCompileDocsIncludeNoModelsDevLookupOption(t *testing.T) { require.NotEqual(t, -1, compileIndex, "CLI setup docs should contain the compile section") compileSection := text[compileIndex:] - assert.Contains(t, compileSection, "`--no-models-dev-lookup`", "compile docs options should include --no-models-dev-lookup") + assert.NotContains(t, compileSection, "`--no-models-dev-lookup`", "compile docs options should not include removed --no-models-dev-lookup") assert.Contains(t, compileSection, "does not run codemods unless you pass `--fix`", "compile docs should explain --fix opt-in behavior") } diff --git a/pkg/cli/compile_compiler_setup.go b/pkg/cli/compile_compiler_setup.go index 66491b6a3e2..b0be336e111 100644 --- a/pkg/cli/compile_compiler_setup.go +++ b/pkg/cli/compile_compiler_setup.go @@ -116,14 +116,6 @@ func createAndConfigureCompiler(config CompileConfig) *workflow.Compiler { // Set up repository context setupRepositoryContext(compiler, config) - if config.DisableModelsDevLookup { - compileCompilerSetupLog.Print("models.dev pricing lookup disabled via --no-models-dev-lookup") - } else { - // Register the models.dev pricing resolver so the compiler can inject pricing for - // models absent from the embedded catalog into GH_AW_INFO_MODEL_COSTS in the lock.yml. - compiler.SetModelPricingResolver(FindOrFetchModelPricing) - } - return compiler } diff --git a/pkg/cli/compile_compiler_setup_test.go b/pkg/cli/compile_compiler_setup_test.go index 804b16b138c..ab1b7ecd61d 100644 --- a/pkg/cli/compile_compiler_setup_test.go +++ b/pkg/cli/compile_compiler_setup_test.go @@ -17,15 +17,8 @@ func hasModelPricingResolver(compiler *workflow.Compiler) bool { func TestCreateAndConfigureCompiler_RegistersModelPricingResolverByDefault(t *testing.T) { compiler := createAndConfigureCompiler(CompileConfig{}) - if !hasModelPricingResolver(compiler) { - t.Fatal("expected model pricing resolver to be registered by default") - } -} - -func TestCreateAndConfigureCompiler_SkipsModelPricingResolverWhenDisabled(t *testing.T) { - compiler := createAndConfigureCompiler(CompileConfig{DisableModelsDevLookup: true}) if hasModelPricingResolver(compiler) { - t.Fatal("expected model pricing resolver to be nil when models.dev lookup is disabled") + t.Fatal("expected model pricing resolver to be nil by default") } } diff --git a/pkg/cli/compile_config.go b/pkg/cli/compile_config.go index 66614c65d01..1336ea866d9 100644 --- a/pkg/cli/compile_config.go +++ b/pkg/cli/compile_config.go @@ -41,7 +41,6 @@ type CompileConfig struct { ValidateImages bool // Require Docker to be available for container image validation (fail instead of skipping when Docker is unavailable) PriorManifestFile string // Path to a JSON file containing pre-cached manifests (map[lockFile]*GHAWManifest) collected at MCP server startup; takes precedence over git HEAD / filesystem reads for safe update enforcement GHESCompat bool // Enable GHES compatibility mode (overrides aw.json ghes field); artifact actions still use latest non-v3 pins - DisableModelsDevLookup bool // Disable compile-time models.dev pricing lookup for models missing from the embedded catalog } // CompileValidationError represents a single validation error or warning diff --git a/pkg/cli/model_costs.go b/pkg/cli/model_costs.go index 545c56eabfb..e3510bfed50 100644 --- a/pkg/cli/model_costs.go +++ b/pkg/cli/model_costs.go @@ -1,7 +1,6 @@ package cli import ( - "context" _ "embed" "encoding/json" "strconv" @@ -183,21 +182,3 @@ func computeModelInferenceCostUSD(provider, model string, inputTokens, outputTok func computeModelInferenceAIC(provider, model string, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens int) float64 { return usdToAIC(computeModelInferenceCostUSD(provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens)) } - -// FindOrFetchModelPricing resolves per-token pricing for the given provider/model. -// It checks the embedded catalog first; for models absent from the embedded catalog it -// attempts to download the models.dev catalog (at most once per process) and queries -// that. Returns (nil, false) when the model is found in the embedded catalog (runtime -// already has it) or when no pricing can be resolved. -// -// This function is intended for compile-time use: the workflow compiler calls it to -// inject pricing for unknown models into GH_AW_INFO_MODEL_COSTS in the compiled lock.yml -// so that the agent job can perform cost accounting without a live catalog download. -func FindOrFetchModelPricing(ctx context.Context, provider, model string) (map[string]float64, bool) { - if _, ok := findModelPricing(provider, model); ok { - // Model is already in the embedded catalog; the runtime actions/setup/js/models.json - // will supply the pricing — no need to inject it into the lock.yml overlay. - return nil, false - } - return modelsdev.FindPricing(ctx, provider, model) -} diff --git a/pkg/cli/model_costs_test.go b/pkg/cli/model_costs_test.go index c4526a4edd9..444c5999bfd 100644 --- a/pkg/cli/model_costs_test.go +++ b/pkg/cli/model_costs_test.go @@ -3,7 +3,6 @@ package cli import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -99,11 +98,3 @@ func TestComputeModelInferenceAICGitHubCopilotNoCacheRead(t *testing.T) { assert.InDelta(t, aicViaAnthropic, aicViaGitHubCopilot, 1e-9, "zero cache reads must not alter the charged input token count") } - -func TestFindOrFetchModelPricing_EmbeddedModelReturnsNil(t *testing.T) { - // claude-sonnet-4.6 is in the embedded catalog; FindOrFetchModelPricing should return - // (nil, false) so the lock.yml overlay does not duplicate what models.json already has. - pricing, ok := FindOrFetchModelPricing(context.Background(), "anthropic", "claude-sonnet-4.6") - assert.False(t, ok) - assert.Nil(t, pricing) -} From f25d7e9cbfee0ba231c7238c501d4c213a2379d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 07:18:43 +0000 Subject: [PATCH 3/3] Rename misleading test: RegistersModelPricingResolverByDefault -> DoesNotRegisterModelPricingResolverByDefault Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/compile_compiler_setup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cli/compile_compiler_setup_test.go b/pkg/cli/compile_compiler_setup_test.go index ab1b7ecd61d..55621070f76 100644 --- a/pkg/cli/compile_compiler_setup_test.go +++ b/pkg/cli/compile_compiler_setup_test.go @@ -15,7 +15,7 @@ func hasModelPricingResolver(compiler *workflow.Compiler) bool { return !reflect.ValueOf(compiler).Elem().FieldByName("modelPricingResolver").IsNil() } -func TestCreateAndConfigureCompiler_RegistersModelPricingResolverByDefault(t *testing.T) { +func TestCreateAndConfigureCompiler_DoesNotRegisterModelPricingResolverByDefault(t *testing.T) { compiler := createAndConfigureCompiler(CompileConfig{}) if hasModelPricingResolver(compiler) { t.Fatal("expected model pricing resolver to be nil by default")