From 2862fd962e1930580339f3c6e99e227d710bfebf Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:39:59 -0700 Subject: [PATCH 1/8] benchmark template --- .../templates/jobs/benchmark-job.yml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 eng/common/pipelines/templates/jobs/benchmark-job.yml diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml new file mode 100644 index 000000000000..e5371b7c88dd --- /dev/null +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -0,0 +1,84 @@ +parameters: + - name: BenchmarkProject + type: string + default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' + - name: ScenarioName + type: string + default: '' + - name: Tags + type: string + default: '' + - name: Model + type: string + default: '' + - name: Parallelism + type: number + default: 5 + +jobs: + - job: Run_Benchmark + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - group: 'AzSDK_Eval_Variable_group' + displayName: 'Run Benchmark' + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - checkout: none + - task: UseDotNet@2 + displayName: "Use .NET SDK 8.0.x" + retryCountOnTaskFailure: 3 + inputs: + packageType: sdk + version: 8.0.x + performMultiLevelLookup: true + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Repositories: + - Name: $(Build.Repository.Name) + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) + Paths: + - 'tools/azsdk-cli/**' + - 'eng/common/**' + + - task: PowerShell@2 + displayName: 'Run benchmarks' + condition: succeeded() + inputs: + targetType: 'inline' + workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' + pwsh: true + script: | + $args = @("run") + + # Determine scenario selection: name, tags, or all + if ("${{ parameters.ScenarioName }}") { + $args += "${{ parameters.ScenarioName }}" + } elseif ("${{ parameters.Tags }}") { + $args += "--tags", "${{ parameters.Tags }}" + } else { + $args += "--all" + } + + # Optional model override + if ("${{ parameters.Model }}") { + $args += "--model", "${{ parameters.Model }}" + } + + $args += "--parallel", "${{ parameters.Parallelism }}" + $args += "--cleanup", "always" + + Write-Host "Running: dotnet run -- $($args -join ' ')" + dotnet run -- @args + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_MULTILEVEL_LOOKUP: 0 + COPILOT_GITHUB_TOKEN: $(azuresdk-copilot-github-pat) From b654120225254f953dc0fd56e67d4b7558f2a607 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:01:14 -0700 Subject: [PATCH 2/8] cli args instead of args --- .../pipelines/templates/jobs/benchmark-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index e5371b7c88dd..4b4701a8cfa9 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -56,27 +56,27 @@ jobs: workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' pwsh: true script: | - $args = @("run") + $cliArgs = @("run") # Determine scenario selection: name, tags, or all if ("${{ parameters.ScenarioName }}") { - $args += "${{ parameters.ScenarioName }}" + $cliArgs += "${{ parameters.ScenarioName }}" } elseif ("${{ parameters.Tags }}") { - $args += "--tags", "${{ parameters.Tags }}" + $cliArgs += "--tags", "${{ parameters.Tags }}" } else { - $args += "--all" + $cliArgs += "--all" } # Optional model override if ("${{ parameters.Model }}") { - $args += "--model", "${{ parameters.Model }}" + $cliArgs += "--model", "${{ parameters.Model }}" } - $args += "--parallel", "${{ parameters.Parallelism }}" - $args += "--cleanup", "always" + $cliArgs += "--parallel", "${{ parameters.Parallelism }}" + $cliArgs += "--cleanup", "always" - Write-Host "Running: dotnet run -- $($args -join ' ')" - dotnet run -- @args + Write-Host "Running: dotnet run -- $($cliArgs -join ' ')" + dotnet run -- @cliArgs env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_CLI_TELEMETRY_OPTOUT: 1 From ae6e36cf3a7dc53fd65b5c8572ca20a84b9f5e82 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:05:10 -0700 Subject: [PATCH 3/8] target path --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 4b4701a8cfa9..2beba3f43f82 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -45,7 +45,7 @@ jobs: Commitish: $(Build.SourceVersion) WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) Paths: - - 'tools/azsdk-cli/**' + - '${{ parameters.BenchmarkProject }}/**' - 'eng/common/**' - task: PowerShell@2 From 1577e53ca78f4c3d3ed0b94791c40146f659cc94 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:06:58 -0700 Subject: [PATCH 4/8] .net 9 --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 2beba3f43f82..99f6cb3c2678 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -30,14 +30,14 @@ jobs: steps: - checkout: none - task: UseDotNet@2 - displayName: "Use .NET SDK 8.0.x" + displayName: "Use .NET SDK 9.0.x" retryCountOnTaskFailure: 3 inputs: packageType: sdk - version: 8.0.x + version: 9.0.x performMultiLevelLookup: true - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - template:/eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: SkipCheckoutNone: true Repositories: From 294ce3d4b44f67347e1ce1284125abcb114495cc Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:08:33 -0700 Subject: [PATCH 5/8] fix --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 99f6cb3c2678..d7d15bc90a78 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -37,7 +37,7 @@ jobs: version: 9.0.x performMultiLevelLookup: true - - template:/eng/common/pipelines/templates/steps/sparse-checkout.yml + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: SkipCheckoutNone: true Repositories: From 14b7f577ebbf78154379d6daabfad12a27ceea6e Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:16:21 -0700 Subject: [PATCH 6/8] blank defaults --- .../pipelines/templates/jobs/benchmark-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index d7d15bc90a78..ef6ce1739583 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -4,13 +4,13 @@ parameters: default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' - name: ScenarioName type: string - default: '' + default: ' ' - name: Tags type: string - default: '' + default: ' ' - name: Model type: string - default: '' + default: ' ' - name: Parallelism type: number default: 5 @@ -59,17 +59,17 @@ jobs: $cliArgs = @("run") # Determine scenario selection: name, tags, or all - if ("${{ parameters.ScenarioName }}") { - $cliArgs += "${{ parameters.ScenarioName }}" - } elseif ("${{ parameters.Tags }}") { - $cliArgs += "--tags", "${{ parameters.Tags }}" + if ("${{ parameters.ScenarioName }}".Trim()) { + $cliArgs += "${{ parameters.ScenarioName }}".Trim() + } elseif ("${{ parameters.Tags }}".Trim()) { + $cliArgs += "--tags", "${{ parameters.Tags }}".Trim() } else { $cliArgs += "--all" } # Optional model override - if ("${{ parameters.Model }}") { - $cliArgs += "--model", "${{ parameters.Model }}" + if ("${{ parameters.Model }}".Trim()) { + $cliArgs += "--model", "${{ parameters.Model }}".Trim() } $cliArgs += "--parallel", "${{ parameters.Parallelism }}" From 928d63b06c73bb36309e67b58e1ede684db03da2 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:32:08 -0700 Subject: [PATCH 7/8] back to 8 --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index ef6ce1739583..97fc8c7d2ff0 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -29,13 +29,7 @@ jobs: steps: - checkout: none - - task: UseDotNet@2 - displayName: "Use .NET SDK 9.0.x" - retryCountOnTaskFailure: 3 - inputs: - packageType: sdk - version: 9.0.x - performMultiLevelLookup: true + - template: /eng/pipelines/templates/steps/install-dotnet.yml - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: From 5ca6d547f2e68974bedfd162e2cb187f21910855 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:48:58 -0700 Subject: [PATCH 8/8] install after checkout --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 97fc8c7d2ff0..d490d8977921 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -29,7 +29,6 @@ jobs: steps: - checkout: none - - template: /eng/pipelines/templates/steps/install-dotnet.yml - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -42,6 +41,8 @@ jobs: - '${{ parameters.BenchmarkProject }}/**' - 'eng/common/**' + - template: /eng/pipelines/templates/steps/install-dotnet.yml + - task: PowerShell@2 displayName: 'Run benchmarks' condition: succeeded()