[Auto] Regenerate api.md for all packages

eng/pipelines/regenerate-apiview-md.yml

@@ -0,0 +1,41 @@
+# Pipeline to mass-regenerate api.md files across all Python SDK packages.
+#
+# Triggers:
+#   - CI: when eng/apiview_reqs.txt changes on main (i.e. when the
+#         apiview-stub-generator version is bumped)
+#   - Manual: can be triggered at any time via "Run pipeline" in Azure DevOps
+#
+# What it does:
+#   1. Runs `azpysdk apistub --md` for every Python SDK package.
+#   2. If any api.md files changed, opens a **draft PR** containing only
+#      those api.md updates for human review before merging.
+
+trigger:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - eng/apiview_reqs.txt
+
+pr: none
+
+parameters:
+  - name: IncludeManagement
+    displayName: 'Include mgmt packages when regenerating api.md (uncheck to skip mgmt packages).'
+    type: boolean
+    default: true
+
+variables:
+  - template: /eng/pipelines/templates/variables/image.yml
+
+extends:
+  template: /eng/pipelines/templates/stages/1es-redirect.yml
+  parameters:
+    stages:
+      - stage: RegenerateApiMd
+        displayName: 'Regenerate api.md'
+        jobs:
+          - template: /eng/pipelines/templates/jobs/apiview-md.yml
+            parameters:
+              IncludeManagement: ${{ parameters.IncludeManagement }}

eng/pipelines/templates/jobs/apiview-md.yml

@@ -0,0 +1,116 @@
+parameters:
+  - name: IncludeManagement
+    type: boolean
+    default: true
+
+jobs:
+  - job: RegenerateApiMd
+    displayName: 'Regenerate api.md for all packages'
+    timeoutInMinutes: 480
+
+    pool:
+      name: azsdk-pool
+      image: ubuntu-24.04
+      os: linux
+
+    variables:
+      - template: /eng/pipelines/templates/variables/globals.yml
+      - ${{ if parameters.IncludeManagement }}:
+        - name: FilterTypeArg
+          value: ''
+      - ${{ else }}:
+        - name: FilterTypeArg
+          value: '--filter-type=Omit_management'
+
+    steps:
+      - checkout: self
+        persistCredentials: true
+        fetchDepth: 0
+
+      - task: UsePythonVersion@0
+        displayName: 'Use Python 3.10'
+        inputs:
+          versionSpec: '3.10'
+
+      - template: /eng/pipelines/templates/steps/auth-dev-feed.yml
+
+      - template: /eng/pipelines/templates/steps/use-venv.yml
+        parameters:
+          VirtualEnvironmentName: "venv-apiview-md"
+
+      - pwsh: |
+          $ErrorActionPreference = 'Stop'
+          $PSNativeCommandUseErrorActionPreference = $true
+          $(PIP_EXE) install -r eng/ci_tools.txt
+        displayName: 'Install CI tools'
+
+      - script: |
+          git config --global user.name "azure-sdk"
+          git config --global user.email "azuresdk@microsoft.com"
+        displayName: 'Configure git'
+
+      # Run `azpysdk apistub --md` for all packages in parallel via dispatch_checks.py.
+      # Each package is processed in its own isolated virtual environment. The generated
+      # api.md is written directly into each package directory so git can detect the changes.
+      #
+      # Excluded packages (--exclude-packages):
+      #   - nspkg: namespace packages with no public API surface; apistub produces no output.
+      #   - azure-storage-extensions: contains C extension modules that are not importable
+      #     in a standard Python environment without a build step; apistub cannot introspect them.
+      #   - azure-mgmt-app, azure-mgmt-videoanalyzer, azure-mgmt-changeanalysis: last released
+      #     in 2022 and depend on the deprecated `msrest` package, which is not available in
+      #     the apistub virtual environment and causes an import error at generation time.
+      #   - azure-mgmt-apimanagement: contains two types whose names differ only by casing
+      #     (`LLMDiagnosticSettings` model and `LlmDiagnosticSettings` enum); PowerShell's
+      #     ConvertFrom-Json treats object keys case-insensitively and fails to parse the
+      #     generated token JSON. The package should rename one of these types to fix this.
+      - task: PythonScript@0
+        displayName: 'Regenerate api.md for all packages'
+        continueOnError: true
+        inputs:
+          scriptPath: 'eng/scripts/dispatch_checks.py'
+          arguments: >-
+            "azure-*"
+            --checks="apistub"
+            --md
+            --disablecov
+            $(FilterTypeArg)
+            --max-parallel 8
+            --exclude-packages="nspkg,azure-storage-extensions,azure-mgmt-app,azure-mgmt-videoanalyzer,azure-mgmt-changeanalysis,azure-mgmt-apimanagement"
+
+      # Stage only api.md files so the PR contains exclusively api.md updates.
+      - bash: |
+          git add -- "sdk/**/api.md"
+          if ! git diff --cached --quiet; then
+            echo "api.md changes detected; will open a draft PR."
+            git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" \
+              commit -m "[Auto] Regenerate api.md for all packages"
+            echo "##vso[task.setvariable variable=HasChanges]true"
+          else
+            echo "No api.md changes detected. Exiting successfully."
+            echo "##vso[task.setvariable variable=HasChanges]false"
+          fi
+        displayName: 'Stage and commit api.md changes'
+
+      # login to azure a single time. we will source and target the same owner, so a single login is sufficient for the PR creation step
+      # versus allowing multiple logins to occur for `Azure` twice
+      - template: /eng/common/pipelines/templates/steps/login-to-github.yml
+
+      - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+        parameters:
+          AuthToken: '$(GH_TOKEN)'
+          PROwner: 'Azure'
+          PRBranchName: 'automated/apiview-md-regen-$(Build.BuildId)'
+          CommitMsg: '[Auto] Regenerate api.md for all packages'
+          PRTitle: '[Auto] Regenerate api.md for all packages'
+          PRBody: |
+            This PR was automatically generated to regenerate `api.md` files across all Python SDK packages.
+
+            The `api.md` files are produced by `azpysdk apistub --md`, which runs the API stub generator and converts the output to a human-readable markdown file using `Export-APIViewMarkdown.ps1`.
+
+            **Why this PR was created**: the `apiview-stub-generator` version in `eng/apiview_reqs.txt` was updated, so all `api.md` files need to be regenerated to reflect any format changes.
+
+            Generated by: https://dev.azure.com/azure-sdk/internal/_build?definitionId=$(System.DefinitionId)&buildId=$(Build.BuildId)
+          BaseBranchName: main
+          OpenAsDraft: true
+          SkipCheckingForChanges: true

eng/scripts/dispatch_checks.py

@@ -138,6 +138,7 @@ async def run_check(
     mark_arg: Optional[str],
     dest_dir: Optional[str] = None,
     service: Optional[str] = None,
+    generate_md: bool = False,
     python_version: Optional[str] = None,
 ) -> CheckResult:
     """Run a single check (subprocess) within a concurrency semaphore, capturing output and timing.
@@ -156,6 +157,8 @@ async def run_check(
     :type total: int
     :param proxy_port: Dedicated proxy port assigned to this check instance.
     :type proxy_port: int
+    :param generate_md: When True and check is 'apistub', pass ``--md`` to generate api.md in-place.
+    :type generate_md: bool
     :returns: A :class:`CheckResult` describing exit code, duration and captured output.
     :rtype: CheckResult
     """
@@ -170,6 +173,8 @@ async def run_check(
             cmd += ["--mark_arg", mark_arg]
         if dest_dir and check == "apistub":
             cmd += ["--dest-dir", dest_dir]
+        if generate_md and check == "apistub":
+            cmd += ["--md"]
         logger.info(f"[START {idx}/{total}] {check} :: {package}\nCMD: {' '.join(cmd)}")
         env = os.environ.copy()
         env["PROXY_URL"] = f"http://localhost:{proxy_port}"
@@ -262,6 +267,7 @@ async def run_all_checks(
     injected_packages: str,
     dest_dir: Optional[str] = None,
     service: Optional[str] = None,
+    generate_md: bool = False,
 ):
     """Run all checks for all packages concurrently and return the worst exit code.
 
@@ -274,6 +280,8 @@ async def run_all_checks(
     :param wheel_dir: The directory where wheels should be located and stored when built.
         In CI should correspond to `$(Build.ArtifactStagingDirectory)`.
     :type wheel_dir: str
+    :param generate_md: When True, pass ``--md`` to ``apistub`` checks to generate api.md in-place.
+    :type generate_md: bool
     :returns: The worst exit code from all checks (0 if all passed).
     :rtype: int
     """
@@ -338,6 +346,7 @@ async def run_all_checks(
                     mark_arg,
                     dest_dir,
                     service,
+                    generate_md,
                     pkg_python_version,
                 )
             )
@@ -488,13 +497,28 @@ def handler(signum, frame):
         help="Maximum number of concurrent checks (default: number of CPU cores).",
     )
 
+    parser.add_argument(
+        "--md",
+        dest="generate_md",
+        action="store_true",
+        default=False,
+        help="When set, pass --md to apistub checks to generate api.md in-place.",
+    )
+
     parser.add_argument(
         "--disable-compatibility-filter",
         dest="disable_compatibility_filter",
         action="store_true",
         help="Flag to disable compatibility filter while discovering packages.",
     )
 
+    parser.add_argument(
+        "--exclude-packages",
+        dest="exclude_packages",
+        default="",
+        help="Comma-separated list of package name substrings to exclude from checks.",
+    )
+
     args = parser.parse_args()
 
     configure_logging(args)
@@ -523,6 +547,15 @@ def handler(signum, frame):
         args.glob_string, target_dir, "", args.filter_type, compatibility_filter
     )
 
+    if args.exclude_packages:
+        exclude_substrings = [s.strip() for s in args.exclude_packages.split(",") if s.strip()]
+        before = len(targeted_packages)
+        targeted_packages = [
+            p for p in targeted_packages
+            if not any(sub in os.path.basename(os.path.normpath(p)) for sub in exclude_substrings)
+        ]
+        logger.info(f"Excluded {before - len(targeted_packages)} packages matching: {exclude_substrings}")
+
     if len(targeted_packages) == 0:
         logger.info(f"No packages collected for targeting string {args.glob_string} and root dir {root_dir}. Exit 0.")
         exit(0)
@@ -580,6 +613,7 @@ def handler(signum, frame):
                 args.injected_packages,
                 args.dest_dir,
                 effective_service,
+                args.generate_md,
             )
         )
     except KeyboardInterrupt:

eng/tools/azure-sdk-tools/azpysdk/apistub.py

@@ -8,15 +8,15 @@
 from .Check import Check
 from ci_tools.functions import install_into_venv, find_whl
 from ci_tools.scenario.generation import create_package_and_install
-from ci_tools.variables import discover_repo_root, set_envvar_defaults
+from ci_tools.variables import discover_repo_root, set_envvar_defaults, in_ci
 from ci_tools.logging import logger
 from ci_tools.parsing import ParsedSetup
 
 REPO_ROOT = discover_repo_root()
 PYTHON_VERSION_LIMIT = (3, 11)  # apistub doesn't support Python 3.11+
 
 
-def get_package_wheel_path(pkg_root: str) -> str:
+def get_package_wheel_path(pkg_root: str, staging_dir: str = None) -> str:
     # parse setup.py to get package name and version
     pkg_details = ParsedSetup.from_path(pkg_root)
 
@@ -33,7 +33,12 @@ def get_package_wheel_path(pkg_root: str) -> str:
                 )
             )
         return pkg_path
-    # Otherwise, use wheel created in staging directory, or fall back on source directory
+    # Check staging directory first (wheel built by create_package_and_install)
+    if staging_dir:
+        found_whl = find_whl(staging_dir, pkg_details.name, pkg_details.version)
+        if found_whl:
+            return os.path.join(staging_dir, found_whl)
+    # Otherwise, use wheel in source directory, or fall back on source directory
     pkg_path = find_whl(pkg_root, pkg_details.name, pkg_details.version) or pkg_root
     return pkg_path
 
@@ -119,23 +124,36 @@ def run(self, args: argparse.Namespace) -> int:
                 return e.returncode
 
             if not os.getenv("PREBUILT_WHEEL_DIR"):
-                create_package_and_install(
-                    distribution_directory=staging_directory,
-                    target_setup=package_dir,
-                    skip_install=True,
-                    cache_dir=None,
-                    work_dir=staging_directory,
-                    force_create=False,
-                    package_type="wheel",
-                    pre_download_disabled=False,
-                    python_executable=executable,
-                )
+                try:
+                    create_package_and_install(
+                        distribution_directory=staging_directory,
+                        target_setup=package_dir,
+                        skip_install=True,
+                        cache_dir=None,
+                        work_dir=staging_directory,
+                        force_create=False,
+                        package_type="wheel",
+                        pre_download_disabled=False,
+                        python_executable=executable,
+                    )
+                except Exception as e:
+                    logger.error(f"{package_name}: failed to build/install wheel: {e}")
+                    results.append(1)
+                    continue
 
             self.pip_freeze(executable)
 
-            pkg_path = get_package_wheel_path(package_dir)
+            pkg_path = get_package_wheel_path(package_dir, staging_directory)
             pkg_path = os.path.abspath(pkg_path)
 
+            if in_ci():
+                # In CI, pre-install the package and its deps into the venv so that when
+                # apistubgen's internal _install_package() runs pip install, all
+                # dependencies are already satisfied and the call finishes instantly
+                # instead of hitting the 120s timeout under parallel CI load. Locally,
+                # apistubgen handles this install itself.
+                install_into_venv(executable, [pkg_path], package_dir)
+
             dest_dir = getattr(args, "dest_dir", None)
             if dest_dir:
                 out_token_path = os.path.join(os.path.abspath(dest_dir), package_name)
@@ -160,14 +178,32 @@ def run(self, args: argparse.Namespace) -> int:
             logger.info("Running apistub {}.".format(cmds))
 
             try:
-                self.run_venv_command(executable, cmds, cwd=staging_directory, check=True, immediately_dump=True)
+                apistub_result = self.run_venv_command(
+                    executable, cmds, cwd=staging_directory, check=False, immediately_dump=False
+                )
+                if apistub_result.stdout:
+                    logger.info(apistub_result.stdout)
+                if apistub_result.stderr:
+                    logger.warning(apistub_result.stderr)
+                if apistub_result.returncode != 0:
+                    logger.error(f"{package_name} apistub exited with code {apistub_result.returncode}")
+                    results.append(apistub_result.returncode)
+                    continue
                 if getattr(args, "generate_md", False):
                     token_json_path = os.path.join(out_token_path, f"{package_name}_python.json")
                     md_script = os.path.join(REPO_ROOT, "eng", "common", "scripts", "Export-APIViewMarkdown.ps1")
+                    if not os.path.exists(token_json_path):
+                        logger.error(f"Token JSON not found at expected path: {token_json_path}")
+                        results.append(1)
+                        continue
                     logger.info(f"Generating api.md for {package_name}")
+                    # When no --dest-dir is given, write api.md directly into the package
+                    # directory so it is tracked by git. When --dest-dir is provided, keep
+                    # the existing behaviour of writing into <dest_dir>/<package_name>/.
+                    md_output_path = package_dir if not dest_dir else out_token_path
                     try:
                         result = run(
-                            ["pwsh", md_script, "-TokenJsonPath", token_json_path, "-OutputPath", out_token_path],
+                            ["pwsh", md_script, "-TokenJsonPath", token_json_path, "-OutputPath", md_output_path],
                             check=True,
                             capture_output=True,
                             text=True,