diff --git a/HISTORY.rst b/HISTORY.rst index 69c478ff3..1e58a38f2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ Release History =============== +0.2.9 ++++++ +* `azdev latest-index`: Add `generate` and `verify` commands to manage Azure CLI packaged latest indices (`commandIndex.latest.json`, `helpIndex.latest.json`) with CI-friendly verify exit behavior. + 0.2.8 ++++++ * Pin pip to 25.2 as pip 25.3 remove support for the legacy setup.py develop editable method in setuptools editable installs; setuptools >= 64 is now required. (#11457) diff --git a/README.md b/README.md index 1b09cf8c8..3ea1af51b 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,25 @@ For instructions on manually writing the commands and tests, see more in By default, test is running in `once` mode. If there are no corresponding recording files (in yaml format), it will run live tests and generate recording files. If recording files are found, the tests will be run in `playback` mode against the recording files. You can use `--live` to force a test run in `live` mode and regenerate the recording files. +## Latest packaged indices + +Use azdev wrappers around Azure CLI's latest index generation script: + +``` +azdev latest-index generate +azdev latest-index verify +``` + +You can pass an explicit Azure CLI checkout path when needed: + +``` +azdev latest-index generate --cli /path/to/azure-cli +azdev latest-index verify --repo /path/to/azure-cli +``` + +`azdev latest-index verify` exits non-zero when generated output differs from the checked-in +`commandIndex.latest.json` or `helpIndex.latest.json`, making it CI-friendly. + ## Submitting a pull request to merge the code 1. After committing your code locally, push it to your forked repository: diff --git a/README.rst b/README.rst index 50f92599a..5a68c1ad4 100644 --- a/README.rst +++ b/README.rst @@ -66,6 +66,49 @@ Setting up your development environment This will launch the interactive setup process. To see non-interactive options run `azdev setup -h`. +Latest packaged indices ++++++++++++++++++++++++ + +Use azdev wrappers around Azure CLI's latest index generation script: + +:: + + azdev latest-index generate + azdev latest-index verify + +You can pass an explicit Azure CLI checkout path when needed: + +:: + + azdev latest-index generate --cli /path/to/azure-cli + azdev latest-index verify --repo /path/to/azure-cli + +``azdev latest-index verify`` exits non-zero when generated output differs from checked-in +``commandIndex.latest.json`` or ``helpIndex.latest.json``. + +Common azdev commands ++++++++++++++++++++++++++++++ + +This README is not an exhaustive command reference. For the complete command surface, use: + +:: + + azdev --help + azdev --help + +Frequently used commands include: + +:: + + azdev setup + azdev style + azdev linter + azdev test + azdev extension add + azdev extension build + azdev latest-index generate + azdev latest-index verify + Reporting issues and feedback +++++++++++++++++++++++++++++ diff --git a/azdev/__init__.py b/azdev/__init__.py index 699b60427..586ab2bd1 100644 --- a/azdev/__init__.py +++ b/azdev/__init__.py @@ -4,4 +4,4 @@ # license information. # ----------------------------------------------------------------------------- -__VERSION__ = '0.2.8' +__VERSION__ = '0.2.9' diff --git a/azdev/commands.py b/azdev/commands.py index 53974827f..91ec2a837 100644 --- a/azdev/commands.py +++ b/azdev/commands.py @@ -82,6 +82,10 @@ def operation_group(name): with CommandGroup(self, 'cli', operation_group('help')) as g: g.command('generate-docs', 'generate_cli_ref_docs') + with CommandGroup(self, 'latest-index', operation_group('latest_index')) as g: + g.command('generate', 'generate_latest_index') + g.command('verify', 'verify_latest_index') + with CommandGroup(self, 'extension', operation_group('help')) as g: g.command('generate-docs', 'generate_extension_ref_docs') diff --git a/azdev/help.py b/azdev/help.py index 93e3f44b9..8edc56901 100644 --- a/azdev/help.py +++ b/azdev/help.py @@ -99,6 +99,34 @@ short-summary: Verify the README and HISTORY files for each module so they format correctly on PyPI. """ +helps['latest-index'] = """ + short-summary: Generate or verify Azure CLI packaged latest index files. + long-summary: > + Wraps azure-cli's scripts/generate_latest_indices.py for deterministic, CI-friendly + generation and verification of commandIndex.latest.json and helpIndex.latest.json. +""" + +helps['latest-index generate'] = """ + short-summary: Generate commandIndex.latest.json and helpIndex.latest.json in an Azure CLI repo. + examples: + - name: Generate latest index files using the configured Azure CLI repo. + text: azdev latest-index generate + + - name: Generate latest index files for an explicit repo checkout. + text: azdev latest-index generate --cli /path/to/azure-cli +""" + +helps['latest-index verify'] = """ + short-summary: Verify latest index files are up-to-date. + long-summary: Returns a non-zero exit code when generated content differs from checked-in files. + examples: + - name: Verify latest index files in CI. + text: azdev latest-index verify + + - name: Verify latest index files for an explicit repo checkout. + text: azdev latest-index verify --repo /path/to/azure-cli +""" + helps['style'] = """ short-summary: Check code style (pylint and PEP8). diff --git a/azdev/operations/latest_index.py b/azdev/operations/latest_index.py new file mode 100644 index 000000000..de47a4b4e --- /dev/null +++ b/azdev/operations/latest_index.py @@ -0,0 +1,71 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import os +import sys + +from knack.util import CLIError + +from azdev.utilities import display, heading, py_cmd +from azdev.utilities.path import get_cli_repo_path + + +_LATEST_INDEX_SCRIPT = os.path.join('scripts', 'generate_latest_indices.py') + + +def _resolve_cli_repo_path(cli_path): + if cli_path: + resolved = os.path.abspath(os.path.expanduser(cli_path)) + else: + resolved = get_cli_repo_path() + + if not resolved or resolved == '_NONE_': + raise CLIError('Azure CLI repo path is not configured. Specify `--cli` or run `azdev setup`.') + + if not os.path.isdir(resolved): + raise CLIError('Azure CLI repo path does not exist: {}'.format(resolved)) + + return resolved + + +def _run_latest_index(mode, cli_path=None, profile='latest', all_profiles=False): + if all_profiles: + raise CLIError('`--all-profiles` is not supported yet. Use `--profile latest`.') + + if profile != 'latest': + raise CLIError("Unsupported profile '{}'. Only `latest` is currently supported.".format(profile)) + + repo_path = _resolve_cli_repo_path(cli_path) + script_path = os.path.join(repo_path, _LATEST_INDEX_SCRIPT) + if not os.path.isfile(script_path): + raise CLIError('Unable to find azure-cli script: {}'.format(script_path)) + + heading('Latest Index: {}'.format(mode.capitalize())) + display('Azure CLI repo: {}'.format(repo_path)) + + command = '{} {}'.format(_LATEST_INDEX_SCRIPT, mode) + result = py_cmd(command, is_module=False, cwd=repo_path) + + output = result.result + if isinstance(output, bytes): + output = output.decode('utf-8', errors='replace') + if output: + output = output.replace( + 'python scripts/generate_latest_indices.py generate', + 'azdev latest-index generate' + ) + display(output) + + if result.exit_code: + sys.exit(result.exit_code) + + +def generate_latest_index(cli_path=None, profile='latest', all_profiles=False): + _run_latest_index('generate', cli_path=cli_path, profile=profile, all_profiles=all_profiles) + + +def verify_latest_index(cli_path=None, profile='latest', all_profiles=False): + _run_latest_index('verify', cli_path=cli_path, profile=profile, all_profiles=all_profiles) diff --git a/azdev/operations/tests/test_latest_index.py b/azdev/operations/tests/test_latest_index.py new file mode 100644 index 000000000..85319fd5d --- /dev/null +++ b/azdev/operations/tests/test_latest_index.py @@ -0,0 +1,78 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import unittest +from unittest import mock +import os + +from knack.util import CLIError, CommandResultItem + +from azdev.operations.latest_index import generate_latest_index, verify_latest_index + + +class LatestIndexTestCase(unittest.TestCase): + + @mock.patch('azdev.operations.latest_index.py_cmd') + @mock.patch('azdev.operations.latest_index.os.path.isfile', return_value=True) + @mock.patch('azdev.operations.latest_index.os.path.isdir', return_value=True) + def test_generate_with_explicit_repo_path(self, _, __, mock_py_cmd): + mock_py_cmd.return_value = CommandResultItem('generated', exit_code=0, error=None) + + generate_latest_index(cli_path='/fake/azure-cli') + + self.assertTrue(mock_py_cmd.called) + command = mock_py_cmd.call_args.args[0] + self.assertIn('generate_latest_indices.py generate', command) + self.assertEqual(os.path.abspath('/fake/azure-cli'), mock_py_cmd.call_args.kwargs['cwd']) + self.assertFalse(mock_py_cmd.call_args.kwargs['is_module']) + + @mock.patch('azdev.operations.latest_index.py_cmd') + @mock.patch('azdev.operations.latest_index.os.path.isfile', return_value=True) + @mock.patch('azdev.operations.latest_index.os.path.isdir', return_value=True) + @mock.patch('azdev.operations.latest_index.get_cli_repo_path', return_value='/configured/azure-cli') + def test_verify_uses_configured_repo_path(self, _, __, ___, mock_py_cmd): + mock_py_cmd.return_value = CommandResultItem('verified', exit_code=0, error=None) + + verify_latest_index() + + self.assertEqual('/configured/azure-cli', mock_py_cmd.call_args.kwargs['cwd']) + + @mock.patch('azdev.operations.latest_index.py_cmd') + @mock.patch('azdev.operations.latest_index.os.path.isfile', return_value=True) + @mock.patch('azdev.operations.latest_index.os.path.isdir', return_value=True) + def test_verify_non_zero_exit_is_propagated(self, _, __, mock_py_cmd): + # simulate bytes output as returned by py_cmd on failure + mock_py_cmd.return_value = CommandResultItem(b'stale\r\n', exit_code=1, error='mismatch') + + with self.assertRaises(SystemExit) as ex: + verify_latest_index(cli_path='/fake/azure-cli') + + self.assertEqual(1, ex.exception.code) + + def test_non_latest_profile_is_rejected(self): + with self.assertRaises(CLIError): + generate_latest_index(cli_path='/fake/azure-cli', profile='2019-03-01-hybrid') + + def test_all_profiles_flag_is_rejected(self): + with self.assertRaises(CLIError): + verify_latest_index(cli_path='/fake/azure-cli', all_profiles=True) + + @mock.patch('azdev.operations.latest_index.display') + @mock.patch('azdev.operations.latest_index.py_cmd') + @mock.patch('azdev.operations.latest_index.os.path.isfile', return_value=True) + @mock.patch('azdev.operations.latest_index.os.path.isdir', return_value=True) + def test_bytes_output_decoded_and_hint_replaced(self, _, __, mock_py_cmd, mock_display): + raw = b'files are out of date\r\nRun:\r\n python scripts/generate_latest_indices.py generate\r\n' + mock_py_cmd.return_value = CommandResultItem(raw, exit_code=1, error='mismatch') + + with self.assertRaises(SystemExit): + verify_latest_index(cli_path='/fake/azure-cli') + + displayed = mock_display.call_args.args[0] + self.assertNotIn("b'", displayed) + self.assertNotIn('\\r\\n', displayed) + self.assertNotIn('python scripts/generate_latest_indices.py generate', displayed) + self.assertIn('azdev latest-index generate', displayed) diff --git a/azdev/params.py b/azdev/params.py index 3ac30beb3..444ae5f16 100644 --- a/azdev/params.py +++ b/azdev/params.py @@ -282,3 +282,11 @@ def load_arguments(self, _): c.argument('no_tail', action='store_true', help='Skip tail when displaying as markdown.') c.argument('include_whl_extensions', action='store_true', help="Allow scanning on extensions installed by `az extension add --source xxx.whl`.") + + with ArgumentsContext(self, 'latest-index') as c: + c.argument('cli_path', options_list=['--cli', '--repo'], + help='Path to an Azure CLI repo checkout. If omitted, use the path configured by `azdev setup`.') + c.argument('profile', choices=['latest'], default='latest', + help='Cloud profile to process. Only `latest` is currently supported.') + c.argument('all_profiles', action='store_true', + help='Not supported yet. Reserved for future multi-profile support.') diff --git a/azure-pipelines-cli.yml b/azure-pipelines-cli.yml index 892a1af89..50a2d5582 100644 --- a/azure-pipelines-cli.yml +++ b/azure-pipelines-cli.yml @@ -82,10 +82,12 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python39: - python.version: '3.9' + Python310: + python.version: '3.10' Python312: python.version: '3.12' + Python313: + python.version: '3.13' steps: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' @@ -139,10 +141,12 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python39: - python.version: '3.9' + Python310: + python.version: '3.10' Python312: python.version: '3.12' + Python313: + python.version: '3.13' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -171,10 +175,12 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python39: - python.version: '3.9' + Python310: + python.version: '3.10' Python312: python.version: '3.12' + Python313: + python.version: '3.13' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -203,10 +209,12 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python39: - python.version: '3.9' + Python310: + python.version: '3.10' Python312: python.version: '3.12' + Python313: + python.version: '3.13' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -235,10 +243,12 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python39: - python.version: '3.9' + Python310: + python.version: '3.10' Python312: python.version: '3.12' + Python313: + python.version: '3.13' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -266,10 +276,12 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python39: - python.version: '3.9' + Python310: + python.version: '3.10' Python312: python.version: '3.12' + Python313: + python.version: '3.13' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build'