diff --git a/lib/devtools/README.md b/lib/devtools/README.md index ae3c5a5085..2faac201f3 100644 --- a/lib/devtools/README.md +++ b/lib/devtools/README.md @@ -39,9 +39,13 @@ devtools release 1.10.3 --skip-enterprise # skip enterprise release phase 7. Opens a `[docs-freeze]` PR against main, polls until merged 8. Tags main and creates GitHub release 9. Triggers PyPI publish workflow -10. Clones enterprise repo, bumps versions and `crewai[tools]` dep, runs `uv sync` -11. Creates enterprise bump PR, polls until merged -12. Tags and creates GitHub release on enterprise repo +10. Updates `crewAIInc/crew_deployment_test` to the exact CrewAI version, + creates a bump PR, and waits for it to merge +11. Updates `crewAIInc/flow_deployment_test` to the exact CrewAI version, + creates a bump PR, and waits for it to merge +12. Clones enterprise repo, bumps versions and `crewai[tools]` dep, runs `uv sync` +13. Creates enterprise bump PR, polls until merged +14. Tags and creates GitHub release on enterprise repo > The `docs-snapshots` CI guard rejects writes under `docs/v*/` and deletions/renames in `docs/images/` unless the PR title starts with `[docs-freeze]`. The release CLI sets that prefix automatically; manual edits to a frozen snapshot need the same prefix to land. > @@ -66,4 +70,4 @@ Tag and release only (phase 2 of `release`). Run after the bump PR is merged. devtools tag devtools tag --no-edit devtools tag --dry-run -``` \ No newline at end of file +``` diff --git a/lib/devtools/pyproject.toml b/lib/devtools/pyproject.toml index 98ba51f595..2c54ab650a 100644 --- a/lib/devtools/pyproject.toml +++ b/lib/devtools/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "openai>=1.83.0,<3", "python-dotenv>=1.2.2,<2", "pygithub~=1.59.1", + "pyyaml~=6.0", "rich>=13.9.4", ] diff --git a/lib/devtools/src/crewai_devtools/cli.py b/lib/devtools/src/crewai_devtools/cli.py index c749bc6686..a3c003580f 100644 --- a/lib/devtools/src/crewai_devtools/cli.py +++ b/lib/devtools/src/crewai_devtools/cli.py @@ -4,6 +4,7 @@ import os from pathlib import Path import re +import shlex import subprocess import sys import tempfile @@ -20,6 +21,7 @@ from rich.panel import Panel from rich.prompt import Confirm import tomlkit +import yaml from crewai_devtools.docs_check import docs_check from crewai_devtools.docs_versioning import ( @@ -1421,7 +1423,10 @@ def _repin_crewai_install(run_value: str, version: str) -> str: return "".join(result) -_DEPLOYMENT_TEST_REPO: Final[str] = "crewAIInc/crew_deployment_test" +_DEPLOYMENT_TEST_REPOS: Final[tuple[str, ...]] = ( + "crewAIInc/crew_deployment_test", + "crewAIInc/flow_deployment_test", +) _PUBLISHED_WORKSPACE_PACKAGES: Final[tuple[str, ...]] = ( "crewai", @@ -1435,26 +1440,164 @@ def _repin_crewai_install(run_value: str, version: str) -> str: _PYPI_POLL_TIMEOUT: Final[int] = 600 -def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None: - """Update the deployment test repo to pin the new crewai version. +_CREWAI_REQUIREMENT_PATTERN: Final[re.Pattern[str]] = re.compile( + r"^crewai(?:\s*\[[^\]]+\])?(?![\w-])" + r"\s*(?:(?P===|==|~=|!=|>=|<=|>|<)\s*" + r"(?P[^\s;]+))?", + re.IGNORECASE, +) + + +def _crewai_requirement_pin(requirement: str) -> str | None: + """Return an exact CrewAI pin, or an empty string for a non-exact pin.""" + match = _CREWAI_REQUIREMENT_PATTERN.match(requirement.strip()) + if not match: + return None + if match.group("operator") != "==": + return "" + return match.group("version") or "" + + +def _pyproject_crewai_requirements(content: str) -> list[tuple[str, str]]: + """Collect active CrewAI dependency requirements from pyproject content.""" + requirements: list[tuple[str, str]] = [] + doc = tomlkit.parse(content) + for key in ("dependencies", "optional-dependencies"): + deps = doc.get("project", {}).get(key) + if deps is None: + continue + dep_lists = deps.values() if isinstance(deps, Mapping) else [deps] + for dep_list in dep_lists: + for dep in dep_list: + spec = str(dep) + pin = _crewai_requirement_pin(spec) + if pin is not None: + requirements.append((spec, pin)) + return requirements + + +def _workflow_run_commands(content: str) -> list[str]: + """Extract shell commands from workflow ``run`` values.""" + commands: list[str] = [] + + def collect_run_commands(node: object) -> None: + if isinstance(node, Mapping): + for key, value in node.items(): + if key == "run" and isinstance(value, str): + commands.append(value) + collect_run_commands(value) + elif isinstance(node, list): + for value in node: + collect_run_commands(value) + + collect_run_commands(yaml.safe_load(content)) + return commands + + +def _workflow_crewai_requirements(content: str) -> list[tuple[str, str]]: + """Collect CrewAI requirements from executable workflow install commands.""" + requirements: list[tuple[str, str]] = [] + for command in _workflow_run_commands(content): + normalized = command.replace("\\\n", " ") + lexer = shlex.shlex(normalized, posix=True, punctuation_chars=";&|\n") + lexer.whitespace = " \t\r" + lexer.whitespace_split = True + lexer.commenters = "#" + try: + tokens = list(lexer) + except ValueError: + continue + + index = 0 + while index < len(tokens): + command_lengths = ( + (tokens[index : index + 3] == ["uv", "pip", "install"], 3), + (tokens[index : index + 2] == ["uv", "add"], 2), + ( + tokens[index : index + 2] + in (["pip", "install"], ["pip3", "install"]), + 2, + ), + ( + tokens[index : index + 4] + in ( + ["python", "-m", "pip", "install"], + ["python3", "-m", "pip", "install"], + ), + 4, + ), + ) + install_length = next( + (length for matched, length in command_lengths if matched), + 0, + ) + if not install_length: + index += 1 + continue + + index += install_length + while index < len(tokens) and tokens[index] not in { + ";", + "&&", + "||", + "|", + "\n", + }: + argument = tokens[index] + pin = _crewai_requirement_pin(argument) + if pin is not None: + requirements.append((argument, pin)) + index += 1 + return requirements + + +def _validate_deployment_repo_crewai_pin( + repo_dir: Path, + pyproject_content: str, + version: str, +) -> None: + """Fail unless every effective canary CrewAI requirement has the exact pin.""" + requirements = _pyproject_crewai_requirements(pyproject_content) + + workflows_dir = repo_dir / ".github" / "workflows" + if workflows_dir.exists(): + for workflow in workflows_dir.iterdir(): + if workflow.is_file() and workflow.suffix in (".yml", ".yaml"): + requirements.extend( + _workflow_crewai_requirements(workflow.read_text(encoding="utf-8")) + ) + + if not requirements: + raise RuntimeError(f"No effective CrewAI dependency found in {repo_dir.name}") + + mismatches = [spec for spec, pin in requirements if pin != version] + if mismatches: + found = ", ".join(repr(spec) for spec in mismatches) + raise RuntimeError( + f"CrewAI dependencies in {repo_dir.name} must all pin {version}; " + f"found {found}" + ) - Clones the repo, updates the crewai[tools] pin in pyproject.toml + +def _update_deployment_test_repo(repo: str, version: str, is_prerelease: bool) -> None: + """Update a deployment test repo to pin the new crewai version. + + Clones the repo, updates the CrewAI pin in pyproject.toml and any crewai[extras] pins in .github/workflows, regenerates the lockfile, commits to a branch, pushes, opens a PR against main, then polls until the PR is merged (or closed). Args: + repo: GitHub repository containing the deployment canary. version: New crewai version string. is_prerelease: Whether this is a pre-release version. """ - console.print( - f"\n[bold cyan]Updating {_DEPLOYMENT_TEST_REPO} to {version}[/bold cyan]" - ) + console.print(f"\n[bold cyan]Updating {repo} to {version}[/bold cyan]") with tempfile.TemporaryDirectory() as tmp: - repo_dir = Path(tmp) / "crew_deployment_test" - run_command(["gh", "repo", "clone", _DEPLOYMENT_TEST_REPO, str(repo_dir)]) - console.print(f"[green]✓[/green] Cloned {_DEPLOYMENT_TEST_REPO}") + repo_dir = Path(tmp) / repo.rsplit("/", 1)[-1] + run_command(["gh", "repo", "clone", repo, str(repo_dir)]) + console.print(f"[green]✓[/green] Cloned {repo}") pyproject = repo_dir / "pyproject.toml" content = pyproject.read_text() @@ -1462,11 +1605,9 @@ def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None: pyproject_changed = new_content != content if pyproject_changed: pyproject.write_text(new_content) - console.print(f"[green]✓[/green] Updated crewai[tools] pin to {version}") + console.print(f"[green]✓[/green] Updated crewai pin to {version}") else: - console.print( - "[yellow]Warning:[/yellow] No crewai[tools] pin found to update" - ) + console.print("[yellow]Warning:[/yellow] No crewai pin found to update") updated_workflows = _update_repo_workflows_crewai_pins(repo_dir, version) for wf in updated_workflows: @@ -1474,6 +1615,8 @@ def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None: f"[green]✓[/green] Updated crewai pin in {wf.relative_to(repo_dir)}" ) + _validate_deployment_repo_crewai_pin(repo_dir, new_content, version) + if not pyproject_changed and not updated_workflows: console.print("[yellow]Nothing to update; skipping commit and PR.[/yellow]") return @@ -1535,12 +1678,18 @@ def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None: ], cwd=repo_dir, ) - console.print(f"[green]✓[/green] Opened PR on {_DEPLOYMENT_TEST_REPO}") + console.print(f"[green]✓[/green] Opened PR on {repo}") console.print(f"[cyan]PR URL:[/cyan] {pr_url.strip()}") _wait_for_pr_merged(branch, repo_dir) +def _update_deployment_test_repos(version: str, is_prerelease: bool) -> None: + """Pin and merge the release version in every deployment canary repo.""" + for repo in _DEPLOYMENT_TEST_REPOS: + _update_deployment_test_repo(repo, version, is_prerelease) + + def _wait_for_pypi(package: str, version: str) -> None: """Poll PyPI until a specific package version is available. @@ -2352,13 +2501,13 @@ def release( try: if not dry_run: - _update_deployment_test_repo(version, is_prerelease) + _update_deployment_test_repos(version, is_prerelease) except BaseException as e: _print_release_error(e) _resume_hint( - f"Phase 2 failed updating deployment test repo. " + f"Phase 2 failed updating deployment test repos. " f"Tag, release, and PyPI are done.\n" - f"Fix the issue and update {_DEPLOYMENT_TEST_REPO} manually." + "Fix the issue and update the Crew and Flow canary repos manually." f"{enterprise_hint}" ) sys.exit(1) diff --git a/lib/devtools/tests/test_toml_updates.py b/lib/devtools/tests/test_toml_updates.py index 80b18648d1..6fd9fa1fd7 100644 --- a/lib/devtools/tests/test_toml_updates.py +++ b/lib/devtools/tests/test_toml_updates.py @@ -3,14 +3,217 @@ from pathlib import Path from textwrap import dedent +from crewai_devtools import cli as devtools_cli from crewai_devtools.cli import ( _DEFAULT_WORKSPACE_PACKAGES, _pin_crewai_deps, _repin_crewai_install, + _validate_deployment_repo_crewai_pin, update_pyproject_dependencies, update_pyproject_version, update_template_dependencies, ) +import pytest + + +def test_release_updates_crew_and_flow_canary_repositories(monkeypatch) -> None: + updates = [] + monkeypatch.setattr( + devtools_cli, + "_update_deployment_test_repo", + lambda repo, version, is_prerelease: updates.append( + (repo, version, is_prerelease) + ), + ) + + devtools_cli._update_deployment_test_repos("2.0.0a1", True) + + assert updates == [ + ("crewAIInc/crew_deployment_test", "2.0.0a1", True), + ("crewAIInc/flow_deployment_test", "2.0.0a1", True), + ] + + +def test_deployment_repo_validation_rejects_missing_crewai_pin(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="No effective CrewAI dependency"): + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_accepts_workflow_pin(tmp_path: Path) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text('run: uv pip install "crewai[a2a]==2.0.0"\n') + + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +@pytest.mark.parametrize( + "run_value", + [ + "'uv pip install \"crewai==2.0.0\"'", + '"uv pip install \\"crewai==2.0.0\\""', + ], +) +def test_deployment_repo_validation_accepts_quoted_workflow_pin( + tmp_path: Path, + run_value: str, +) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text( + f"run: {run_value}\n", + encoding="utf-8", + ) + + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_rejects_mixed_versions(tmp_path: Path) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text('run: uv pip install "crewai[a2a]==2.0.0"\n') + + with pytest.raises(RuntimeError, match=r"must all pin 2\.0\.0"): + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["crewai==1.0.0"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_ignores_comments_and_echo(tmp_path: Path) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text( + 'run: echo "crewai==2.0.0"\n# run: pip install crewai==2.0.0\n' + ) + + with pytest.raises(RuntimeError, match="No effective CrewAI dependency"): + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_ignores_pyproject_comment_pin( + tmp_path: Path, +) -> None: + with pytest.raises(RuntimeError, match=r"must all pin 2\.0\.0"): + _validate_deployment_repo_crewai_pin( + tmp_path, + ( + "# documented pin: crewai==2.0.0\n" + '[project]\ndependencies = ["crewai>=1.0"]\n' + ), + "2.0.0", + ) + + +def test_deployment_repo_validation_accepts_spaced_extras_and_marker( + tmp_path: Path, +) -> None: + _validate_deployment_repo_crewai_pin( + tmp_path, + ( + "[project]\ndependencies = [\n" + " \"crewai[tools, embeddings]==2.0.0; python_version >= '3.10'\",\n" + "]\n" + ), + "2.0.0", + ) + + +def test_deployment_repo_validation_reads_multiline_workflow_install( + tmp_path: Path, +) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text( + "steps:\n" + " - name: Install\n" + " run: |\n" + " uv pip install \\\n" + " \"crewai[tools, embeddings]==2.0.0; python_version >= '3.10'\"\n" + ) + + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_reads_install_after_comment( + tmp_path: Path, +) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text( + "steps:\n" + " - name: Install\n" + " run: |\n" + " # Install the canary dependency\n" + ' uv pip install "crewai==2.0.0"\n', + encoding="utf-8", + ) + + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_reads_folded_workflow_install( + tmp_path: Path, +) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "test.yml").write_text( + "steps:\n" + " - name: Install\n" + " run: >\n" + " uv pip install\n" + ' "crewai==2.0.0"\n', + encoding="utf-8", + ) + + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) + + +def test_deployment_repo_validation_skips_non_file_workflow_entries( + tmp_path: Path, +) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "ignored.yml").mkdir() + (workflows / "test.yaml").write_text( + '# UTF-8 workflow: déploiement\nrun: uv pip install "crewai==2.0.0"\n', + encoding="utf-8", + ) + + _validate_deployment_repo_crewai_pin( + tmp_path, + '[project]\ndependencies = ["requests>=2"]\n', + "2.0.0", + ) class TestUpdatePyprojectVersion: diff --git a/uv.lock b/uv.lock index 0498c5df92..dc36588e02 100644 --- a/uv.lock +++ b/uv.lock @@ -1564,6 +1564,7 @@ dependencies = [ { name = "openai" }, { name = "pygithub" }, { name = "python-dotenv" }, + { name = "pyyaml" }, { name = "rich" }, { name = "tomlkit" }, ] @@ -1574,6 +1575,7 @@ requires-dist = [ { name = "openai", specifier = ">=1.83.0,<3" }, { name = "pygithub", specifier = "~=1.59.1" }, { name = "python-dotenv", specifier = ">=1.2.2,<2" }, + { name = "pyyaml", specifier = "~=6.0" }, { name = "rich", specifier = ">=13.9.4" }, { name = "tomlkit", specifier = "~=0.13.2" }, ]