From 7df316fe1d5c8aaddca4b982d0e88f40f8fe6722 Mon Sep 17 00:00:00 2001 From: Uday Date: Tue, 28 Apr 2026 14:57:57 +0530 Subject: [PATCH 01/14] feat: add caller and CI provider to User-Agent and tracking payload Addresses LCHUX-315. The backend can now identify whether CLI invocations come from direct usage, our GitHub Action wrapper, or another integration by reading the User-Agent header on every request. - SMART_TESTS_CALLER env var (defaults to "cli") lets wrappers identify themselves (e.g. github-action, jenkins-plugin) - detect_ci_provider() auto-detects the CI environment from standard env vars (GITHUB_ACTIONS, JENKINS_URL, CIRCLECI, CODEBUILD_BUILD_ID) - Both values are appended to User-Agent as Caller/{value} CI/{value} - Both values are also included in the cli_tracking payload Co-Authored-By: Claude Opus 4.6 (1M context) --- smart_tests/utils/env_keys.py | 13 +++++++++++++ smart_tests/utils/http_client.py | 9 ++++++++- smart_tests/utils/tracking.py | 6 ++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/smart_tests/utils/env_keys.py b/smart_tests/utils/env_keys.py index 4076d4da8..b4ad2ff9d 100644 --- a/smart_tests/utils/env_keys.py +++ b/smart_tests/utils/env_keys.py @@ -9,6 +9,7 @@ COMMIT_TIMEOUT = "SMART_TESTS_COMMIT_TIMEOUT" SKIP_CERT_VERIFICATION = "SMART_TESTS_SKIP_CERT_VERIFICATION" SESSION_DIR_KEY = "SMART_TESTS_SESSION_DIR" +CALLER_KEY = "SMART_TESTS_CALLER" # Legacy token key for backward compatibility LEGACY_TOKEN_KEY = "LAUNCHABLE_TOKEN" @@ -17,3 +18,15 @@ def get_token(): """Get token with backward compatibility for LAUNCHABLE_TOKEN.""" return os.getenv(TOKEN_KEY) or os.getenv(LEGACY_TOKEN_KEY) + + +def detect_ci_provider() -> str: + if os.environ.get("GITHUB_ACTIONS"): + return "github-actions" + if os.environ.get("JENKINS_URL"): + return "jenkins" + if os.environ.get("CIRCLECI"): + return "circleci" + if os.environ.get("CODEBUILD_BUILD_ID"): + return "codebuild" + return "" diff --git a/smart_tests/utils/http_client.py b/smart_tests/utils/http_client.py index ee6a9523d..bf8214b5a 100644 --- a/smart_tests/utils/http_client.py +++ b/smart_tests/utils/http_client.py @@ -12,7 +12,7 @@ from ..app import Application from .authentication import authentication_headers -from .env_keys import BASE_URL_KEY, SKIP_TIMEOUT_RETRY +from .env_keys import BASE_URL_KEY, CALLER_KEY, SKIP_TIMEOUT_RETRY, detect_ci_provider from .gzipgen import compress as gzipgen_compress from .logger import Logger @@ -134,6 +134,13 @@ def _headers(self, compress): if self.test_runner: h["User-Agent"] = h["User-Agent"] + f" TestRunner/{self.test_runner}" + caller = os.environ.get(CALLER_KEY) or "cli" + h["User-Agent"] += f" Caller/{caller}" + + ci_provider = detect_ci_provider() + if ci_provider: + h["User-Agent"] += f" CI/{ci_provider}" + return {**h, **authentication_headers()} diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index 8eaade13e..ba52478af 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -1,3 +1,4 @@ +import os from enum import Enum from typing import Any, Dict, Union @@ -5,6 +6,7 @@ from smart_tests.app import Application from smart_tests.utils.authentication import get_org_workspace +from smart_tests.utils.env_keys import CALLER_KEY, detect_ci_provider from smart_tests.utils.http_client import _HttpClient, _join_paths from smart_tests.version import __version__ @@ -79,11 +81,15 @@ def _post_payload( event_name: Union[Tracking.Event, Tracking.ErrorEvent], metadata: Dict[str, Any] ): + caller = os.environ.get(CALLER_KEY) or "cli" + ci_provider = detect_ci_provider() payload = { "command": self.command.value, "eventName": event_name.value, "cliVersion": __version__, "metadata": metadata, + "caller": caller, + "ciProvider": ci_provider, } path = _join_paths( '/intake', From bc0f093586c13996794a796c2812c775355c2528 Mon Sep 17 00:00:00 2001 From: Uday Date: Tue, 28 Apr 2026 14:58:08 +0530 Subject: [PATCH 02/14] test: add tests for caller/CI tracking and fix clear=True usage - Add test_tracking.py covering detect_ci_provider() and caller/ ciProvider fields in tracking payloads - Add User-Agent tests for caller and CI provider segments - Replace clear=True with surgical env cleanup in test_http_client.py to avoid wiping SMART_TESTS_BASE_URL (same issue as PR #1279) Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/utils/test_http_client.py | 92 ++++++++++++++++++-- tests/utils/test_tracking.py | 145 ++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 tests/utils/test_tracking.py diff --git a/tests/utils/test_http_client.py b/tests/utils/test_http_client.py index 6333c5f82..325b094a6 100644 --- a/tests/utils/test_http_client.py +++ b/tests/utils/test_http_client.py @@ -7,23 +7,52 @@ from smart_tests.version import __version__ +def _clean_ci_env(): + """ + Returns env overrides that surgically clear CI-specific variables. + Avoids clear=True which wipes the entire environment including + SMART_TESTS_BASE_URL, causing tests to hit the production URL. + See PR #1279 for context. + """ + return { + 'GITHUB_ACTIONS': '', + 'GITHUB_RUN_ID': '', + 'GITHUB_REPOSITORY': '', + 'GITHUB_WORKFLOW': '', + 'GITHUB_RUN_NUMBER': '', + 'GITHUB_EVENT_NAME': '', + 'GITHUB_SHA': '', + 'GITHUB_JOB': '', + 'JENKINS_URL': '', + 'CIRCLECI': '', + 'CODEBUILD_BUILD_ID': '', + 'SMART_TESTS_CALLER': '', + } + + class HttpClientTest(TestCase): - @mock.patch.dict( - os.environ, - {"SMART_TESTS_ORGANIZATION": "launchableinc", "SMART_TESTS_WORKSPACE": "test"}, - clear=True, - ) + @mock.patch.dict(os.environ, { + **_clean_ci_env(), + "SMART_TESTS_ORGANIZATION": "launchableinc", + "SMART_TESTS_WORKSPACE": "test", + # Clear auth tokens so _headers() produces no Authorization header. + # Without this, tokens from .envrc leak in and the full-dict assertEqual fails. + "SMART_TESTS_TOKEN": "", + "LAUNCHABLE_TOKEN": "", + }) def test_header(self): + base_ua = f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})" + cli = _HttpClient("/test") self.assertEqual(cli._headers(True), { 'Content-Encoding': 'gzip', 'Content-Type': 'application/json', - "User-Agent": f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})", + "User-Agent": f"{base_ua} Caller/cli", }) self.assertEqual(cli._headers(False), { 'Content-Type': 'application/json', - "User-Agent": f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})", + "User-Agent": f"{base_ua} Caller/cli", }) app = Application() @@ -31,10 +60,55 @@ def test_header(self): cli = _HttpClient("/test", app=app) self.assertEqual(cli._headers(False), { 'Content-Type': 'application/json', - "User-Agent": f"Launchable/{__version__} (Python {platform.python_version()}, " - f"{platform.platform()}) TestRunner/dummy", + "User-Agent": f"{base_ua} TestRunner/dummy Caller/cli", }) + @mock.patch.dict(os.environ, { + **_clean_ci_env(), + "SMART_TESTS_ORGANIZATION": "launchableinc", + "SMART_TESTS_WORKSPACE": "test", + "SMART_TESTS_CALLER": "github-action", + "GITHUB_ACTIONS": "true", + "GITHUB_RUN_ID": "123", + "GITHUB_REPOSITORY": "org/repo", + "GITHUB_WORKFLOW": "ci", + "GITHUB_RUN_NUMBER": "1", + "GITHUB_EVENT_NAME": "push", + "GITHUB_SHA": "abc123", + }) + def test_header_with_caller_and_ci(self): + base_ua = f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})" + + cli = _HttpClient("/test") + headers = cli._headers(False) + self.assertEqual( + headers["User-Agent"], + f"{base_ua} Caller/github-action CI/github-actions", + ) + + @mock.patch.dict(os.environ, { + **_clean_ci_env(), + "SMART_TESTS_ORGANIZATION": "launchableinc", + "SMART_TESTS_WORKSPACE": "test", + "GITHUB_ACTIONS": "true", + "GITHUB_RUN_ID": "123", + "GITHUB_REPOSITORY": "org/repo", + "GITHUB_WORKFLOW": "ci", + "GITHUB_RUN_NUMBER": "1", + "GITHUB_EVENT_NAME": "push", + "GITHUB_SHA": "abc123", + }) + def test_header_direct_cli_in_ci(self): + """User calls CLI directly in GitHub Actions (no wrapper).""" + base_ua = f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})" + + cli = _HttpClient("/test") + headers = cli._headers(False) + self.assertEqual( + headers["User-Agent"], + f"{base_ua} Caller/cli CI/github-actions", + ) + def test_sanitize_headers_with_bearer_token(self): headers = { 'Authorization': 'Bearer v1:konboi/arm-testing:cfcxxxxxxxxxxxxxx', diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py new file mode 100644 index 000000000..67dba5de9 --- /dev/null +++ b/tests/utils/test_tracking.py @@ -0,0 +1,145 @@ +import json +import os +from unittest import TestCase, mock + +import responses + +from smart_tests.utils.commands import Command +from smart_tests.utils.http_client import get_base_url +from smart_tests.utils.env_keys import detect_ci_provider +from smart_tests.utils.tracking import Tracking, TrackingClient + + +class DetectCiProviderTest(TestCase): + + def test_no_ci(self): + with mock.patch.dict(os.environ, {}, clear=True): + self.assertEqual(detect_ci_provider(), "") + + def test_github_actions(self): + with mock.patch.dict(os.environ, {"GITHUB_ACTIONS": "true"}, clear=True): + self.assertEqual(detect_ci_provider(), "github-actions") + + def test_jenkins(self): + with mock.patch.dict(os.environ, {"JENKINS_URL": "https://jenkins.example.com"}, clear=True): + self.assertEqual(detect_ci_provider(), "jenkins") + + def test_circleci(self): + with mock.patch.dict(os.environ, {"CIRCLECI": "true"}, clear=True): + self.assertEqual(detect_ci_provider(), "circleci") + + def test_codebuild(self): + with mock.patch.dict(os.environ, {"CODEBUILD_BUILD_ID": "build:123"}, clear=True): + self.assertEqual(detect_ci_provider(), "codebuild") + + +class TrackingCallerTest(TestCase): + + @mock.patch.dict( + os.environ, + {"SMART_TESTS_TOKEN": "v1:org/ws:token"}, + ) + @responses.activate + def test_default_caller_is_cli(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + client = TrackingClient(Command.VERIFY, base_url=get_base_url()) + client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) + + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["caller"], "cli") + + @mock.patch.dict( + os.environ, + { + "SMART_TESTS_TOKEN": "v1:org/ws:token", + "SMART_TESTS_CALLER": "github-action", + }, + ) + @responses.activate + def test_caller_from_env(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + client = TrackingClient(Command.RECORD_BUILD, base_url=get_base_url()) + client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) + + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["caller"], "github-action") + + @mock.patch.dict( + os.environ, + { + "SMART_TESTS_TOKEN": "v1:org/ws:token", + "GITHUB_ACTIONS": "true", + }, + ) + @responses.activate + def test_ci_provider_auto_detected(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + client = TrackingClient(Command.SUBSET, base_url=get_base_url()) + client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) + + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["ciProvider"], "github-actions") + # caller should still default to "cli" when wrapper hasn't set it + self.assertEqual(payload["caller"], "cli") + + @mock.patch.dict( + os.environ, + { + "SMART_TESTS_TOKEN": "v1:org/ws:token", + "GITHUB_ACTIONS": "true", + "SMART_TESTS_CALLER": "github-action", + }, + ) + @responses.activate + def test_caller_and_ci_provider_together(self): + """The key scenario: our GitHub Action wrapper sets SMART_TESTS_CALLER + while GITHUB_ACTIONS is auto-set by the runner.""" + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + client = TrackingClient(Command.RECORD_BUILD, base_url=get_base_url()) + client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) + + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["caller"], "github-action") + self.assertEqual(payload["ciProvider"], "github-actions") + + @mock.patch.dict( + os.environ, + {"SMART_TESTS_TOKEN": "v1:org/ws:token"}, + ) + @responses.activate + def test_error_event_includes_caller(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + client = TrackingClient(Command.GATE, base_url=get_base_url()) + client.send_error_event( + event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, + stack_trace="some error", + ) + + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["caller"], "cli") + self.assertIn("ciProvider", payload) From ea01dcbe839f3ba728353f88b28df0909364b5f3 Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 4 May 2026 14:45:10 +0530 Subject: [PATCH 03/14] feat: add command tracking with rawCommand, exitCode, caller, ciProvider - Add send_command_tracking() to send a COMMAND_INVOCATION event with the full CLI argv, exit code, caller, and CI provider in metadata - Add _detect_command() to map argv to Command enum values; falls back to UNKNOWN for typos or unrecognized commands - Refactor TrackingClient: extract construct_payload() from _post_payload() so both send_command_tracking and existing methods share payload construction - Move caller/ciProvider into metadata JSONB (not top-level fields) - Add UNKNOWN and COMMAND_INVOCATION to respective enums Co-Authored-By: Claude Opus 4.6 (1M context) --- smart_tests/utils/commands.py | 1 + smart_tests/utils/tracking.py | 103 +++++++++++++++++++++++++--------- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/smart_tests/utils/commands.py b/smart_tests/utils/commands.py index 6424d336c..fc05b0c29 100644 --- a/smart_tests/utils/commands.py +++ b/smart_tests/utils/commands.py @@ -12,6 +12,7 @@ class Command(Enum): GATE = 'GATE' UPDATE_ALIAS = 'UPDATE_ALIAS' RECORD_DEPLOYMENT = 'RECORD_DEPLOYMENT' + UNKNOWN = 'UNKNOWN' # when you add a new constant here, the server also needs to get a new constant in cli_tracking.proto diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index ba52478af..a0cf6b6e0 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -12,12 +12,54 @@ from .commands import Command +# Map CLI subcommand tokens to Command enum values. +# Longer matches are tried first so "record build" matches before "record". +_COMMAND_MAP = { + ("verify",): Command.VERIFY, + ("record", "build"): Command.RECORD_BUILD, + ("record", "session"): Command.RECORD_SESSION, + ("record", "tests"): Command.RECORD_TESTS, + ("record", "commit"): Command.COMMIT, + ("record", "deployment"): Command.RECORD_DEPLOYMENT, + ("subset",): Command.SUBSET, + ("detect-flakes",): Command.DETECT_FLAKE, + ("gate",): Command.GATE, + ("update", "alias"): Command.UPDATE_ALIAS, +} + + +def _detect_command(argv: list[str]) -> Command: + """Best-effort detection of the Command from argv. Returns UNKNOWN for typos.""" + args = argv[1:] + for tokens, command in sorted(_COMMAND_MAP.items(), key=lambda x: -len(x[0])): + for i in range(len(args) - len(tokens) + 1): + if tuple(args[i:i + len(tokens)]) == tokens: + return command + return Command.UNKNOWN + + +def send_command_tracking(argv: list[str], exit_code: int): + """Send a single COMMAND_INVOCATION event with the full command string. Fire-and-forget.""" + client = TrackingClient(_detect_command(argv)) + metadata = { + "exitCode": exit_code, + } + + payload = client.construct_payload( + event_name=Tracking.Event.COMMAND_INVOCATION, + metadata=metadata, + raw_command=" ".join(argv), + ) + + client.post_payload(payload=payload) + class Tracking: # General events class Event(Enum): SHALLOW_CLONE = 'SHALLOW_CLONE' # this event is an example PERFORMANCE = 'PERFORMANCE' + COMMAND_INVOCATION = 'COMMAND_INVOCATION' # Error events class ErrorEvent(Enum): @@ -47,14 +89,8 @@ def send_event( event_name: Tracking.Event, metadata: Dict[str, Any] | None = None ): - org, workspace = get_org_workspace() - if metadata is None: - metadata = {} - metadata["organization"] = org or "" - metadata["workspace"] = workspace or "" - self._post_payload( - event_name=event_name, - metadata=metadata, + self.post_payload( + payload=self.construct_payload(event_name=event_name, metadata=metadata), ) def send_error_event( @@ -64,33 +100,18 @@ def send_error_event( api: str = "", metadata: Dict[str, Any] | None = None ): - org, workspace = get_org_workspace() if metadata is None: metadata = {} metadata["stackTrace"] = stack_trace - metadata["organization"] = org or "" - metadata["workspace"] = workspace or "" metadata["api"] = api - self._post_payload( - event_name=event_name, - metadata=metadata, - ) - def _post_payload( + payload = self.construct_payload(event_name=event_name, metadata=metadata) + self.post_payload(payload=payload) + + def post_payload( self, - event_name: Union[Tracking.Event, Tracking.ErrorEvent], - metadata: Dict[str, Any] + payload: dict, ): - caller = os.environ.get(CALLER_KEY) or "cli" - ci_provider = detect_ci_provider() - payload = { - "command": self.command.value, - "eventName": event_name.value, - "cliVersion": __version__, - "metadata": metadata, - "caller": caller, - "ciProvider": ci_provider, - } path = _join_paths( '/intake', 'cli_tracking' @@ -99,3 +120,29 @@ def _post_payload( self.http_client.request('post', payload=payload, path=path) except Exception: pass + + def construct_payload( + self, + event_name: Union[Tracking.Event, Tracking.ErrorEvent], + metadata: Dict[str, Any] | None = None, + raw_command: str | None = None + ) -> dict: + org, workspace = get_org_workspace() + + if metadata is None: + metadata = {} + + metadata["organization"] = org or "" + metadata["workspace"] = workspace or "" + metadata["caller"] = os.environ.get(CALLER_KEY) or "cli" + metadata["ciProvider"] = detect_ci_provider() + + payload = { + "command": self.command.value, + "eventName": event_name.value, + "cliVersion": __version__, + "metadata": metadata, + "rawCommand": raw_command or "", + } + + return payload From 30af022c2ed5f0437aa65ff6ffcfb2ba4934ba3d Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 4 May 2026 14:45:18 +0530 Subject: [PATCH 04/14] feat: track every CLI invocation from __main__.py Capture sys.argv and exit code, send COMMAND_INVOCATION tracking event after every CLI run (including typos and failures). Co-Authored-By: Claude Opus 4.6 (1M context) --- smart_tests/__main__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/smart_tests/__main__.py b/smart_tests/__main__.py index 08ce06bad..7e48df83c 100644 --- a/smart_tests/__main__.py +++ b/smart_tests/__main__.py @@ -16,6 +16,7 @@ from smart_tests.commands.subset import subset from smart_tests.commands.update import update from smart_tests.commands.verify import verify +from smart_tests.utils.tracking import send_command_tracking cli = Group(name="cli", callback=Application) cli.add_command(record) @@ -59,7 +60,18 @@ def _load_test_runners(): def main(): - cli.main() + argv = sys.argv[:] + exit_code = 0 + try: + cli.main() + except SystemExit as e: + exit_code = e.code if isinstance(e.code, int) else 1 + finally: + try: + send_command_tracking(argv=argv, exit_code=exit_code) + except Exception: + pass + sys.exit(exit_code) if __name__ == '__main__': From c120b15c780f79e0b1f7fa56ea339e16cea34f2d Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 4 May 2026 14:45:26 +0530 Subject: [PATCH 05/14] refactor: remove Caller/CI from User-Agent, update tests Caller and CI provider are now sent via cli_tracking metadata only. Remove Caller/ and CI/ segments from User-Agent header. Update test_tracking with tests for _detect_command, send_command_tracking, and caller/ciProvider in metadata. Revert test_http_client to original User-Agent assertions. Co-Authored-By: Claude Opus 4.6 (1M context) --- smart_tests/utils/http_client.py | 9 +- tests/utils/test_http_client.py | 97 +++----------------- tests/utils/test_tracking.py | 146 ++++++++++++++++++++++++++++--- 3 files changed, 148 insertions(+), 104 deletions(-) diff --git a/smart_tests/utils/http_client.py b/smart_tests/utils/http_client.py index bf8214b5a..ee6a9523d 100644 --- a/smart_tests/utils/http_client.py +++ b/smart_tests/utils/http_client.py @@ -12,7 +12,7 @@ from ..app import Application from .authentication import authentication_headers -from .env_keys import BASE_URL_KEY, CALLER_KEY, SKIP_TIMEOUT_RETRY, detect_ci_provider +from .env_keys import BASE_URL_KEY, SKIP_TIMEOUT_RETRY from .gzipgen import compress as gzipgen_compress from .logger import Logger @@ -134,13 +134,6 @@ def _headers(self, compress): if self.test_runner: h["User-Agent"] = h["User-Agent"] + f" TestRunner/{self.test_runner}" - caller = os.environ.get(CALLER_KEY) or "cli" - h["User-Agent"] += f" Caller/{caller}" - - ci_provider = detect_ci_provider() - if ci_provider: - h["User-Agent"] += f" CI/{ci_provider}" - return {**h, **authentication_headers()} diff --git a/tests/utils/test_http_client.py b/tests/utils/test_http_client.py index 325b094a6..e4040320e 100644 --- a/tests/utils/test_http_client.py +++ b/tests/utils/test_http_client.py @@ -7,52 +7,28 @@ from smart_tests.version import __version__ -def _clean_ci_env(): - """ - Returns env overrides that surgically clear CI-specific variables. - Avoids clear=True which wipes the entire environment including - SMART_TESTS_BASE_URL, causing tests to hit the production URL. - See PR #1279 for context. - """ - return { - 'GITHUB_ACTIONS': '', - 'GITHUB_RUN_ID': '', - 'GITHUB_REPOSITORY': '', - 'GITHUB_WORKFLOW': '', - 'GITHUB_RUN_NUMBER': '', - 'GITHUB_EVENT_NAME': '', - 'GITHUB_SHA': '', - 'GITHUB_JOB': '', - 'JENKINS_URL': '', - 'CIRCLECI': '', - 'CODEBUILD_BUILD_ID': '', - 'SMART_TESTS_CALLER': '', - } - - class HttpClientTest(TestCase): - @mock.patch.dict(os.environ, { - **_clean_ci_env(), - "SMART_TESTS_ORGANIZATION": "launchableinc", - "SMART_TESTS_WORKSPACE": "test", - # Clear auth tokens so _headers() produces no Authorization header. - # Without this, tokens from .envrc leak in and the full-dict assertEqual fails. - "SMART_TESTS_TOKEN": "", - "LAUNCHABLE_TOKEN": "", - }) + @mock.patch.dict( + os.environ, + { + "SMART_TESTS_ORGANIZATION": "launchableinc", + "SMART_TESTS_WORKSPACE": "test", + "SMART_TESTS_TOKEN": "", + "LAUNCHABLE_TOKEN": "", + }, + clear=True, + ) def test_header(self): - base_ua = f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})" - cli = _HttpClient("/test") self.assertEqual(cli._headers(True), { 'Content-Encoding': 'gzip', 'Content-Type': 'application/json', - "User-Agent": f"{base_ua} Caller/cli", + "User-Agent": f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})", }) self.assertEqual(cli._headers(False), { 'Content-Type': 'application/json', - "User-Agent": f"{base_ua} Caller/cli", + "User-Agent": f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})", }) app = Application() @@ -60,55 +36,10 @@ def test_header(self): cli = _HttpClient("/test", app=app) self.assertEqual(cli._headers(False), { 'Content-Type': 'application/json', - "User-Agent": f"{base_ua} TestRunner/dummy Caller/cli", + "User-Agent": f"Launchable/{__version__} (Python {platform.python_version()}, " + f"{platform.platform()}) TestRunner/dummy", }) - @mock.patch.dict(os.environ, { - **_clean_ci_env(), - "SMART_TESTS_ORGANIZATION": "launchableinc", - "SMART_TESTS_WORKSPACE": "test", - "SMART_TESTS_CALLER": "github-action", - "GITHUB_ACTIONS": "true", - "GITHUB_RUN_ID": "123", - "GITHUB_REPOSITORY": "org/repo", - "GITHUB_WORKFLOW": "ci", - "GITHUB_RUN_NUMBER": "1", - "GITHUB_EVENT_NAME": "push", - "GITHUB_SHA": "abc123", - }) - def test_header_with_caller_and_ci(self): - base_ua = f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})" - - cli = _HttpClient("/test") - headers = cli._headers(False) - self.assertEqual( - headers["User-Agent"], - f"{base_ua} Caller/github-action CI/github-actions", - ) - - @mock.patch.dict(os.environ, { - **_clean_ci_env(), - "SMART_TESTS_ORGANIZATION": "launchableinc", - "SMART_TESTS_WORKSPACE": "test", - "GITHUB_ACTIONS": "true", - "GITHUB_RUN_ID": "123", - "GITHUB_REPOSITORY": "org/repo", - "GITHUB_WORKFLOW": "ci", - "GITHUB_RUN_NUMBER": "1", - "GITHUB_EVENT_NAME": "push", - "GITHUB_SHA": "abc123", - }) - def test_header_direct_cli_in_ci(self): - """User calls CLI directly in GitHub Actions (no wrapper).""" - base_ua = f"Launchable/{__version__} (Python {platform.python_version()}, {platform.platform()})" - - cli = _HttpClient("/test") - headers = cli._headers(False) - self.assertEqual( - headers["User-Agent"], - f"{base_ua} Caller/cli CI/github-actions", - ) - def test_sanitize_headers_with_bearer_token(self): headers = { 'Authorization': 'Bearer v1:konboi/arm-testing:cfcxxxxxxxxxxxxxx', diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index 67dba5de9..5c2be3652 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -5,9 +5,45 @@ import responses from smart_tests.utils.commands import Command -from smart_tests.utils.http_client import get_base_url from smart_tests.utils.env_keys import detect_ci_provider -from smart_tests.utils.tracking import Tracking, TrackingClient +from smart_tests.utils.http_client import get_base_url +from smart_tests.utils.tracking import Tracking, TrackingClient, send_command_tracking, _detect_command + + +class DetectCommandTest(TestCase): + + def test_verify(self): + self.assertEqual(_detect_command(["smart-tests", "verify"]), Command.VERIFY) + + def test_record_build(self): + self.assertEqual(_detect_command(["smart-tests", "record", "build", "--name", "foo"]), Command.RECORD_BUILD) + + def test_record_session(self): + self.assertEqual(_detect_command(["smart-tests", "record", "session", "--build", "123"]), Command.RECORD_SESSION) + + def test_subset(self): + self.assertEqual(_detect_command(["smart-tests", "subset", "pytest", "--target", "30%"]), Command.SUBSET) + + def test_detect_flakes(self): + self.assertEqual(_detect_command(["smart-tests", "detect-flakes", "pytest"]), Command.DETECT_FLAKE) + + def test_gate(self): + self.assertEqual(_detect_command(["smart-tests", "gate", "--session", "builds/1/test_sessions/2"]), Command.GATE) + + def test_update_alias(self): + self.assertEqual(_detect_command(["smart-tests", "update", "alias", "--build", "foo"]), Command.UPDATE_ALIAS) + + def test_typo_returns_unknown(self): + self.assertEqual(_detect_command(["smart-tests", "recrd", "build"]), Command.UNKNOWN) + + def test_no_subcommand_returns_unknown(self): + self.assertEqual(_detect_command(["smart-tests"]), Command.UNKNOWN) + + def test_global_options_before_command(self): + self.assertEqual( + _detect_command(["smart-tests", "--dry-run", "record", "build", "--name", "foo"]), + Command.RECORD_BUILD, + ) class DetectCiProviderTest(TestCase): @@ -51,7 +87,7 @@ def test_default_caller_is_cli(self): client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) payload = json.loads(responses.calls[0].request.body) - self.assertEqual(payload["caller"], "cli") + self.assertEqual(payload["metadata"]["caller"], "cli") @mock.patch.dict( os.environ, @@ -72,7 +108,7 @@ def test_caller_from_env(self): client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) payload = json.loads(responses.calls[0].request.body) - self.assertEqual(payload["caller"], "github-action") + self.assertEqual(payload["metadata"]["caller"], "github-action") @mock.patch.dict( os.environ, @@ -93,9 +129,8 @@ def test_ci_provider_auto_detected(self): client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) payload = json.loads(responses.calls[0].request.body) - self.assertEqual(payload["ciProvider"], "github-actions") - # caller should still default to "cli" when wrapper hasn't set it - self.assertEqual(payload["caller"], "cli") + self.assertEqual(payload["metadata"]["ciProvider"], "github-actions") + self.assertEqual(payload["metadata"]["caller"], "cli") @mock.patch.dict( os.environ, @@ -107,8 +142,6 @@ def test_ci_provider_auto_detected(self): ) @responses.activate def test_caller_and_ci_provider_together(self): - """The key scenario: our GitHub Action wrapper sets SMART_TESTS_CALLER - while GITHUB_ACTIONS is auto-set by the runner.""" responses.add( responses.POST, f"{get_base_url()}/intake/cli_tracking", @@ -119,8 +152,8 @@ def test_caller_and_ci_provider_together(self): client.send_event(Tracking.Event.PERFORMANCE, {"duration": 100}) payload = json.loads(responses.calls[0].request.body) - self.assertEqual(payload["caller"], "github-action") - self.assertEqual(payload["ciProvider"], "github-actions") + self.assertEqual(payload["metadata"]["caller"], "github-action") + self.assertEqual(payload["metadata"]["ciProvider"], "github-actions") @mock.patch.dict( os.environ, @@ -141,5 +174,92 @@ def test_error_event_includes_caller(self): ) payload = json.loads(responses.calls[0].request.body) - self.assertEqual(payload["caller"], "cli") - self.assertIn("ciProvider", payload) + self.assertEqual(payload["metadata"]["caller"], "cli") + self.assertIn("ciProvider", payload["metadata"]) + + +class SendCommandTrackingTest(TestCase): + + @mock.patch.dict( + os.environ, + {"SMART_TESTS_TOKEN": "v1:org/ws:token"}, + ) + @responses.activate + def test_sends_command_invocation(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + send_command_tracking( + argv=["smart-tests", "record", "build", "--name", "foo"], + exit_code=0, + ) + + self.assertEqual(len(responses.calls), 1) + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["command"], "RECORD_BUILD") + self.assertEqual(payload["eventName"], "COMMAND_INVOCATION") + self.assertEqual(payload["rawCommand"], "smart-tests record build --name foo") + self.assertIn("cliVersion", payload) + metadata = payload["metadata"] + self.assertEqual(metadata["exitCode"], 0) + self.assertEqual(metadata["caller"], "cli") + + @mock.patch.dict( + os.environ, + { + "SMART_TESTS_TOKEN": "v1:org/ws:token", + "SMART_TESTS_CALLER": "github-action", + "GITHUB_ACTIONS": "true", + }, + ) + @responses.activate + def test_includes_caller_and_ci_in_metadata(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + send_command_tracking(argv=["smart-tests", "verify"], exit_code=0) + + payload = json.loads(responses.calls[0].request.body) + metadata = payload["metadata"] + self.assertEqual(metadata["caller"], "github-action") + self.assertEqual(metadata["ciProvider"], "github-actions") + + @mock.patch.dict( + os.environ, + {"SMART_TESTS_TOKEN": "v1:org/ws:token"}, + ) + @responses.activate + def test_swallows_exceptions(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={"error": "server error"}, + status=500, + ) + # Should not raise + send_command_tracking(argv=["smart-tests", "verify"], exit_code=0) + + @mock.patch.dict( + os.environ, + {"SMART_TESTS_TOKEN": "v1:org/ws:token"}, + ) + @responses.activate + def test_typo_maps_to_unknown(self): + responses.add( + responses.POST, + f"{get_base_url()}/intake/cli_tracking", + json={}, + status=200, + ) + send_command_tracking(argv=["smart-tests", "recrd", "build"], exit_code=1) + + payload = json.loads(responses.calls[0].request.body) + self.assertEqual(payload["command"], "UNKNOWN") + self.assertEqual(payload["metadata"]["exitCode"], 1) + self.assertEqual(payload["rawCommand"], "smart-tests recrd build") From f8a093769ef3850471ce49eb8a022906c7f77a62 Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 4 May 2026 15:10:56 +0530 Subject: [PATCH 06/14] refactor: rawCommand can be None instead of an empty string --- smart_tests/utils/tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index a0cf6b6e0..ee82025f6 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -142,7 +142,7 @@ def construct_payload( "eventName": event_name.value, "cliVersion": __version__, "metadata": metadata, - "rawCommand": raw_command or "", + "rawCommand": raw_command, } return payload From a24628edc7b59ccaf8cbfa8f8d9c09f9b1ade9d5 Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 4 May 2026 16:19:14 +0530 Subject: [PATCH 07/14] fix: send exitCode as string in tracking metadata Co-Authored-By: Claude Opus 4.6 (1M context) --- smart_tests/utils/tracking.py | 2 +- tests/utils/test_tracking.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index ee82025f6..f03c326a1 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -42,7 +42,7 @@ def send_command_tracking(argv: list[str], exit_code: int): """Send a single COMMAND_INVOCATION event with the full command string. Fire-and-forget.""" client = TrackingClient(_detect_command(argv)) metadata = { - "exitCode": exit_code, + "exitCode": str(exit_code), } payload = client.construct_payload( diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index 5c2be3652..800348aa6 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -204,7 +204,7 @@ def test_sends_command_invocation(self): self.assertEqual(payload["rawCommand"], "smart-tests record build --name foo") self.assertIn("cliVersion", payload) metadata = payload["metadata"] - self.assertEqual(metadata["exitCode"], 0) + self.assertEqual(metadata["exitCode"], "0") self.assertEqual(metadata["caller"], "cli") @mock.patch.dict( @@ -261,5 +261,5 @@ def test_typo_maps_to_unknown(self): payload = json.loads(responses.calls[0].request.body) self.assertEqual(payload["command"], "UNKNOWN") - self.assertEqual(payload["metadata"]["exitCode"], 1) + self.assertEqual(payload["metadata"]["exitCode"], "1") self.assertEqual(payload["rawCommand"], "smart-tests recrd build") From 89301329cef02c07d3b3e709200a6358ff47f08a Mon Sep 17 00:00:00 2001 From: Uday Date: Wed, 6 May 2026 11:44:33 +0530 Subject: [PATCH 08/14] Truncate the raw command length to 2000 characters --- smart_tests/utils/tracking.py | 4 +++- tests/utils/test_tracking.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index f03c326a1..2537118c4 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -45,10 +45,12 @@ def send_command_tracking(argv: list[str], exit_code: int): "exitCode": str(exit_code), } + raw_command = " ".join(argv)[:2000] + payload = client.construct_payload( event_name=Tracking.Event.COMMAND_INVOCATION, metadata=metadata, - raw_command=" ".join(argv), + raw_command=raw_command, ) client.post_payload(payload=payload) diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index 800348aa6..8342165da 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -7,7 +7,7 @@ from smart_tests.utils.commands import Command from smart_tests.utils.env_keys import detect_ci_provider from smart_tests.utils.http_client import get_base_url -from smart_tests.utils.tracking import Tracking, TrackingClient, send_command_tracking, _detect_command +from smart_tests.utils.tracking import Tracking, TrackingClient, _detect_command, send_command_tracking class DetectCommandTest(TestCase): From 8001fce23f546d643400e918e83b5ad868f73f96 Mon Sep 17 00:00:00 2001 From: Uday Date: Thu, 7 May 2026 12:29:01 +0530 Subject: [PATCH 09/14] feat: add missing commands to tracking map and enum Add record attachment, inspect model, inspect subset, stats test_sessions, compare subsets, and get docs to both the Command enum and _COMMAND_MAP. Co-Authored-By: Claude Opus 4.6 (1M context) --- smart_tests/utils/commands.py | 8 +++++++- smart_tests/utils/tracking.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/smart_tests/utils/commands.py b/smart_tests/utils/commands.py index fc05b0c29..dbfd24d6e 100644 --- a/smart_tests/utils/commands.py +++ b/smart_tests/utils/commands.py @@ -6,12 +6,18 @@ class Command(Enum): RECORD_TESTS = 'RECORD_TESTS' RECORD_BUILD = 'RECORD_BUILD' RECORD_SESSION = 'RECORD_SESSION' + RECORD_ATTACHMENT = 'RECORD_ATTACHMENT' + RECORD_DEPLOYMENT = 'RECORD_DEPLOYMENT' SUBSET = 'SUBSET' COMMIT = 'COMMIT' DETECT_FLAKE = 'DETECT_FLAKE' GATE = 'GATE' UPDATE_ALIAS = 'UPDATE_ALIAS' - RECORD_DEPLOYMENT = 'RECORD_DEPLOYMENT' + INSPECT_MODEL = 'INSPECT_MODEL' + INSPECT_SUBSET = 'INSPECT_SUBSET' + STATS_TEST_SESSIONS = 'STATS_TEST_SESSIONS' + COMPARE_SUBSETS = 'COMPARE_SUBSETS' + GET_DOCS = 'GET_DOCS' UNKNOWN = 'UNKNOWN' # when you add a new constant here, the server also needs to get a new constant in cli_tracking.proto diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index 2537118c4..6314b11ef 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -20,11 +20,17 @@ ("record", "session"): Command.RECORD_SESSION, ("record", "tests"): Command.RECORD_TESTS, ("record", "commit"): Command.COMMIT, + ("record", "attachment"): Command.RECORD_ATTACHMENT, ("record", "deployment"): Command.RECORD_DEPLOYMENT, ("subset",): Command.SUBSET, ("detect-flakes",): Command.DETECT_FLAKE, ("gate",): Command.GATE, ("update", "alias"): Command.UPDATE_ALIAS, + ("inspect", "model"): Command.INSPECT_MODEL, + ("inspect", "subset"): Command.INSPECT_SUBSET, + ("stats", "test_sessions"): Command.STATS_TEST_SESSIONS, + ("compare", "subsets"): Command.COMPARE_SUBSETS, + ("get", "docs"): Command.GET_DOCS, } From 1c6855c09498502bab7d7ff387fe37302564d67e Mon Sep 17 00:00:00 2001 From: Uday Date: Thu, 7 May 2026 12:33:42 +0530 Subject: [PATCH 10/14] test: add guard test ensuring _COMMAND_MAP covers all Command enum values Also adds individual detection tests for the 7 newly mapped commands. If a new Command is added to the enum without updating _COMMAND_MAP, test_command_map_covers_all_enum_values will fail. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/utils/test_tracking.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index 8342165da..b221d433f 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -7,7 +7,7 @@ from smart_tests.utils.commands import Command from smart_tests.utils.env_keys import detect_ci_provider from smart_tests.utils.http_client import get_base_url -from smart_tests.utils.tracking import Tracking, TrackingClient, _detect_command, send_command_tracking +from smart_tests.utils.tracking import Tracking, TrackingClient, _COMMAND_MAP, _detect_command, send_command_tracking class DetectCommandTest(TestCase): @@ -45,6 +45,33 @@ def test_global_options_before_command(self): Command.RECORD_BUILD, ) + def test_record_attachment(self): + self.assertEqual(_detect_command(["smart-tests", "record", "attachment", "--session", "s1"]), Command.RECORD_ATTACHMENT) + + def test_record_deployment(self): + self.assertEqual(_detect_command(["smart-tests", "record", "deployment", "--build", "b1"]), Command.RECORD_DEPLOYMENT) + + def test_inspect_model(self): + self.assertEqual(_detect_command(["smart-tests", "inspect", "model"]), Command.INSPECT_MODEL) + + def test_inspect_subset(self): + self.assertEqual(_detect_command(["smart-tests", "inspect", "subset", "--subset-id", "123"]), Command.INSPECT_SUBSET) + + def test_stats_test_sessions(self): + self.assertEqual(_detect_command(["smart-tests", "stats", "test_sessions", "--days", "7"]), Command.STATS_TEST_SESSIONS) + + def test_compare_subsets(self): + self.assertEqual(_detect_command(["smart-tests", "compare", "subsets"]), Command.COMPARE_SUBSETS) + + def test_get_docs(self): + self.assertEqual(_detect_command(["smart-tests", "get", "docs"]), Command.GET_DOCS) + + def test_command_map_covers_all_enum_values(self): + mapped_commands = set(_COMMAND_MAP.values()) + all_commands = {c for c in Command if c != Command.UNKNOWN} + self.assertEqual(mapped_commands, all_commands, + f"Commands missing from _COMMAND_MAP: {all_commands - mapped_commands}") + class DetectCiProviderTest(TestCase): From 228ca3522977827642b98ed2f3642f9e93e48ccc Mon Sep 17 00:00:00 2001 From: Uday Date: Thu, 7 May 2026 12:41:24 +0530 Subject: [PATCH 11/14] Pre-commit fix --- tests/utils/test_tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index b221d433f..53e94e0f5 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -7,7 +7,7 @@ from smart_tests.utils.commands import Command from smart_tests.utils.env_keys import detect_ci_provider from smart_tests.utils.http_client import get_base_url -from smart_tests.utils.tracking import Tracking, TrackingClient, _COMMAND_MAP, _detect_command, send_command_tracking +from smart_tests.utils.tracking import _COMMAND_MAP, Tracking, TrackingClient, _detect_command, send_command_tracking class DetectCommandTest(TestCase): From 3525a3528f1b8b6812f47303f96fe838470a7248 Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 11 May 2026 16:30:38 +0530 Subject: [PATCH 12/14] Fix wrongful identification of command, where the sliding window was considering option values as commands --- smart_tests/utils/tracking.py | 10 ++++++---- tests/utils/test_tracking.py | 7 +++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index 6314b11ef..d08d05e36 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -36,11 +36,13 @@ def _detect_command(argv: list[str]) -> Command: """Best-effort detection of the Command from argv. Returns UNKNOWN for typos.""" - args = argv[1:] + # Commands are always positional tokens (not starting with '-'). + # We match only the first positional tokens against known command patterns. + positional = [a for a in argv[1:] if not a.startswith("-")] + for tokens, command in sorted(_COMMAND_MAP.items(), key=lambda x: -len(x[0])): - for i in range(len(args) - len(tokens) + 1): - if tuple(args[i:i + len(tokens)]) == tokens: - return command + if tuple(positional[:len(tokens)]) == tokens: + return command return Command.UNKNOWN diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index 53e94e0f5..463ae5ce1 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -33,6 +33,13 @@ def test_gate(self): def test_update_alias(self): self.assertEqual(_detect_command(["smart-tests", "update", "alias", "--build", "foo"]), Command.UPDATE_ALIAS) + def test_command_token_in_flag_value_not_misdetected(self): + """Regression: 'smart-tests record --name verify' should not detect as VERIFY.""" + self.assertEqual( + _detect_command(["smart-tests", "record", "build", "--name", "verify"]), + Command.RECORD_BUILD, + ) + def test_typo_returns_unknown(self): self.assertEqual(_detect_command(["smart-tests", "recrd", "build"]), Command.UNKNOWN) From 34ebbf6ebb82d5a4fb6020bf522ad805cf0a5305 Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 11 May 2026 16:54:20 +0530 Subject: [PATCH 13/14] Ignore commands used with dry-run flag since that adds complexity to command identification --- smart_tests/utils/tracking.py | 7 +++---- tests/utils/test_tracking.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index d08d05e36..e1712af25 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -1,5 +1,6 @@ import os from enum import Enum +from itertools import takewhile from typing import Any, Dict, Union from requests import Session @@ -36,12 +37,10 @@ def _detect_command(argv: list[str]) -> Command: """Best-effort detection of the Command from argv. Returns UNKNOWN for typos.""" - # Commands are always positional tokens (not starting with '-'). - # We match only the first positional tokens against known command patterns. - positional = [a for a in argv[1:] if not a.startswith("-")] + command_tokens = list(takewhile(lambda a: not a.startswith("-"), argv[1:])) for tokens, command in sorted(_COMMAND_MAP.items(), key=lambda x: -len(x[0])): - if tuple(positional[:len(tokens)]) == tokens: + if tuple(command_tokens[:len(tokens)]) == tokens: return command return Command.UNKNOWN diff --git a/tests/utils/test_tracking.py b/tests/utils/test_tracking.py index 463ae5ce1..6538011d3 100644 --- a/tests/utils/test_tracking.py +++ b/tests/utils/test_tracking.py @@ -34,11 +34,24 @@ def test_update_alias(self): self.assertEqual(_detect_command(["smart-tests", "update", "alias", "--build", "foo"]), Command.UPDATE_ALIAS) def test_command_token_in_flag_value_not_misdetected(self): - """Regression: 'smart-tests record --name verify' should not detect as VERIFY.""" + """Flag values that match command names must not affect detection.""" + # "verify" is a flag value, not a command self.assertEqual( _detect_command(["smart-tests", "record", "build", "--name", "verify"]), Command.RECORD_BUILD, ) + # "build" is a flag value, not a subcommand of record + self.assertEqual( + _detect_command(["smart-tests", "record", "tests", "--name", "build"]), + Command.RECORD_TESTS, + ) + + def test_flag_value_matching_subcommand_not_misdetected(self): + """'record --name build' must not be detected as RECORD_BUILD.""" + self.assertEqual( + _detect_command(["smart-tests", "record", "--name", "build"]), + Command.UNKNOWN, + ) def test_typo_returns_unknown(self): self.assertEqual(_detect_command(["smart-tests", "recrd", "build"]), Command.UNKNOWN) @@ -49,7 +62,7 @@ def test_no_subcommand_returns_unknown(self): def test_global_options_before_command(self): self.assertEqual( _detect_command(["smart-tests", "--dry-run", "record", "build", "--name", "foo"]), - Command.RECORD_BUILD, + Command.UNKNOWN, ) def test_record_attachment(self): From b118b6f053f7a9ff89ef7fe5f8a4d01ded51d0bc Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 11 May 2026 19:13:09 +0530 Subject: [PATCH 14/14] Add timeout for cli tracking API call --- smart_tests/utils/tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_tests/utils/tracking.py b/smart_tests/utils/tracking.py index e1712af25..68429bfa1 100644 --- a/smart_tests/utils/tracking.py +++ b/smart_tests/utils/tracking.py @@ -126,7 +126,7 @@ def post_payload( 'cli_tracking' ) try: - self.http_client.request('post', payload=payload, path=path) + self.http_client.request('post', payload=payload, path=path, timeout=(2, 2)) except Exception: pass