diff --git a/launchable/commands/inspect/subset.py b/launchable/commands/inspect/subset.py index fa5a07de8..2f2a6aae4 100644 --- a/launchable/commands/inspect/subset.py +++ b/launchable/commands/inspect/subset.py @@ -2,7 +2,7 @@ import sys from abc import ABCMeta, abstractmethod from http import HTTPStatus -from typing import List +from typing import Any, Dict, List import click from tabulate import tabulate @@ -13,6 +13,8 @@ class SubsetResult (object): def __init__(self, result: dict, is_subset: bool): self._estimated_duration_sec = result.get("duration", 0.0) / 1000 # convert to sec from msec + self._density = result.get("density", 0.0) + self._is_new = result.get("numNewTests", 0) > 0 self._test_path = "#".join([path["type"] + "=" + path["name"] for path in result["testPath"] if path.keys() >= {"type", "name"}]) self._is_subset = is_subset @@ -45,7 +47,7 @@ def __init__(self, results: SubsetResults): self._results = results @abstractmethod - def display(self): + def display(self, new_tests_only: bool = False): raise NotImplementedError("display method is not implemented") @@ -53,39 +55,52 @@ class SubsetResultTableDisplay(SubsetResultAbstractDisplay): def __init__(self, results: SubsetResults): super().__init__(results) - def display(self): - header = ["Order", "Test Path", "In Subset", "Estimated duration (sec)"] + def display(self, new_tests_only: bool = False): + header = ["Order", "Test Path", "In Subset", "Density", "Duration", "New"] + results = self._results.list() + if new_tests_only: + results = [r for r in results if r._is_new] rows = [] - for idx, result in enumerate(self._results.list()): + for idx, result in enumerate(results): rows.append( [ idx + 1, result._test_path, "✔" if result._is_subset else "", - result._estimated_duration_sec, + result._density, + "{:.3f}s".format(result._estimated_duration_sec), + "Yes" if result._is_new else "No", ] ) - click.echo_via_pager(tabulate(rows, header, tablefmt="github", floatfmt=".2f")) + click.echo_via_pager(tabulate(rows, header, tablefmt="github", floatfmt=".3f")) class SubsetResultJSONDisplay(SubsetResultAbstractDisplay): def __init__(self, results: SubsetResults): super().__init__(results) - def display(self): - result_json = { + def display(self, new_tests_only: bool = False): + result_json: Dict[str, List[Dict[str, Any]]] = { "subset": [], "rest": [] } for result in self._results.list_subset(): + if new_tests_only and not result._is_new: + continue result_json["subset"].append({ "test_path": result._test_path, "estimated_duration_sec": round(result._estimated_duration_sec, 2), + "density": result._density, + "is_new": result._is_new, }) for result in self._results.list_rest(): + if new_tests_only and not result._is_new: + continue result_json["rest"].append({ "test_path": result._test_path, "estimated_duration_sec": round(result._estimated_duration_sec, 2), + "density": result._density, + "is_new": result._is_new, }) click.echo(json.dumps(result_json, indent=2)) @@ -104,8 +119,14 @@ def display(self): help='display JSON format', is_flag=True ) +@click.option( + '--new-tests-only', + 'new_tests_only', + help='Only display new tests', + is_flag=True +) @click.pass_context -def subset(context: click.core.Context, subset_id: int, is_json_format: bool): +def subset(context: click.core.Context, subset_id: int, is_json_format: bool, new_tests_only: bool): subset = [] rest = [] client = LaunchableClient(app=context.obj) @@ -133,4 +154,4 @@ def subset(context: click.core.Context, subset_id: int, is_json_format: bool): else: displayer = SubsetResultTableDisplay(results) - displayer.display() + displayer.display(new_tests_only=new_tests_only) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index b25e2b2fb..419792ed8 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -750,6 +750,13 @@ def run(self): if "subset" not in summary.keys() or "rest" not in summary.keys(): return + new_test_count = summary["subset"].get("newTestCount", 0) + subset_label = "Subset (New Tests)" if new_test_count > 0 else "Subset" + if new_test_count > 0: + subset_candidates = "{} ({})".format(len(original_subset), new_test_count) + else: + subset_candidates = len(original_subset) + build_name, test_session_id = parse_session(session_id) org, workspace = get_org_workspace() @@ -757,8 +764,8 @@ def run(self): "Estimated duration (%)", "Estimated duration (min)"] rows = [ [ - "Subset", - len(original_subset), + subset_label, + subset_candidates, summary["subset"].get("rate", 0.0), summary["subset"].get("duration", 0.0), ], diff --git a/tests/commands/inspect/test_subset.py b/tests/commands/inspect/test_subset.py index 1738279b5..b14ad2493 100644 --- a/tests/commands/inspect/test_subset.py +++ b/tests/commands/inspect/test_subset.py @@ -10,17 +10,29 @@ class SubsetTest(CliTestCase): mock_json = { "testPaths": [ - {"testPath": [ - {"type": "file", "name": "test_file1.py"}], "duration": 1200}, - {"testPath": [ - {"type": "file", "name": "test_file3.py"}], "duration": 600}, + {"testPath": [ + {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 0}, + {"testPath": [ + {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, ], "rest": [ {"testPath": [ - {"type": "file", "name": "test_file4.py"}], "duration": 1800}, + {"type": "file", "name": "test_file4.py"}], "duration": 1800, "density": 0.8, "numNewTests": 0}, {"testPath": [ - {"type": "file", "name": "test_file2.py"}], "duration": 100} + {"type": "file", "name": "test_file2.py"}], "duration": 100, "density": 0.1, "numNewTests": 0}, + ] + } + mock_json_with_new_tests = { + "testPaths": [ + {"testPath": [ + {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 1}, + {"testPath": [ + {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, + ], + "rest": [ + {"testPath": [ + {"type": "file", "name": "test_file4.py"}], "duration": 1800, "density": 0.8, "numNewTests": 0}, ] } @@ -31,16 +43,50 @@ def test_subset(self): get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json, status=200) result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, mix_stderr=False) - expect = """| Order | Test Path | In Subset | Estimated duration (sec) | -|---------|--------------------|-------------|----------------------------| -| 1 | file=test_file1.py | ✔ | 1.20 | -| 2 | file=test_file3.py | ✔ | 0.60 | -| 3 | file=test_file4.py | | 1.80 | -| 4 | file=test_file2.py | | 0.10 | + expect = """| Order | Test Path | In Subset | Density | Duration | New | +|---------|--------------------|-------------|-----------|------------|-------| +| 1 | file=test_file1.py | ✔ | 0.500 | 1.200s | No | +| 2 | file=test_file3.py | ✔ | 0.300 | 0.600s | No | +| 3 | file=test_file4.py | | 0.800 | 1.800s | No | +| 4 | file=test_file2.py | | 0.100 | 0.100s | No | """ self.assertEqual(result.stdout, expect) + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_shows_new_column(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), + json=self.mock_json_with_new_tests, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, mix_stderr=False) + self.assertIn("Yes", result.stdout) + self.assertIn("No", result.stdout) + self.assertIn("file=test_file1.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_new_tests_only(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), + json=self.mock_json_with_new_tests, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, '--new-tests-only', mix_stderr=False) + self.assertIn("file=test_file1.py", result.stdout) + self.assertNotIn("file=test_file3.py", result.stdout) + self.assertNotIn("file=test_file4.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_new_tests_only_empty_in_brainless_mode(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, '--new-tests-only', mix_stderr=False) + self.assertNotIn("file=test_file1.py", result.stdout) + self.assertNotIn("file=test_file3.py", result.stdout) + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset_json_format(self): @@ -52,24 +98,55 @@ def test_subset_json_format(self): "subset": [ { "test_path": "file=test_file1.py", - "estimated_duration_sec": 1.2 + "estimated_duration_sec": 1.2, + "density": 0.5, + "is_new": false }, { "test_path": "file=test_file3.py", - "estimated_duration_sec": 0.6 + "estimated_duration_sec": 0.6, + "density": 0.3, + "is_new": false } ], "rest": [ { "test_path": "file=test_file4.py", - "estimated_duration_sec": 1.8 + "estimated_duration_sec": 1.8, + "density": 0.8, + "is_new": false }, { "test_path": "file=test_file2.py", - "estimated_duration_sec": 0.1 + "estimated_duration_sec": 0.1, + "density": 0.1, + "is_new": false } ] } +""" + + self.assertEqual(result.stdout, expect) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_json_new_tests_only(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), + json=self.mock_json_with_new_tests, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, "--json", "--new-tests-only", mix_stderr=False) + expect = """{ + "subset": [ + { + "test_path": "file=test_file1.py", + "estimated_duration_sec": 1.2, + "density": 0.5, + "is_new": true + } + ], + "rest": [] +} """ self.assertEqual(result.stdout, expect) diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index 55d0f0343..8711f1de0 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -917,3 +917,63 @@ def test_subset_id_file_no_id_returned(self): finally: if os.path.exists(id_file_path): os.unlink(id_file_path) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_shows_new_test_count_in_table(self): + pipe = "test_1.py\ntest_2.py\ntest_3.py" + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_1.py"}], + [{"type": "file", "name": "test_2.py"}], + ], + "testRunner": "file", + "rest": [ + [{"type": "file", "name": "test_3.py"}], + ], + "subsettingId": 123, + "summary": { + "subset": {"duration": 10, "candidates": 2, "rate": 50, "newTestCount": 1}, + "rest": {"duration": 10, "candidates": 1, "rate": 50} + }, + "isObservation": False, + } + responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace), + json=mock_json_response, status=200) + + result = self.cli("subset", "--target", "50%", "--session", + self.session, "file", mix_stderr=False, input=pipe) + self.assert_success(result) + self.assertIn("Subset (New Tests)", result.stderr) + self.assertIn("2 (1)", result.stderr) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_shows_plain_subset_label_when_no_new_tests(self): + pipe = "test_1.py\ntest_2.py\ntest_3.py" + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_1.py"}], + [{"type": "file", "name": "test_2.py"}], + ], + "testRunner": "file", + "rest": [ + [{"type": "file", "name": "test_3.py"}], + ], + "subsettingId": 123, + "summary": { + "subset": {"duration": 10, "candidates": 2, "rate": 50}, + "rest": {"duration": 10, "candidates": 1, "rate": 50} + }, + "isObservation": False, + } + responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace), + json=mock_json_response, status=200) + + result = self.cli("subset", "--target", "50%", "--session", + self.session, "file", mix_stderr=False, input=pipe) + self.assert_success(result) + self.assertIn("| Subset", result.stderr) + self.assertNotIn("Subset (New Tests)", result.stderr)