From c8fa8703c88c64b0661761ba485629da1a419f53 Mon Sep 17 00:00:00 2001 From: Rahul Tripathi Date: Sat, 7 Mar 2026 03:18:53 +0530 Subject: [PATCH 1/4] feat(regression): add BaselineStatus to distinguish never-worked tests Adds a baseline_status field to RegressionTest with three states: - unknown - newly added test, no run history - never_worked - has run history, but never passed on any CCExtractor version - established - has at least one passing run (a failure here is a regression) This makes it possible to distinguish true regressions from samples that have never produced correct output, addressing the 'Never worked' test state goal from the Sample Platform NG project brief. Includes: - BaselineStatus DeclEnum in mod_regression/models.py - RegressionTest.update_baseline_status(passed) state-machine method - RegressionTest.is_regression property - Flask-Migrate migration (d1f3a9c2e8b7) with upgrade/downgrade - 12 unit tests covering all state transitions and DB persistence --- migrations/versions/d1f3a9c2e8b7_.py | 37 +++++ mod_regression/models.py | 63 ++++++- tests/test_regression/test_baseline_status.py | 157 ++++++++++++++++++ 3 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/d1f3a9c2e8b7_.py create mode 100644 tests/test_regression/test_baseline_status.py diff --git a/migrations/versions/d1f3a9c2e8b7_.py b/migrations/versions/d1f3a9c2e8b7_.py new file mode 100644 index 00000000..7fd2cad1 --- /dev/null +++ b/migrations/versions/d1f3a9c2e8b7_.py @@ -0,0 +1,37 @@ +"""Add baseline_status to regression_test for never-worked tracking + +Revision ID: d1f3a9c2e8b7 +Revises: c8f3a2b1d4e5 +Create Date: 2026-03-07 00:00:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'd1f3a9c2e8b7' +down_revision = 'c8f3a2b1d4e5' +branch_labels = None +depends_on = None + +# Enum values mirror BaselineStatus in mod_regression/models.py +baseline_status_enum = sa.Enum('unknown', 'never_worked', 'established', name='baselinestatus') + + +def upgrade(): + """Add baseline_status column to regression_test table.""" + # Add column with default so existing rows get 'unknown' immediately + op.add_column( + 'regression_test', + sa.Column( + 'baseline_status', + baseline_status_enum, + nullable=False, + server_default='unknown' + ) + ) + + +def downgrade(): + """Remove baseline_status column from regression_test table.""" + op.drop_column('regression_test', 'baseline_status') diff --git a/mod_regression/models.py b/mod_regression/models.py index 9ade0a61..1c573c02 100644 --- a/mod_regression/models.py +++ b/mod_regression/models.py @@ -59,6 +59,24 @@ def __repr__(self) -> str: return f"" +class BaselineStatus(DeclEnum): + """Enum to track whether a regression test has ever passed. + + This distinguishes true regressions (tests that used to pass but now fail) + from tests that have never produced correct output on any CCExtractor version. + + Transitions: + unknown -> established (first test run passes) + unknown -> never_worked (first test run fails) + never_worked -> established (a passing run occurs; test now works) + established stays established (a failure is a regression, not "never worked") + """ + + unknown = "unknown", "Unknown" + never_worked = "never_worked", "Never Worked" + established = "established", "Established" + + class InputType(DeclEnum): """Enumerator types for input.""" @@ -97,6 +115,7 @@ class RegressionTest(Base): last_passed_on_windows = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL")) last_passed_on_linux = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL")) description = Column(String(length=1024)) + baseline_status = Column(BaselineStatus.db_type(), nullable=False, default=BaselineStatus.unknown) def __init__(self, sample_id, command, input_type, output_type, category_id, expected_rc, active=True, description="") -> None: @@ -117,7 +136,6 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp :type expected_rc: int :param active: The value of the 'active' field of RegressionTest model :type active: bool - """ self.sample_id = sample_id self.command = command @@ -127,6 +145,7 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp self.expected_rc = expected_rc self.active = active self.description = description + self.baseline_status = BaselineStatus.unknown def __repr__(self) -> str: """ @@ -137,6 +156,48 @@ def __repr__(self) -> str: """ return f"" + def update_baseline_status(self, passed: bool) -> bool: + """ + Update baseline_status based on the outcome of a test run. + + Called after each completed test run for this regression test. + Returns True if the status changed, False if it stayed the same. + + Transition table:: + + unknown + pass -> established + unknown + fail -> never_worked + never_worked + pass -> established + never_worked + fail -> never_worked (no change) + established + pass -> established (no change) + established + fail -> established (it's a regression, not "never worked") + + :param passed: True if exit_code matched expected_rc for this test run. + :type passed: bool + :return: True if the baseline_status changed, False otherwise. + :rtype: bool + """ + previous = self.baseline_status + if passed: + self.baseline_status = BaselineStatus.established + elif self.baseline_status == BaselineStatus.unknown: + self.baseline_status = BaselineStatus.never_worked + return self.baseline_status != previous + + @property + def is_regression(self) -> bool: + """ + Return True if a failing result on this test is a true regression. + + A result is a regression only when the test is established (has passed before) + but is currently failing. Tests with 'never_worked' or 'unknown' status are + not regressions; they are pre-existing issues. + + :return: True if this test can produce a regression result. + :rtype: bool + """ + return self.baseline_status == BaselineStatus.established + class RegressionTestOutput(Base): """Model to store output of regression test.""" diff --git a/tests/test_regression/test_baseline_status.py b/tests/test_regression/test_baseline_status.py new file mode 100644 index 00000000..f8ab478d --- /dev/null +++ b/tests/test_regression/test_baseline_status.py @@ -0,0 +1,157 @@ +"""Tests for the BaselineStatus feature on RegressionTest. + +Verifies the state-machine transitions in RegressionTest.update_baseline_status +and the is_regression property. +""" + +from flask import g + +from mod_regression.models import BaselineStatus, RegressionTest +from tests.base import BaseTestCase + + +class TestBaselineStatus(BaseTestCase): + """Tests for the BaselineStatus state machine.""" + + def _get_regression_test(self, test_id: int = 1) -> RegressionTest: + """Fetch a RegressionTest from the test database.""" + return RegressionTest.query.filter(RegressionTest.id == test_id).first() + + # ------------------------------------------------------------------ + # Initial state + # ------------------------------------------------------------------ + + def test_new_regression_test_has_unknown_baseline(self): + """A freshly created regression test must have baseline_status = unknown.""" + rt = self._get_regression_test() + self.assertEqual(rt.baseline_status, BaselineStatus.unknown) + + # ------------------------------------------------------------------ + # unknown -> established (first run passes) + # ------------------------------------------------------------------ + + def test_unknown_pass_transitions_to_established(self): + """unknown + pass -> established.""" + rt = self._get_regression_test() + self.assertEqual(rt.baseline_status, BaselineStatus.unknown) + + changed = rt.update_baseline_status(passed=True) + + self.assertTrue(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # unknown -> never_worked (first run fails) + # ------------------------------------------------------------------ + + def test_unknown_fail_transitions_to_never_worked(self): + """unknown + fail -> never_worked.""" + rt = self._get_regression_test() + self.assertEqual(rt.baseline_status, BaselineStatus.unknown) + + changed = rt.update_baseline_status(passed=False) + + self.assertTrue(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.never_worked) + + # ------------------------------------------------------------------ + # never_worked -> established (first pass after never working) + # ------------------------------------------------------------------ + + def test_never_worked_pass_transitions_to_established(self): + """never_worked + pass -> established.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) # unknown -> never_worked + self.assertEqual(rt.baseline_status, BaselineStatus.never_worked) + + changed = rt.update_baseline_status(passed=True) + + self.assertTrue(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # never_worked + fail -> never_worked (no change) + # ------------------------------------------------------------------ + + def test_never_worked_fail_stays_never_worked(self): + """never_worked + fail -> never_worked (no transition).""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) # unknown -> never_worked + + changed = rt.update_baseline_status(passed=False) + + self.assertFalse(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.never_worked) + + # ------------------------------------------------------------------ + # established + pass -> established (no change) + # ------------------------------------------------------------------ + + def test_established_pass_stays_established(self): + """established + pass -> established (no transition).""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) # unknown -> established + + changed = rt.update_baseline_status(passed=True) + + self.assertFalse(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # established + fail -> established (regression, not never_worked) + # ------------------------------------------------------------------ + + def test_established_fail_stays_established(self): + """established + fail -> established (it is a regression, not never_worked).""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) # unknown -> established + + changed = rt.update_baseline_status(passed=False) + + self.assertFalse(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # is_regression property + # ------------------------------------------------------------------ + + def test_is_regression_false_when_unknown(self): + """A test with unknown baseline cannot be a regression.""" + rt = self._get_regression_test() + self.assertFalse(rt.is_regression) + + def test_is_regression_false_when_never_worked(self): + """A test that never worked is not a regression when it fails.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) + self.assertFalse(rt.is_regression) + + def test_is_regression_true_when_established(self): + """An established test that fails is a true regression.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) + self.assertTrue(rt.is_regression) + + # ------------------------------------------------------------------ + # Persistence: changes survive a database round-trip + # ------------------------------------------------------------------ + + def test_baseline_status_persists_to_database(self): + """Verify baseline_status is actually saved and reloaded from DB.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) # unknown -> never_worked + g.db.add(rt) + g.db.commit() + + reloaded = RegressionTest.query.filter(RegressionTest.id == 1).first() + self.assertEqual(reloaded.baseline_status, BaselineStatus.never_worked) + + def test_established_status_persists_to_database(self): + """Verify established status is saved and reloaded from DB.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) # unknown -> established + g.db.add(rt) + g.db.commit() + + reloaded = RegressionTest.query.filter(RegressionTest.id == 1).first() + self.assertEqual(reloaded.baseline_status, BaselineStatus.established) From 38534ab62a79350885bbb12c96e1c269e721a8e4 Mon Sep 17 00:00:00 2001 From: Rahul Tripathi Date: Wed, 11 Mar 2026 12:52:09 +0530 Subject: [PATCH 2/4] Integrate never-worked baseline handling --- migrations/versions/d1f3a9c2e8b7_.py | 11 ++ mod_ci/controllers.py | 54 +++++++- mod_ci/models.py | 1 + templates/ci/pr_comment.txt | 17 ++- tests/test_ci/test_controllers.py | 189 +++++++++++++++++++++++++-- 5 files changed, 255 insertions(+), 17 deletions(-) diff --git a/migrations/versions/d1f3a9c2e8b7_.py b/migrations/versions/d1f3a9c2e8b7_.py index 7fd2cad1..8e95d17c 100644 --- a/migrations/versions/d1f3a9c2e8b7_.py +++ b/migrations/versions/d1f3a9c2e8b7_.py @@ -31,6 +31,17 @@ def upgrade(): ) ) + # Historical backfill: + # - if the regression test has ever passed on either tracked platform, it is established + # - otherwise keep the trusted state as unknown until a main-repo commit run refreshes it + op.execute( + """ + UPDATE regression_test + SET baseline_status = 'established' + WHERE last_passed_on_linux IS NOT NULL OR last_passed_on_windows IS NOT NULL + """ + ) + def downgrade(): """Remove baseline_status column from regression_test table.""" diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index d625103a..d1a9a5f6 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -41,7 +41,7 @@ Status) from mod_customized.models import CustomizedTest from mod_home.models import CCExtractorVersion, GeneralData -from mod_regression.models import (Category, RegressionTest, +from mod_regression.models import (BaselineStatus, Category, RegressionTest, RegressionTestOutput) from mod_sample.models import Issue from mod_test.controllers import get_test_results @@ -2421,6 +2421,8 @@ def progress_type_request(log, test, test_id, request) -> bool: message = 'Tests aborted due to an error; please check' elif status == TestStatus.completed: + if test.test_type == TestType.commit and is_main_repo(test.fork.github): + refresh_baseline_statuses_for_test(test) # Determine if success or failure # It fails if any of these happen: # - A crash (unexpected exit code) @@ -2707,6 +2709,39 @@ def finish_type_request(log, test_id, test, request): log.error(f"Could not save the results for test {test_id}") +def refresh_baseline_statuses_for_test(test: Test) -> None: + """ + Persist baseline status for each regression test touched by a completed test run. + + This uses the same full-result logic as the UI and PR comment paths, so output-file + mismatches and missing expected outputs count as failures in addition to exit-code + mismatches. + + :param test: The completed test run whose regression results should refresh baseline state. + :type test: Test + """ + from run import log + + if test.test_type != TestType.commit or not is_main_repo(test.fork.github): + return + + changed = False + processed_ids = set() + for category_results in get_test_results(test): + for category_test in category_results['tests']: + regression_test = category_test['test'] + if regression_test.id in processed_ids or category_test['result'] is None: + continue + + processed_ids.add(regression_test.id) + if regression_test.update_baseline_status(passed=not category_test['error']): + g.db.add(regression_test) + changed = True + + if changed and not safe_db_commit(g.db, f"refreshing baseline status for test {test.id}"): + log.error(f"Failed to refresh baseline status for completed test {test.id}") + + def set_avg_time(platform, process_type: str, time_taken: int) -> None: """ Set average platform preparation time. @@ -2756,6 +2791,7 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo: extra_failed_tests = [] common_failed_tests = [] fixed_tests = [] + never_worked_tests = [] category_stats = [] test_results = get_test_results(test) @@ -2765,19 +2801,29 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo: category_test_pass_count = 0 for test in category_results['tests']: + platform_last_passed = getattr(test['test'], platform_column) if not test['error']: category_test_pass_count += 1 - if last_test_master and getattr(test['test'], platform_column) != last_test_master.id: + if last_test_master and platform_last_passed != last_test_master.id: fixed_tests.append(test['test']) else: - if last_test_master and getattr(test['test'], platform_column) != last_test_master.id: + if platform_last_passed is None and test['test'].baseline_status != BaselineStatus.unknown: + never_worked_tests.append(test['test']) + elif last_test_master and platform_last_passed != last_test_master.id: common_failed_tests.append(test['test']) else: extra_failed_tests.append(test['test']) category_stats.append(CategoryTestInfo(category_name, len(category_results['tests']), category_test_pass_count)) - return PrCommentInfo(category_stats, extra_failed_tests, fixed_tests, common_failed_tests, last_test_master) + return PrCommentInfo( + category_stats, + extra_failed_tests, + fixed_tests, + common_failed_tests, + never_worked_tests, + last_test_master, + ) def comment_pr(test: Test) -> str: diff --git a/mod_ci/models.py b/mod_ci/models.py index 1b5c9ebf..5d89bbff 100644 --- a/mod_ci/models.py +++ b/mod_ci/models.py @@ -178,4 +178,5 @@ class PrCommentInfo: extra_failed_tests: List[RegressionTest] fixed_tests: List[RegressionTest] common_failed_tests: List[RegressionTest] + never_worked_tests: List[RegressionTest] last_test_master: Test diff --git a/templates/ci/pr_comment.txt b/templates/ci/pr_comment.txt index 56ae2936..57853b4b 100644 --- a/templates/ci/pr_comment.txt +++ b/templates/ci/pr_comment.txt @@ -41,11 +41,22 @@ NOTE: The following tests have been failing on the master branch as well as the {% endfor %} {% endif %} +{% if comment_info.never_worked_tests | length %} +NOTE: The following tests have never passed on the platform yet: +
    +{% for test in comment_info.never_worked_tests %} +
  • ccextractor {{ test.command }} {{ test.sample.sha[:10] }}..., Last passed: +Never +
    • +{% endfor %} +
+{% endif %} {% if comment_info.fixed_tests | length %} Congratulations: Merging this PR would fix the following tests: {% endif %} @@ -54,8 +65,8 @@ Congratulations: Merging this PR would fix the following tests: {% if comment_info.extra_failed_tests | length %} It seems that not all tests were passed completely. This is an indication that the output of some files is not as expected (but might be according to you). -{% elif comment_info.common_failed_tests | length %} -This PR does not introduce any new test failures. However, some tests are failing on both master and this PR (see above). +{% elif comment_info.common_failed_tests | length or comment_info.never_worked_tests | length %} +This PR does not introduce any new test failures. However, some tests are already failing on master or have never worked on the platform yet (see above). {% else %} All tests passed completely. {% endif %} diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index cf62edd2..ab268b49 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -14,9 +14,11 @@ from mod_ci.models import BlockedUsers from mod_customized.models import CustomizedTest from mod_home.models import CCExtractorVersion, GeneralData -from mod_regression.models import (RegressionTest, RegressionTestOutput, +from mod_regression.models import (BaselineStatus, RegressionTest, + RegressionTestOutput, RegressionTestOutputFiles) -from mod_test.models import Test, TestPlatform, TestResultFile, TestType +from mod_test.models import (Test, TestPlatform, TestProgress, TestResult, + TestResultFile, TestStatus, TestType) from tests.base import (BaseTestCase, MockResponse, create_mock_db_query, create_mock_regression_test, empty_github_token, generate_git_api_header, generate_signature, @@ -68,6 +70,177 @@ def __init__(self): class TestControllers(BaseTestCase): """Test CI-related controllers.""" + def test_refresh_baseline_statuses_for_test_uses_full_result_status(self): + """Refresh baseline status using full regression results, not just exit codes.""" + from mod_ci.controllers import refresh_baseline_statuses_for_test + + test = Test(TestPlatform.linux, TestType.commit, 1, "master", "abc1234") + g.db.add(test) + g.db.commit() + + g.db.add_all([ + TestProgress(test.id, TestStatus.preparation, "prep"), + TestProgress(test.id, TestStatus.testing, "testing"), + TestProgress(test.id, TestStatus.completed, "done"), + TestResult(test.id, 1, 250, 1, 0), + TestResult(test.id, 2, 250, 0, 0), + TestResultFile(test.id, 1, 1, "sample_out1", "wrong"), + TestResultFile(test.id, 2, 2, "sample_out2"), + ]) + g.db.commit() + + regression_failed = RegressionTest.query.get(1) + regression_passed = RegressionTest.query.get(2) + + self.assertEqual(regression_failed.baseline_status, BaselineStatus.unknown) + self.assertEqual(regression_passed.baseline_status, BaselineStatus.unknown) + + refresh_baseline_statuses_for_test(test) + + self.assertEqual(regression_failed.baseline_status, BaselineStatus.never_worked) + self.assertEqual(regression_passed.baseline_status, BaselineStatus.established) + + def test_refresh_baseline_statuses_for_test_skips_pull_request_runs(self): + """Pull request runs must not rewrite the shared regression baseline.""" + from mod_ci.controllers import refresh_baseline_statuses_for_test + + test = Test.query.get(2) + regression_failed = RegressionTest.query.get(1) + regression_passed = RegressionTest.query.get(2) + passing_result_file = TestResultFile.query.filter( + TestResultFile.test_id == test.id, + TestResultFile.regression_test_id == regression_passed.id, + ).first() + passing_result_file.got = None + g.db.add(passing_result_file) + g.db.commit() + + refresh_baseline_statuses_for_test(test) + + self.assertEqual(regression_failed.baseline_status, BaselineStatus.unknown) + self.assertEqual(regression_passed.baseline_status, BaselineStatus.unknown) + + def test_comment_info_separates_never_worked_failures(self): + """Pre-existing never-worked tests should not be reported as PR breakage.""" + from mod_ci.controllers import get_info_for_pr_comment + from mod_customized.models import CustomizedTest + + test = Test.query.get(2) + regression_test = RegressionTest.query.get(1) + regression_test.baseline_status = BaselineStatus.never_worked + g.db.add(regression_test) + g.db.add(CustomizedTest(test.id, regression_test.id)) + g.db.commit() + + comment_info = get_info_for_pr_comment(test) + + self.assertIn(regression_test, comment_info.never_worked_tests) + self.assertEqual(comment_info.extra_failed_tests, []) + self.assertEqual(comment_info.common_failed_tests, []) + + def test_comment_info_tracks_never_worked_per_platform(self): + """A test that has only passed on Linux should still be never-worked on Windows.""" + from mod_ci.controllers import get_info_for_pr_comment + from mod_customized.models import CustomizedTest + + regression_test = RegressionTest.query.get(1) + regression_test.baseline_status = BaselineStatus.established + regression_test.last_passed_on_linux = 1 + regression_test.last_passed_on_windows = None + g.db.add(regression_test) + g.db.commit() + + test = Test(TestPlatform.windows, TestType.pull_request, 1, "pull_request", "windowssha", 1) + g.db.add(test) + g.db.commit() + + g.db.add(CustomizedTest(test.id, regression_test.id)) + g.db.add(TestResult(test.id, regression_test.id, 250, 1, 0)) + g.db.commit() + + comment_info = get_info_for_pr_comment(test) + + self.assertIn(regression_test, comment_info.never_worked_tests) + self.assertEqual(comment_info.extra_failed_tests, []) + self.assertEqual(comment_info.common_failed_tests, []) + + @mock.patch('mod_ci.controllers.refresh_baseline_statuses_for_test') + @mock.patch('mod_ci.controllers.retry_with_backoff') + @mock.patch('mod_ci.controllers.update_build_badge') + @mock.patch('mod_ci.controllers.update_status_on_github') + @mock.patch('mod_ci.controllers.delete_instance_with_tracking') + @mock.patch('mod_ci.controllers.get_compute_service_object') + @mock.patch('mod_ci.controllers.Github') + def test_progress_type_request_completed_refreshes_baseline_status( + self, + mock_github, + mock_get_compute_service_object, + mock_delete_instance_with_tracking, + mock_update_status, + mock_update_build_badge, + mock_retry_with_backoff, + mock_refresh_baseline_statuses, + ): + """Completed test reporting should refresh baseline state before downstream reporting.""" + test = Test(TestPlatform.linux, TestType.commit, 1, "master", "abc1234") + g.db.add(test) + g.db.commit() + + progress_entries = [ + TestProgress(test.id, TestStatus.preparation, "prep"), + TestProgress(test.id, TestStatus.testing, "testing"), + ] + g.db.add_all(progress_entries) + g.db.add(TestResult(test.id, 1, 250, 0, 0)) + g.db.add(TestResultFile(test.id, 1, 1, "sample_out1")) + g.db.commit() + + request = MagicMock() + request.form = { + 'status': 'completed', + 'message': 'done', + } + log = MagicMock() + mock_retry_with_backoff.side_effect = lambda func, **kwargs: func() + mock_delete_instance_with_tracking.return_value = {'name': 'delete-op'} + + progress_type_request(log, test, test.id, request) + + mock_refresh_baseline_statuses.assert_called_once_with(test) + + @mock.patch('mod_ci.controllers.refresh_baseline_statuses_for_test') + @mock.patch('mod_ci.controllers.retry_with_backoff') + @mock.patch('mod_ci.controllers.update_build_badge') + @mock.patch('mod_ci.controllers.update_status_on_github') + @mock.patch('mod_ci.controllers.delete_instance_with_tracking') + @mock.patch('mod_ci.controllers.get_compute_service_object') + @mock.patch('mod_ci.controllers.Github') + def test_progress_type_request_completed_does_not_refresh_baseline_for_pull_requests( + self, + mock_github, + mock_get_compute_service_object, + mock_delete_instance_with_tracking, + mock_update_status, + mock_update_build_badge, + mock_retry_with_backoff, + mock_refresh_baseline_statuses, + ): + """Pull request completion should not mutate shared baseline state.""" + test = Test.query.get(2) + + request = MagicMock() + request.form = { + 'status': 'completed', + 'message': 'done', + } + log = MagicMock() + mock_retry_with_backoff.side_effect = lambda func, **kwargs: func() + mock_delete_instance_with_tracking.return_value = {'name': 'delete-op'} + + progress_type_request(log, test, test.id, request) + + mock_refresh_baseline_statuses.assert_not_called() + def test_comment_info_handles_variant_files_correctly(self): """Test that allowed variants of output files are handled correctly in PR comments. @@ -394,14 +567,12 @@ def test_set_avg_time(self, mock_g, mock_gd, mock_int): self.assertEqual(mock_g.db.add.call_count, 0) mock_g.db.commit.assert_called_once() - @mock.patch('github.Github') + @mock.patch('mod_ci.controllers.Github') def test_comments_successfully_in_passed_pr_test(self, mock_github): """Check comments in passed PR test.""" - import mod_ci.controllers - reload(mod_ci.controllers) from github.IssueComment import IssueComment - from mod_ci.controllers import Status, comment_pr + from mod_ci.controllers import comment_pr from mod_test.models import Test pull_request = mock_github.return_value.get_repo.return_value.get_pull(number=1) pull_request.get_issue_comments.return_value = [MagicMock(IssueComment)] @@ -415,12 +586,10 @@ def test_comments_successfully_in_passed_pr_test(self, mock_github): if "passed" not in message: assert False, "Message not Correct" - @mock.patch('mod_test.controllers.get_test_results') - @mock.patch('github.Github') + @mock.patch('mod_ci.controllers.get_test_results') + @mock.patch('mod_ci.controllers.Github') def test_comments_successfuly_in_failed_pr_test(self, mock_github, mock_get_test_results): """Check comments in failed PR test.""" - import mod_ci.controllers - reload(mod_ci.controllers) from github.IssueComment import IssueComment from mod_ci.controllers import comment_pr From 1f688b46fdf8daf423fc95cf03fa6dc3a1816d9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 11 Mar 2026 07:27:50 +0000 Subject: [PATCH 3/4] Initial plan From a0ab2e161fe23e5227501897b304053a52cd7fde Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 11 Mar 2026 07:39:06 +0000 Subject: [PATCH 4/4] Fix variable shadowing and missing docstring in baseline integration Co-authored-by: Rahul-2k4 <216878448+Rahul-2k4@users.noreply.github.com> --- mod_ci/controllers.py | 16 ++++++++-------- mod_regression/models.py | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index d1a9a5f6..63792891 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -2800,19 +2800,19 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo: category_name = category_results['category'].name category_test_pass_count = 0 - for test in category_results['tests']: - platform_last_passed = getattr(test['test'], platform_column) - if not test['error']: + for category_test in category_results['tests']: + platform_last_passed = getattr(category_test['test'], platform_column) + if not category_test['error']: category_test_pass_count += 1 if last_test_master and platform_last_passed != last_test_master.id: - fixed_tests.append(test['test']) + fixed_tests.append(category_test['test']) else: - if platform_last_passed is None and test['test'].baseline_status != BaselineStatus.unknown: - never_worked_tests.append(test['test']) + if platform_last_passed is None and category_test['test'].baseline_status != BaselineStatus.unknown: + never_worked_tests.append(category_test['test']) elif last_test_master and platform_last_passed != last_test_master.id: - common_failed_tests.append(test['test']) + common_failed_tests.append(category_test['test']) else: - extra_failed_tests.append(test['test']) + extra_failed_tests.append(category_test['test']) category_stats.append(CategoryTestInfo(category_name, len(category_results['tests']), category_test_pass_count)) diff --git a/mod_regression/models.py b/mod_regression/models.py index 1c573c02..08240128 100644 --- a/mod_regression/models.py +++ b/mod_regression/models.py @@ -136,6 +136,8 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp :type expected_rc: int :param active: The value of the 'active' field of RegressionTest model :type active: bool + :param description: The value of the 'description' field of RegressionTest model + :type description: str """ self.sample_id = sample_id self.command = command