diff --git a/migrations/versions/d1f3a9c2e8b7_.py b/migrations/versions/d1f3a9c2e8b7_.py new file mode 100644 index 00000000..8e95d17c --- /dev/null +++ b/migrations/versions/d1f3a9c2e8b7_.py @@ -0,0 +1,48 @@ +"""Add baseline_status to regression_test for never-worked tracking + +Revision ID: d1f3a9c2e8b7 +Revises: c8f3a2b1d4e5 +Create Date: 2026-03-07 00:00:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'd1f3a9c2e8b7' +down_revision = 'c8f3a2b1d4e5' +branch_labels = None +depends_on = None + +# Enum values mirror BaselineStatus in mod_regression/models.py +baseline_status_enum = sa.Enum('unknown', 'never_worked', 'established', name='baselinestatus') + + +def upgrade(): + """Add baseline_status column to regression_test table.""" + # Add column with default so existing rows get 'unknown' immediately + op.add_column( + 'regression_test', + sa.Column( + 'baseline_status', + baseline_status_enum, + nullable=False, + server_default='unknown' + ) + ) + + # Historical backfill: + # - if the regression test has ever passed on either tracked platform, it is established + # - otherwise keep the trusted state as unknown until a main-repo commit run refreshes it + op.execute( + """ + UPDATE regression_test + SET baseline_status = 'established' + WHERE last_passed_on_linux IS NOT NULL OR last_passed_on_windows IS NOT NULL + """ + ) + + +def downgrade(): + """Remove baseline_status column from regression_test table.""" + op.drop_column('regression_test', 'baseline_status') diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index d625103a..63792891 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -41,7 +41,7 @@ Status) from mod_customized.models import CustomizedTest from mod_home.models import CCExtractorVersion, GeneralData -from mod_regression.models import (Category, RegressionTest, +from mod_regression.models import (BaselineStatus, Category, RegressionTest, RegressionTestOutput) from mod_sample.models import Issue from mod_test.controllers import get_test_results @@ -2421,6 +2421,8 @@ def progress_type_request(log, test, test_id, request) -> bool: message = 'Tests aborted due to an error; please check' elif status == TestStatus.completed: + if test.test_type == TestType.commit and is_main_repo(test.fork.github): + refresh_baseline_statuses_for_test(test) # Determine if success or failure # It fails if any of these happen: # - A crash (unexpected exit code) @@ -2707,6 +2709,39 @@ def finish_type_request(log, test_id, test, request): log.error(f"Could not save the results for test {test_id}") +def refresh_baseline_statuses_for_test(test: Test) -> None: + """ + Persist baseline status for each regression test touched by a completed test run. + + This uses the same full-result logic as the UI and PR comment paths, so output-file + mismatches and missing expected outputs count as failures in addition to exit-code + mismatches. + + :param test: The completed test run whose regression results should refresh baseline state. + :type test: Test + """ + from run import log + + if test.test_type != TestType.commit or not is_main_repo(test.fork.github): + return + + changed = False + processed_ids = set() + for category_results in get_test_results(test): + for category_test in category_results['tests']: + regression_test = category_test['test'] + if regression_test.id in processed_ids or category_test['result'] is None: + continue + + processed_ids.add(regression_test.id) + if regression_test.update_baseline_status(passed=not category_test['error']): + g.db.add(regression_test) + changed = True + + if changed and not safe_db_commit(g.db, f"refreshing baseline status for test {test.id}"): + log.error(f"Failed to refresh baseline status for completed test {test.id}") + + def set_avg_time(platform, process_type: str, time_taken: int) -> None: """ Set average platform preparation time. @@ -2756,6 +2791,7 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo: extra_failed_tests = [] common_failed_tests = [] fixed_tests = [] + never_worked_tests = [] category_stats = [] test_results = get_test_results(test) @@ -2764,20 +2800,30 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo: category_name = category_results['category'].name category_test_pass_count = 0 - for test in category_results['tests']: - if not test['error']: + for category_test in category_results['tests']: + platform_last_passed = getattr(category_test['test'], platform_column) + if not category_test['error']: category_test_pass_count += 1 - if last_test_master and getattr(test['test'], platform_column) != last_test_master.id: - fixed_tests.append(test['test']) + if last_test_master and platform_last_passed != last_test_master.id: + fixed_tests.append(category_test['test']) else: - if last_test_master and getattr(test['test'], platform_column) != last_test_master.id: - common_failed_tests.append(test['test']) + if platform_last_passed is None and category_test['test'].baseline_status != BaselineStatus.unknown: + never_worked_tests.append(category_test['test']) + elif last_test_master and platform_last_passed != last_test_master.id: + common_failed_tests.append(category_test['test']) else: - extra_failed_tests.append(test['test']) + extra_failed_tests.append(category_test['test']) category_stats.append(CategoryTestInfo(category_name, len(category_results['tests']), category_test_pass_count)) - return PrCommentInfo(category_stats, extra_failed_tests, fixed_tests, common_failed_tests, last_test_master) + return PrCommentInfo( + category_stats, + extra_failed_tests, + fixed_tests, + common_failed_tests, + never_worked_tests, + last_test_master, + ) def comment_pr(test: Test) -> str: diff --git a/mod_ci/models.py b/mod_ci/models.py index 1b5c9ebf..5d89bbff 100644 --- a/mod_ci/models.py +++ b/mod_ci/models.py @@ -178,4 +178,5 @@ class PrCommentInfo: extra_failed_tests: List[RegressionTest] fixed_tests: List[RegressionTest] common_failed_tests: List[RegressionTest] + never_worked_tests: List[RegressionTest] last_test_master: Test diff --git a/mod_regression/models.py b/mod_regression/models.py index 9ade0a61..08240128 100644 --- a/mod_regression/models.py +++ b/mod_regression/models.py @@ -59,6 +59,24 @@ def __repr__(self) -> str: return f"" +class BaselineStatus(DeclEnum): + """Enum to track whether a regression test has ever passed. + + This distinguishes true regressions (tests that used to pass but now fail) + from tests that have never produced correct output on any CCExtractor version. + + Transitions: + unknown -> established (first test run passes) + unknown -> never_worked (first test run fails) + never_worked -> established (a passing run occurs; test now works) + established stays established (a failure is a regression, not "never worked") + """ + + unknown = "unknown", "Unknown" + never_worked = "never_worked", "Never Worked" + established = "established", "Established" + + class InputType(DeclEnum): """Enumerator types for input.""" @@ -97,6 +115,7 @@ class RegressionTest(Base): last_passed_on_windows = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL")) last_passed_on_linux = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL")) description = Column(String(length=1024)) + baseline_status = Column(BaselineStatus.db_type(), nullable=False, default=BaselineStatus.unknown) def __init__(self, sample_id, command, input_type, output_type, category_id, expected_rc, active=True, description="") -> None: @@ -117,7 +136,8 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp :type expected_rc: int :param active: The value of the 'active' field of RegressionTest model :type active: bool - + :param description: The value of the 'description' field of RegressionTest model + :type description: str """ self.sample_id = sample_id self.command = command @@ -127,6 +147,7 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp self.expected_rc = expected_rc self.active = active self.description = description + self.baseline_status = BaselineStatus.unknown def __repr__(self) -> str: """ @@ -137,6 +158,48 @@ def __repr__(self) -> str: """ return f"" + def update_baseline_status(self, passed: bool) -> bool: + """ + Update baseline_status based on the outcome of a test run. + + Called after each completed test run for this regression test. + Returns True if the status changed, False if it stayed the same. + + Transition table:: + + unknown + pass -> established + unknown + fail -> never_worked + never_worked + pass -> established + never_worked + fail -> never_worked (no change) + established + pass -> established (no change) + established + fail -> established (it's a regression, not "never worked") + + :param passed: True if exit_code matched expected_rc for this test run. + :type passed: bool + :return: True if the baseline_status changed, False otherwise. + :rtype: bool + """ + previous = self.baseline_status + if passed: + self.baseline_status = BaselineStatus.established + elif self.baseline_status == BaselineStatus.unknown: + self.baseline_status = BaselineStatus.never_worked + return self.baseline_status != previous + + @property + def is_regression(self) -> bool: + """ + Return True if a failing result on this test is a true regression. + + A result is a regression only when the test is established (has passed before) + but is currently failing. Tests with 'never_worked' or 'unknown' status are + not regressions; they are pre-existing issues. + + :return: True if this test can produce a regression result. + :rtype: bool + """ + return self.baseline_status == BaselineStatus.established + class RegressionTestOutput(Base): """Model to store output of regression test.""" diff --git a/templates/ci/pr_comment.txt b/templates/ci/pr_comment.txt index 56ae2936..57853b4b 100644 --- a/templates/ci/pr_comment.txt +++ b/templates/ci/pr_comment.txt @@ -41,11 +41,22 @@ NOTE: The following tests have been failing on the master branch as well as the {% endfor %} {% endif %} +{% if comment_info.never_worked_tests | length %} +NOTE: The following tests have never passed on the platform yet: + +{% endif %} {% if comment_info.fixed_tests | length %} Congratulations: Merging this PR would fix the following tests: {% endif %} @@ -54,8 +65,8 @@ Congratulations: Merging this PR would fix the following tests: {% if comment_info.extra_failed_tests | length %} It seems that not all tests were passed completely. This is an indication that the output of some files is not as expected (but might be according to you). -{% elif comment_info.common_failed_tests | length %} -This PR does not introduce any new test failures. However, some tests are failing on both master and this PR (see above). +{% elif comment_info.common_failed_tests | length or comment_info.never_worked_tests | length %} +This PR does not introduce any new test failures. However, some tests are already failing on master or have never worked on the platform yet (see above). {% else %} All tests passed completely. {% endif %} diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index cf62edd2..ab268b49 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -14,9 +14,11 @@ from mod_ci.models import BlockedUsers from mod_customized.models import CustomizedTest from mod_home.models import CCExtractorVersion, GeneralData -from mod_regression.models import (RegressionTest, RegressionTestOutput, +from mod_regression.models import (BaselineStatus, RegressionTest, + RegressionTestOutput, RegressionTestOutputFiles) -from mod_test.models import Test, TestPlatform, TestResultFile, TestType +from mod_test.models import (Test, TestPlatform, TestProgress, TestResult, + TestResultFile, TestStatus, TestType) from tests.base import (BaseTestCase, MockResponse, create_mock_db_query, create_mock_regression_test, empty_github_token, generate_git_api_header, generate_signature, @@ -68,6 +70,177 @@ def __init__(self): class TestControllers(BaseTestCase): """Test CI-related controllers.""" + def test_refresh_baseline_statuses_for_test_uses_full_result_status(self): + """Refresh baseline status using full regression results, not just exit codes.""" + from mod_ci.controllers import refresh_baseline_statuses_for_test + + test = Test(TestPlatform.linux, TestType.commit, 1, "master", "abc1234") + g.db.add(test) + g.db.commit() + + g.db.add_all([ + TestProgress(test.id, TestStatus.preparation, "prep"), + TestProgress(test.id, TestStatus.testing, "testing"), + TestProgress(test.id, TestStatus.completed, "done"), + TestResult(test.id, 1, 250, 1, 0), + TestResult(test.id, 2, 250, 0, 0), + TestResultFile(test.id, 1, 1, "sample_out1", "wrong"), + TestResultFile(test.id, 2, 2, "sample_out2"), + ]) + g.db.commit() + + regression_failed = RegressionTest.query.get(1) + regression_passed = RegressionTest.query.get(2) + + self.assertEqual(regression_failed.baseline_status, BaselineStatus.unknown) + self.assertEqual(regression_passed.baseline_status, BaselineStatus.unknown) + + refresh_baseline_statuses_for_test(test) + + self.assertEqual(regression_failed.baseline_status, BaselineStatus.never_worked) + self.assertEqual(regression_passed.baseline_status, BaselineStatus.established) + + def test_refresh_baseline_statuses_for_test_skips_pull_request_runs(self): + """Pull request runs must not rewrite the shared regression baseline.""" + from mod_ci.controllers import refresh_baseline_statuses_for_test + + test = Test.query.get(2) + regression_failed = RegressionTest.query.get(1) + regression_passed = RegressionTest.query.get(2) + passing_result_file = TestResultFile.query.filter( + TestResultFile.test_id == test.id, + TestResultFile.regression_test_id == regression_passed.id, + ).first() + passing_result_file.got = None + g.db.add(passing_result_file) + g.db.commit() + + refresh_baseline_statuses_for_test(test) + + self.assertEqual(regression_failed.baseline_status, BaselineStatus.unknown) + self.assertEqual(regression_passed.baseline_status, BaselineStatus.unknown) + + def test_comment_info_separates_never_worked_failures(self): + """Pre-existing never-worked tests should not be reported as PR breakage.""" + from mod_ci.controllers import get_info_for_pr_comment + from mod_customized.models import CustomizedTest + + test = Test.query.get(2) + regression_test = RegressionTest.query.get(1) + regression_test.baseline_status = BaselineStatus.never_worked + g.db.add(regression_test) + g.db.add(CustomizedTest(test.id, regression_test.id)) + g.db.commit() + + comment_info = get_info_for_pr_comment(test) + + self.assertIn(regression_test, comment_info.never_worked_tests) + self.assertEqual(comment_info.extra_failed_tests, []) + self.assertEqual(comment_info.common_failed_tests, []) + + def test_comment_info_tracks_never_worked_per_platform(self): + """A test that has only passed on Linux should still be never-worked on Windows.""" + from mod_ci.controllers import get_info_for_pr_comment + from mod_customized.models import CustomizedTest + + regression_test = RegressionTest.query.get(1) + regression_test.baseline_status = BaselineStatus.established + regression_test.last_passed_on_linux = 1 + regression_test.last_passed_on_windows = None + g.db.add(regression_test) + g.db.commit() + + test = Test(TestPlatform.windows, TestType.pull_request, 1, "pull_request", "windowssha", 1) + g.db.add(test) + g.db.commit() + + g.db.add(CustomizedTest(test.id, regression_test.id)) + g.db.add(TestResult(test.id, regression_test.id, 250, 1, 0)) + g.db.commit() + + comment_info = get_info_for_pr_comment(test) + + self.assertIn(regression_test, comment_info.never_worked_tests) + self.assertEqual(comment_info.extra_failed_tests, []) + self.assertEqual(comment_info.common_failed_tests, []) + + @mock.patch('mod_ci.controllers.refresh_baseline_statuses_for_test') + @mock.patch('mod_ci.controllers.retry_with_backoff') + @mock.patch('mod_ci.controllers.update_build_badge') + @mock.patch('mod_ci.controllers.update_status_on_github') + @mock.patch('mod_ci.controllers.delete_instance_with_tracking') + @mock.patch('mod_ci.controllers.get_compute_service_object') + @mock.patch('mod_ci.controllers.Github') + def test_progress_type_request_completed_refreshes_baseline_status( + self, + mock_github, + mock_get_compute_service_object, + mock_delete_instance_with_tracking, + mock_update_status, + mock_update_build_badge, + mock_retry_with_backoff, + mock_refresh_baseline_statuses, + ): + """Completed test reporting should refresh baseline state before downstream reporting.""" + test = Test(TestPlatform.linux, TestType.commit, 1, "master", "abc1234") + g.db.add(test) + g.db.commit() + + progress_entries = [ + TestProgress(test.id, TestStatus.preparation, "prep"), + TestProgress(test.id, TestStatus.testing, "testing"), + ] + g.db.add_all(progress_entries) + g.db.add(TestResult(test.id, 1, 250, 0, 0)) + g.db.add(TestResultFile(test.id, 1, 1, "sample_out1")) + g.db.commit() + + request = MagicMock() + request.form = { + 'status': 'completed', + 'message': 'done', + } + log = MagicMock() + mock_retry_with_backoff.side_effect = lambda func, **kwargs: func() + mock_delete_instance_with_tracking.return_value = {'name': 'delete-op'} + + progress_type_request(log, test, test.id, request) + + mock_refresh_baseline_statuses.assert_called_once_with(test) + + @mock.patch('mod_ci.controllers.refresh_baseline_statuses_for_test') + @mock.patch('mod_ci.controllers.retry_with_backoff') + @mock.patch('mod_ci.controllers.update_build_badge') + @mock.patch('mod_ci.controllers.update_status_on_github') + @mock.patch('mod_ci.controllers.delete_instance_with_tracking') + @mock.patch('mod_ci.controllers.get_compute_service_object') + @mock.patch('mod_ci.controllers.Github') + def test_progress_type_request_completed_does_not_refresh_baseline_for_pull_requests( + self, + mock_github, + mock_get_compute_service_object, + mock_delete_instance_with_tracking, + mock_update_status, + mock_update_build_badge, + mock_retry_with_backoff, + mock_refresh_baseline_statuses, + ): + """Pull request completion should not mutate shared baseline state.""" + test = Test.query.get(2) + + request = MagicMock() + request.form = { + 'status': 'completed', + 'message': 'done', + } + log = MagicMock() + mock_retry_with_backoff.side_effect = lambda func, **kwargs: func() + mock_delete_instance_with_tracking.return_value = {'name': 'delete-op'} + + progress_type_request(log, test, test.id, request) + + mock_refresh_baseline_statuses.assert_not_called() + def test_comment_info_handles_variant_files_correctly(self): """Test that allowed variants of output files are handled correctly in PR comments. @@ -394,14 +567,12 @@ def test_set_avg_time(self, mock_g, mock_gd, mock_int): self.assertEqual(mock_g.db.add.call_count, 0) mock_g.db.commit.assert_called_once() - @mock.patch('github.Github') + @mock.patch('mod_ci.controllers.Github') def test_comments_successfully_in_passed_pr_test(self, mock_github): """Check comments in passed PR test.""" - import mod_ci.controllers - reload(mod_ci.controllers) from github.IssueComment import IssueComment - from mod_ci.controllers import Status, comment_pr + from mod_ci.controllers import comment_pr from mod_test.models import Test pull_request = mock_github.return_value.get_repo.return_value.get_pull(number=1) pull_request.get_issue_comments.return_value = [MagicMock(IssueComment)] @@ -415,12 +586,10 @@ def test_comments_successfully_in_passed_pr_test(self, mock_github): if "passed" not in message: assert False, "Message not Correct" - @mock.patch('mod_test.controllers.get_test_results') - @mock.patch('github.Github') + @mock.patch('mod_ci.controllers.get_test_results') + @mock.patch('mod_ci.controllers.Github') def test_comments_successfuly_in_failed_pr_test(self, mock_github, mock_get_test_results): """Check comments in failed PR test.""" - import mod_ci.controllers - reload(mod_ci.controllers) from github.IssueComment import IssueComment from mod_ci.controllers import comment_pr diff --git a/tests/test_regression/test_baseline_status.py b/tests/test_regression/test_baseline_status.py new file mode 100644 index 00000000..f8ab478d --- /dev/null +++ b/tests/test_regression/test_baseline_status.py @@ -0,0 +1,157 @@ +"""Tests for the BaselineStatus feature on RegressionTest. + +Verifies the state-machine transitions in RegressionTest.update_baseline_status +and the is_regression property. +""" + +from flask import g + +from mod_regression.models import BaselineStatus, RegressionTest +from tests.base import BaseTestCase + + +class TestBaselineStatus(BaseTestCase): + """Tests for the BaselineStatus state machine.""" + + def _get_regression_test(self, test_id: int = 1) -> RegressionTest: + """Fetch a RegressionTest from the test database.""" + return RegressionTest.query.filter(RegressionTest.id == test_id).first() + + # ------------------------------------------------------------------ + # Initial state + # ------------------------------------------------------------------ + + def test_new_regression_test_has_unknown_baseline(self): + """A freshly created regression test must have baseline_status = unknown.""" + rt = self._get_regression_test() + self.assertEqual(rt.baseline_status, BaselineStatus.unknown) + + # ------------------------------------------------------------------ + # unknown -> established (first run passes) + # ------------------------------------------------------------------ + + def test_unknown_pass_transitions_to_established(self): + """unknown + pass -> established.""" + rt = self._get_regression_test() + self.assertEqual(rt.baseline_status, BaselineStatus.unknown) + + changed = rt.update_baseline_status(passed=True) + + self.assertTrue(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # unknown -> never_worked (first run fails) + # ------------------------------------------------------------------ + + def test_unknown_fail_transitions_to_never_worked(self): + """unknown + fail -> never_worked.""" + rt = self._get_regression_test() + self.assertEqual(rt.baseline_status, BaselineStatus.unknown) + + changed = rt.update_baseline_status(passed=False) + + self.assertTrue(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.never_worked) + + # ------------------------------------------------------------------ + # never_worked -> established (first pass after never working) + # ------------------------------------------------------------------ + + def test_never_worked_pass_transitions_to_established(self): + """never_worked + pass -> established.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) # unknown -> never_worked + self.assertEqual(rt.baseline_status, BaselineStatus.never_worked) + + changed = rt.update_baseline_status(passed=True) + + self.assertTrue(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # never_worked + fail -> never_worked (no change) + # ------------------------------------------------------------------ + + def test_never_worked_fail_stays_never_worked(self): + """never_worked + fail -> never_worked (no transition).""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) # unknown -> never_worked + + changed = rt.update_baseline_status(passed=False) + + self.assertFalse(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.never_worked) + + # ------------------------------------------------------------------ + # established + pass -> established (no change) + # ------------------------------------------------------------------ + + def test_established_pass_stays_established(self): + """established + pass -> established (no transition).""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) # unknown -> established + + changed = rt.update_baseline_status(passed=True) + + self.assertFalse(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # established + fail -> established (regression, not never_worked) + # ------------------------------------------------------------------ + + def test_established_fail_stays_established(self): + """established + fail -> established (it is a regression, not never_worked).""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) # unknown -> established + + changed = rt.update_baseline_status(passed=False) + + self.assertFalse(changed) + self.assertEqual(rt.baseline_status, BaselineStatus.established) + + # ------------------------------------------------------------------ + # is_regression property + # ------------------------------------------------------------------ + + def test_is_regression_false_when_unknown(self): + """A test with unknown baseline cannot be a regression.""" + rt = self._get_regression_test() + self.assertFalse(rt.is_regression) + + def test_is_regression_false_when_never_worked(self): + """A test that never worked is not a regression when it fails.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) + self.assertFalse(rt.is_regression) + + def test_is_regression_true_when_established(self): + """An established test that fails is a true regression.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) + self.assertTrue(rt.is_regression) + + # ------------------------------------------------------------------ + # Persistence: changes survive a database round-trip + # ------------------------------------------------------------------ + + def test_baseline_status_persists_to_database(self): + """Verify baseline_status is actually saved and reloaded from DB.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=False) # unknown -> never_worked + g.db.add(rt) + g.db.commit() + + reloaded = RegressionTest.query.filter(RegressionTest.id == 1).first() + self.assertEqual(reloaded.baseline_status, BaselineStatus.never_worked) + + def test_established_status_persists_to_database(self): + """Verify established status is saved and reloaded from DB.""" + rt = self._get_regression_test() + rt.update_baseline_status(passed=True) # unknown -> established + g.db.add(rt) + g.db.commit() + + reloaded = RegressionTest.query.filter(RegressionTest.id == 1).first() + self.assertEqual(reloaded.baseline_status, BaselineStatus.established)