test: Make tests more reliable in CI (#181)

# Summary
Some of the tests are flaky when run in Github actions. This PR makes
them more reliable to reduce the need for re-runs.

# Changes
* Adjust tuner integration test assertion.
* Adjust allowed repeats on websocket unit test.

Author: toby-coleman

tests/integration/test_tuner.py

@@ -225,8 +225,10 @@ async def test_process_parameter_tuning(config: dict, ray_ctx: None) -> None:
     result = tuner.result_grid
     # There must be no failed trials
     assert not [t for t in result if t.error]
-    # Correct optimimum must be found, i.e. a strong negative factor
-    assert best_result.config["process.default.parameter.factor"] < -1
+    # The optimization process is stochastic and with a small number of samples,
+    # it may not find the true optimum. We assert that it finds a negative value
+    # that is reasonably far from 0 to ensure the optimization is working correctly.
+    assert best_result.config["process.default.parameter.factor"] < -0.4
 
 
 @pytest.mark.tuner

tests/unit/test_websocket_reader_writer.py

@@ -35,7 +35,7 @@ async def connected_client() -> _t.AsyncGenerator[ClientConnection, None]:
             yield client
 
 
-@pytest.mark.flaky(reruns=3)
+@pytest.mark.flaky(reruns=6)  # Flaky on Github Actions
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "parse_json,initial_message,n_skip_messages",