Improve the integration tests

- separate the integration tests into smaller scripts
- add a new integration test

Author: elisescu

.github/workflows/integration_tests.yml

@@ -88,17 +88,7 @@ jobs:
         if: steps.cache-playwright.outputs.cache-hit != 'true'
         run: poetry run python -m playwright install chromium --with-deps
       - name: Restore the min DB for testing
-        run: |
-          wget https://github.com/Metaculus/metaculus/releases/download/v0.0.1-alpha/test_metaculus.sql.zip
-          unzip test_metaculus.sql.zip
-          psql $DATABASE_URL -c "CREATE EXTENSION vector;"
-          pg_restore --dbname=$DATABASE_URL \
-            --clean \
-            --no-privileges \
-            --no-owner \
-            --if-exists \
-            --schema=public \
-            test_metaculus.sql
+        run: scripts/tests/setup_test_db.sh "$DATABASE_URL"
       - name: "Install Node"
         uses: actions/setup-node@v4
         with:
@@ -118,7 +108,6 @@ jobs:
         run: cd front_end && npm run build
       - name: Run the integration tests
         run: |
-          poetry run ./manage.py migrate
           PLAYWRIGHT_BROWSERS_PATH=${PLAYWRIGHT_BROWSERS_PATH} scripts/tests/run_integration_tests.sh
       - name: Create trace.zip
         if: ${{ !cancelled() }}

scripts/tests/run_all_services.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -x
+set -e
+set -o pipefail
+
+cleanup() {
+    for process in "next-server" "gunicorn" "dramatiq"; do
+        PID=$(ps aux | awk -v process=$process '$0 ~ process && $0 !~ /awk/ {print $2}')
+        kill -s SIGTERM $PID || true
+    done
+}
+
+trap cleanup EXIT
+trap "exit 0" INT TERM
+trap "exit 1" ERR
+
+export DATABASE_URL=postgres://postgres:postgres@localhost:5432/test_metaculus
+
+# Start the frontend and backend processes in the background,
+# and use sed to add a prefix to each of their stdout and stderr printed lines
+poetry run  ./manage.py collectstatic --noinput
+(poetry run gunicorn metaculus_web.wsgi:application --log-level=debug --bind 0.0.0.0:8000 2>&1 | sed 's/^/[Backend]: /') &
+(poetry run ./manage.py rundramatiq --processes 1 --threads 1 2>&1 | sed 's/^/[Dramatiq]: /') &
+(npm run --prefix front_end start 2>&1 | sed 's/^/[Frontend]: /') &
+
+wait

scripts/tests/run_integration_tests.sh

@@ -3,28 +3,21 @@ set -x
 set -e
 set -o pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"$SCRIPT_DIR/run_all_services.sh" &
+SERVICE_PID=$!
+
 cleanup() {
-    for process in "next-server" "gunicorn" "dramatiq"; do
-        PID=$(ps aux | awk -v process=$process '$0 ~ process && $0 !~ /awk/ {print $2}')
-        kill -s SIGTERM $PID || true
-    done
+  exit_code=$?
+  if [ -n "${SERVICE_PID:-}" ]; then
+    kill $SERVICE_PID 2>/dev/null || true
+    wait $SERVICE_PID 2>/dev/null || true
+  fi
+  exit $exit_code
 }
-
 trap cleanup EXIT
-trap "exit" INT TERM ERR
-
-export DATABASE_URL=postgres://postgres:postgres@localhost:5432/test_metaculus
-
-# Start the frontend and backend processes in the background,
-# and use sed to add a prefix to each of their stdout and stderr printed lines
-poetry run  ./manage.py collectstatic --noinput
-(poetry run gunicorn metaculus_web.wsgi:application --log-level=debug --bind 0.0.0.0:8000 2>&1 | sed 's/^/[Backend]: /') &
-(poetry run ./manage.py rundramatiq --processes 1 --threads 1 2>&1 | sed 's/^/[Dramatiq]: /') &
-(npm run --prefix front_end start 2>&1 | sed 's/^/[Frontend]: /') &
 
 npx wait-on http://localhost:8000/api/healthcheck/ --timeout 30000
 npx wait-on http://localhost:3000 --timeout 30000
 
-# Run pytest without Django plugins or the conf test, as the global DB setup/parmas
-# interfere with running the backend in prod "mode"
 poetry run pytest -s -p no:django -c - --noconftest --log-cli-level=INFO tests/integration/*.py | sed 's/^/[Tests] /'

scripts/tests/setup_test_db.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+export DATABASE_URL="${1:-postgres://postgres:postgres@localhost:5432/test_metaculus}"
+DB_NAME="${DATABASE_URL##*/}"
+ADMIN_DB_URL="${DATABASE_URL%/*}/postgres"
+
+rm -f test_metaculus.sql.zip
+wget https://github.com/Metaculus/metaculus/releases/download/v0.0.1-alpha/test_metaculus.sql.zip -O test_metaculus.sql.zip
+unzip test_metaculus.sql.zip
+
+psql "$ADMIN_DB_URL" -c "DROP DATABASE IF EXISTS $DB_NAME;"
+psql "$ADMIN_DB_URL" -c "CREATE DATABASE $DB_NAME;"
+psql "$DATABASE_URL" -c "CREATE EXTENSION IF NOT EXISTS vector;"
+pg_restore --dbname="$DATABASE_URL" \
+  --clean \
+  --no-privileges \
+  --no-owner \
+  --if-exists \
+  --schema=public \
+  test_metaculus.sql
+
+poetry run ./manage.py migrate
+
+# Make a binary question open for prediction so integration tests can use it
+psql "$DATABASE_URL" <<'SQL'
+WITH target AS (
+    SELECT q.id AS question_id, p.id AS post_id
+    FROM questions_question q
+    JOIN posts_post p ON p.question_id = q.id
+    WHERE q.type = 'binary'
+      AND q.resolution IS NULL
+    ORDER BY q.id
+    LIMIT 1
+),
+update_question AS (
+    UPDATE questions_question
+    SET open_time = NOW() - INTERVAL '30 days',
+        scheduled_close_time = NOW() + INTERVAL '365 days',
+        scheduled_resolve_time = NOW() + INTERVAL '395 days',
+        actual_close_time = NULL,
+        actual_resolve_time = NULL,
+        resolution = NULL
+    FROM target
+    WHERE questions_question.id = target.question_id
+)
+UPDATE posts_post
+SET published_at = NOW() - INTERVAL '30 days',
+    open_time = NOW() - INTERVAL '30 days'
+FROM target
+WHERE posts_post.id = target.post_id;
+SQL

tests/integration/test_simple.py

@@ -1,4 +1,6 @@
+import re
 import time
+from typing import Any, cast
 
 from playwright.sync_api import expect, sync_playwright
 
@@ -12,7 +14,7 @@ def setup_all(cls):
     cls.context = cls.browser.new_context()
     cls.context.tracing.start(screenshots=True, snapshots=True, sources=True)
 
-    expect.set_options(timeout=2)
+    expect.set_options(timeout=200)
 
 
 def teardown_all(cls):
@@ -21,7 +23,35 @@ def teardown_all(cls):
     cls.playwright.stop()
 
 
+def drag_slider(page, offset_px=80):
+    """Drag the slider handle by offset_px (positive = right, negative = left)."""
+    slider = page.get_by_role("slider")
+    box = slider.bounding_box()
+    assert box, "Slider handle should have bounding box"
+    center_x = box["x"] + box["width"] / 2
+    center_y = box["y"] + box["height"] / 2
+    page.mouse.move(center_x, center_y)
+    page.mouse.down()
+    page.mouse.move(center_x + offset_px, center_y)
+    page.mouse.up()
+    time.sleep(0.3)
+
+
+def login(page, base_url="http://localhost:3000/"):
+    """Log in with username_1 / Test1234 on the given page."""
+    page.goto(base_url)
+    page.get_by_role("button", name="Log in").click()
+    page.get_by_placeholder("username or email").fill("username_1")
+    page.get_by_placeholder("password").fill("Test1234")
+    page.get_by_role("button", name="Log in", exact=True).last.click()
+    time.sleep(1)
+
+
 class TestSimpleFunctionality:
+    playwright = None
+    browser = None
+    context = None
+
     @classmethod
     def setup_class(cls):
         setup_all(cls)
@@ -30,23 +60,68 @@ def setup_class(cls):
     def teardown_class(cls):
         teardown_all(cls)
 
+    def teardown_method(self, method):
+        context = cast(Any, self.__class__.context)
+        if context is not None:
+            for page in context.pages:
+                page.close()
+            context.clear_cookies()
+
     @classmethod
     def test_login(cls):
-        context = cls.context
+        context = cast(Any, cls.context)
+        assert context is not None
         page = context.new_page()
         page.on("console", lambda msg: print("[Browser console] ", msg.text))
 
-        url = "http://localhost:3000/"
-        page.goto(url)
+        login(page)
+
+        page.get_by_role("banner").get_by_role("link", name="Questions").click()
+        page.get_by_role("button", name="Filter").click()
+        page.get_by_role("button", name="Binary").click()
+        page.get_by_role("button", name="Open").click()
+        page.get_by_role("button", name="Done").click()
 
-        page.get_by_role("button", name="Log in").click()
+    @classmethod
+    def test_create_binary_forecast_via_ui(cls):
+        context = cast(Any, cls.context)
+        assert context is not None
 
-        page.get_by_placeholder("username or email").fill("username_1")
-        page.get_by_placeholder("password").fill("Test1234")
-        page.get_by_role("button", name="Log in", exact=True).last.click()
-        time.sleep(1)
+        page = context.new_page()
+        page.on("console", lambda msg: print("[Browser console] ", msg.text))
+
+        login(page)
         page.get_by_role("banner").get_by_role("link", name="Questions").click()
         page.get_by_role("button", name="Filter").click()
         page.get_by_role("button", name="Binary").click()
         page.get_by_role("button", name="Open").click()
+        time.sleep(1)
         page.get_by_role("button", name="Done").click()
+        time.sleep(1)
+        links = page.locator("a[href^='/questions/']")
+        for i in range(links.count()):
+            href = links.nth(i).get_attribute("href") or ""
+            if re.match(r"/questions/\d+/", href):
+                links.nth(i).click()
+                break
+
+        time.sleep(1)
+
+        drag_slider(page, offset_px=80)
+
+        slider = page.get_by_role("slider")
+        value_div = slider.locator("div")
+        predicted_value = value_div.inner_text()
+
+        predict_button = page.get_by_role(
+            "button", name=re.compile(r"(Predict|Save changes|Reaffirm)", re.IGNORECASE)
+        ).first
+        predict_button.click()
+
+        time.sleep(1)
+        page.reload()
+        time.sleep(1)
+
+        slider_after = page.get_by_role("slider")
+        value_div_after = slider_after.locator("div")
+        expect(value_div_after).to_have_text(predicted_value)