speed up integration tests

Author: joecorall

.github/workflows/lint-test.yml

@@ -75,8 +75,8 @@ jobs:
         run: docker compose logs --tail 100 nginx nginx2 traefik && docker compose down
         working-directory: ./ci
 
-  integration-test-backwards-compatibility:
-    needs: [integration-test]
+  integration-test:
+    needs: [integration-test-latest]
     permissions:
       contents: read
     runs-on: ubuntu-24.04

ci/docker-compose.yml

@@ -21,6 +21,7 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"
         healthcheck:
             test: curl -fs http://localhost/healthz | grep -q OK || exit 1
         volumes:

ci/test.go

@@ -52,26 +52,15 @@ func main() {
 	fmt.Printf("Making sure %d attempt(s) pass\n", rateLimit)
 	runParallelChecks(ips, rateLimit, "http://localhost")
 
+	time.Sleep(cp.StateSaveInterval + cp.StateSaveJitter + (1 * time.Second))
+	runCommand("jq", ".", "tmp/state.json")
+
 	fmt.Printf("Making sure attempt #%d causes a redirect to the challenge page\n", rateLimit+1)
 	ensureRedirect(ips, "http://localhost")
 
 	fmt.Println("\nTesting state sharing between nginx instances...")
-	fmt.Println("Waiting 2 seconds for state to save to disk...")
-	time.Sleep(cp.StateSaveInterval + (5 * time.Second))
 	testStateSharing(ips)
 
-	fmt.Println("Sleeping for 2m")
-	time.Sleep(125 * time.Second)
-	fmt.Println("Making sure one attempt passes after 2m window")
-	runParallelChecks(ips, 1, "http://localhost")
-	fmt.Println("All good 🚀")
-
-	// make sure the state has time to save
-	fmt.Println("Waiting for state to save")
-	runCommand("jq", ".", "tmp/state.json")
-	time.Sleep(cp.StateSaveInterval + (5 * time.Second))
-	runCommand("jq", ".", "tmp/state.json")
-
 	runCommand("docker", "container", "stats", "--no-stream")
 
 	// now restart the containers and make sure the previous state reloaded

internal/state/state.go

@@ -1,6 +1,9 @@
 package state
 
 import (
+	"encoding/json"
+	"log/slog"
+	"os"
 	"reflect"
 	"time"
 
@@ -255,3 +258,105 @@ func ReconcileState(fileState State, rateCache, botCache, verifiedCache *lru.Cac
 		}
 	}
 }
+
+// SaveStateToFile saves state to a file with locking and optional reconciliation.
+// When reconcile is true, it reads and merges existing file state before saving.
+// Returns timing metrics for debugging.
+func SaveStateToFile(
+	filePath string,
+	reconcile bool,
+	rateCache, botCache, verifiedCache *lru.Cache,
+	log *slog.Logger,
+) (lockMs, readMs, reconcileMs, marshalMs, writeMs, totalMs int64, err error) {
+	startTime := time.Now()
+
+	lock, err := NewFileLock(filePath + ".lock")
+	if err != nil {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+	defer lock.Close()
+
+	if err := lock.Lock(); err != nil {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+	lockDuration := time.Since(startTime)
+
+	var readDuration, reconcileDuration, marshalDuration, writeDuration time.Duration
+
+	// Reconcile with existing file state if enabled
+	if reconcile {
+		readStart := time.Now()
+		fileContent, readErr := os.ReadFile(filePath)
+		readDuration = time.Since(readStart)
+
+		if readErr == nil && len(fileContent) > 0 {
+			reconcileStart := time.Now()
+			var fileState State
+			if unmarshalErr := json.Unmarshal(fileContent, &fileState); unmarshalErr == nil {
+				log.Debug("Reconciling state before save", "fileBytes", len(fileContent))
+				ReconcileState(fileState, rateCache, botCache, verifiedCache)
+			}
+			reconcileDuration = time.Since(reconcileStart)
+		}
+	}
+
+	// Marshal current state
+	marshalStart := time.Now()
+	currentState := GetState(rateCache.Items(), botCache.Items(), verifiedCache.Items())
+	jsonData, err := json.Marshal(currentState)
+	marshalDuration = time.Since(marshalStart)
+
+	if err != nil {
+		return lockDuration.Milliseconds(), readDuration.Milliseconds(),
+			reconcileDuration.Milliseconds(), marshalDuration.Milliseconds(),
+			0, 0, err
+	}
+
+	// Write to disk
+	writeStart := time.Now()
+	err = os.WriteFile(filePath, jsonData, 0644)
+	writeDuration = time.Since(writeStart)
+
+	if err != nil {
+		return lockDuration.Milliseconds(), readDuration.Milliseconds(),
+			reconcileDuration.Milliseconds(), marshalDuration.Milliseconds(),
+			writeDuration.Milliseconds(), 0, err
+	}
+
+	totalDuration := time.Since(startTime)
+	return lockDuration.Milliseconds(), readDuration.Milliseconds(),
+		reconcileDuration.Milliseconds(), marshalDuration.Milliseconds(),
+		writeDuration.Milliseconds(), totalDuration.Milliseconds(), nil
+}
+
+// LoadStateFromFile loads state from a file with locking.
+func LoadStateFromFile(
+	filePath string,
+	rateCache, botCache, verifiedCache *lru.Cache,
+) error {
+	lock, err := NewFileLock(filePath + ".lock")
+	if err != nil {
+		return err
+	}
+	defer lock.Close()
+
+	if err := lock.Lock(); err != nil {
+		return err
+	}
+
+	fileContent, err := os.ReadFile(filePath)
+	if err != nil || len(fileContent) == 0 {
+		return err
+	}
+
+	var loadedState State
+	err = json.Unmarshal(fileContent, &loadedState)
+	if err != nil {
+		return err
+	}
+
+	// Use SetState which properly handles expiration times
+	SetState(loadedState, rateCache, botCache, verifiedCache)
+
+	return nil
+}