(WIP) feat: scenarios based testing

.githooks/install.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Installs the repo's git hooks by pointing core.hooksPath at .githooks/
+set -euo pipefail
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+git -C "$REPO_ROOT" config core.hooksPath .githooks
+chmod +x "$REPO_ROOT"/.githooks/*
+echo "✓ Git hooks installed (.githooks/)"

.githooks/pre-commit

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────
+# pre-commit hook for foc-devnet
+#
+# Checks staged files before each commit:
+#   - Rust: cargo fmt, cargo clippy
+#   - Shell: shfmt, shellcheck, executable bit
+#
+# Install:  bash .githooks/install.sh
+# Skip once: git commit --no-verify
+#
+# Auto-fix mode (formats code and re-stages):
+#   FIX=1 git commit
+# ─────────────────────────────────────────────────────────────
+set -euo pipefail
+
+FIX="${FIX:-0}"
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+FAIL=0
+
+pass() { printf "${GREEN}✓${NC} %s\n" "$1"; }
+fail() { printf "${RED}✗${NC} %s\n" "$1"; FAIL=1; }
+skip() { printf "${YELLOW}⊘${NC} %s (skipped — tool not found)\n" "$1"; }
+fixed() { printf "${BLUE}⟳${NC} %s (auto-fixed & re-staged)\n" "$1"; }
+
+# Collect staged files
+STAGED=$(git diff --cached --name-only --diff-filter=ACM)
+
+# ── Rust checks ──────────────────────────────────────────────
+HAS_RS=$(echo "$STAGED" | grep -c '\.rs$' || true)
+if [[ $HAS_RS -gt 0 ]]; then
+  if command -v cargo &>/dev/null; then
+    if cargo fmt --all -- --check &>/dev/null; then
+      pass "cargo fmt"
+    elif [[ "$FIX" == "1" ]]; then
+      cargo fmt --all
+      git diff --name-only -- '*.rs' | xargs -r git add
+      fixed "cargo fmt"
+    else
+      fail "cargo fmt — run 'cargo fmt' or FIX=1 git commit"
+    fi
+
+    if cargo clippy --all-targets --all-features -- -D warnings &>/dev/null; then
+      pass "cargo clippy"
+    else
+      fail "cargo clippy — fix warnings before committing"
+    fi
+  else
+    skip "cargo (Rust checks)"
+  fi
+fi
+
+# ── Shell checks ─────────────────────────────────────────────
+STAGED_SH=$(echo "$STAGED" | grep '\.sh$' || true)
+if [[ -n "$STAGED_SH" ]]; then
+  # Executable bit
+  for f in $STAGED_SH; do
+    if [[ -f "$f" && ! -x "$f" ]]; then
+      if [[ "$FIX" == "1" ]]; then
+        chmod +x "$f"
+        git add "$f"
+        fixed "${f} +x"
+      else
+        fail "${f} is not executable — run 'chmod +x ${f}'"
+      fi
+    fi
+  done
+
+  # shfmt
+  if command -v shfmt &>/dev/null; then
+    if shfmt -d -i 2 -bn $STAGED_SH &>/dev/null; then
+      pass "shfmt"
+    elif [[ "$FIX" == "1" ]]; then
+      shfmt -w -i 2 -bn $STAGED_SH
+      echo "$STAGED_SH" | xargs -r git add
+      fixed "shfmt"
+    else
+      fail "shfmt — run 'shfmt -w -i 2 -bn <file>' or FIX=1 git commit"
+    fi
+  else
+    skip "shfmt"
+  fi
+
+  # shellcheck (no auto-fix available)
+  if command -v shellcheck &>/dev/null; then
+    if shellcheck -S warning $STAGED_SH &>/dev/null; then
+      pass "shellcheck"
+    else
+      fail "shellcheck — run 'shellcheck -S warning <file>' to see issues"
+    fi
+  else
+    skip "shellcheck"
+  fi
+fi
+
+# ── Result ───────────────────────────────────────────────────
+if [[ $FAIL -ne 0 ]]; then
+  echo ""
+  printf "${RED}Pre-commit checks failed.${NC} Fix issues above or skip with: git commit --no-verify\n"
+  exit 1
+fi

.github/workflows/ci.yml

@@ -6,6 +6,19 @@ on:
     branches: ['*']
   pull_request:
     branches: [main]
+  schedule:
+    # Nightly at 03:00 UTC
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+    inputs:
+      reporting:
+        description: 'Create GitHub issue with scenario report'
+        type: boolean
+        default: false
+      skip_report_on_pass:
+        description: 'Skip filing issue when all scenarios pass'
+        type: boolean
+        default: true
 
 jobs:
   fmt-clippy:
@@ -26,9 +39,38 @@ jobs:
     - name: Run clippy
       run: cargo clippy --all-targets --all-features -- -D warnings
 
+  shellcheck:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install shfmt
+      run: sudo snap install shfmt
+
+    # Ensure all .sh files under scenarios/ are executable
+    - name: Check executable permissions
+      run: |
+        bad=$(find scenarios -name '*.sh' ! -perm -u+x)
+        if [ -n "$bad" ]; then
+          echo "Missing +x on:" >&2; echo "$bad" >&2; exit 1
+        fi
+
+    # Lint with shellcheck
+    - name: shellcheck
+      run: find scenarios -name '*.sh' -exec shellcheck -S warning {} +
+
+    # Verify consistent formatting (indent=2, binary ops start of line)
+    - name: shfmt
+      run: shfmt -d -i 2 -bn scenarios/
+
   foc-start-test:
     runs-on: ["self-hosted", "linux", "x64", "16xlarge+gpu"]
-    timeout-minutes: 60
+    timeout-minutes: 100
+    permissions:
+      contents: read
+      issues: write
 
     steps:
     - uses: actions/checkout@v4
@@ -346,6 +388,26 @@ jobs:
         node check-balances.js "$DEVNET_INFO"
         echo "✓ All examples ran well"
 
+    # Run scenario tests against the live devnet
+    - name: "TEST: {Run scenario tests}"
+      if: steps.start_cluster.outcome == 'success'
+      env:
+        # Enable reporting for nightly schedule or when explicitly requested
+        REPORTING: ${{ github.event_name == 'schedule' || inputs.reporting == true }}
+        # By default, don't file an issue if everything passes
+        SKIP_REPORT_ON_PASS: ${{ inputs.skip_report_on_pass != false }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: bash scenarios/run.sh
+
+    # Upload scenario report as artifact
+    - name: "EXEC: {Upload scenario report}"
+      if: always() && steps.start_cluster.outcome == 'success'
+      uses: actions/upload-artifact@v4
+      with:
+        name: scenario-report
+        path: ~/.foc-devnet/state/latest/scenario_*.md
+        if-no-files-found: ignore
+
     # Clean shutdown
     - name: "EXEC: {Stop cluster}, independent"
       run: ./foc-devnet stop

README_ADVANCED.md

@@ -1296,3 +1296,58 @@ docker run --rm --network host \
   --broadcast
 ```
 
+## Scenario Tests
+
+Scenario tests are lightweight shell scripts that validate scenarios on the devnet after startup. They share a single running devnet and execute serially in a defined order.
+
+### Running scenarios
+
+```bash
+# Run all scenarios against a running devnet
+bash scenarios/run.sh
+
+# Run a single scenario
+bash scenarios/test_basic_balances.sh
+
+# Point at a specific devnet run
+DEVNET_INFO=~/.foc-devnet/state/<run-id>/devnet-info.json bash scenarios/run.sh
+```
+
+Reports are written to `~/.foc-devnet/state/latest/scenario_<timestamp>.md`.
+
+### Writing a new scenario
+
+1. Create `scenarios/test_<name>.sh`:
+
+```bash
+#!/usr/bin/env bash
+set -euo pipefail
+SCENARIO_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "${SCENARIO_DIR}/lib.sh"
+scenario_start "<name>"
+
+# Use helpers: jq_devnet, assert_eq, assert_gt, assert_not_empty, assert_ok
+RPC_URL=$(jq_devnet '.info.lotus.host_rpc_url')
+BALANCE=$(cast balance 0x... --rpc-url "$RPC_URL")
+assert_gt "$BALANCE" 0 "account has funds"
+
+scenario_end
+```
+
+2. Add `test_<name>` to the `SCENARIOS` array in `scenarios/order.sh`.
+3. `chmod +x scenarios/test_<name>.sh`
+
+### Available assertion helpers (from `lib.sh`)
+
+| Helper | Usage | Description |
+|--------|-------|-------------|
+| `assert_eq` | `assert_eq "$a" "$b" "msg"` | Equality check |
+| `assert_gt` | `assert_gt "$a" "$b" "msg"` | Integer greater-than (handles wei-scale) |
+| `assert_not_empty` | `assert_not_empty "$v" "msg"` | Value is non-empty |
+| `assert_ok` | `assert_ok cmd arg... "msg"` | Command exits 0 |
+| `jq_devnet` | `jq_devnet '.info.lotus.host_rpc_url'` | Query devnet-info.json |
+
+### CI integration
+
+Scenarios run automatically in CI after the devnet starts. On nightly runs (or manual dispatch with `reporting` enabled), failures automatically create a GitHub issue with a full report.
+

scenarios/lib.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────
+# lib.sh — Shared helpers for scenario tests.
+#
+# Sourced (not executed) by each test_*.sh script.
+# Provides: assertions, devnet-info access, and result tracking.
+#
+# ── Writing a new scenario ───────────────────────────────────
+#
+#   1. Create scenarios/test_<name>.sh with this skeleton:
+#
+#        #!/usr/bin/env bash
+#        set -euo pipefail
+#        SCENARIO_DIR="$(cd "$(dirname "$0")" && pwd)"
+#        source "${SCENARIO_DIR}/lib.sh"
+#        scenario_start "<name>"
+#
+#        # ... your checks using assert_*, jq_devnet, etc. ...
+#
+#        scenario_end
+#
+#   2. Add "test_<name>" to the SCENARIOS array in order.sh.
+#   3. chmod +x scenarios/test_<name>.sh
+#   4. Run:  bash scenarios/test_<name>.sh
+#
+# ── Available helpers ────────────────────────────────────────
+#   jq_devnet <filter>           — query devnet-info.json
+#   assert_eq  <a> <b> <msg>     — equality check
+#   assert_gt  <a> <b> <msg>     — integer greater-than
+#   assert_not_empty <v> <msg>   — value is non-empty
+#   assert_ok  <cmd...> <msg>    — command exits 0
+#   info / ok / fail             — logging
+# ─────────────────────────────────────────────────────────────
+# shellcheck disable=SC2034  # Variables here are used by scripts that source this file
+set -euo pipefail
+
+# ── Paths ────────────────────────────────────────────────────
+DEVNET_INFO="${DEVNET_INFO:-$HOME/.foc-devnet/state/latest/devnet-info.json}"
+SCENARIO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPORT_DIR="${REPORT_DIR:-$HOME/.foc-devnet/state/latest}"
+
+# Per-scenario counters (reset by scenario_start)
+_PASS=0
+_FAIL=0
+_SCENARIO_NAME=""
+
+# ── devnet-info helpers ──────────────────────────────────────
+
+# Shorthand: jq_devnet '.info.users[0].evm_addr'
+jq_devnet() { jq -r "$1" "$DEVNET_INFO"; }
+
+# ── Logging ──────────────────────────────────────────────────
+_log() { printf "[%s] %s\n" "$1" "$2"; }
+info() { _log "[INFO]" "$*"; }
+ok() {
+  _log "[ OK ]" "$*"
+  ((_PASS++)) || true
+}
+fail() {
+  _log "[FAIL]" "$*"
+  ((_FAIL++)) || true
+}
+
+# ── Assertions ───────────────────────────────────────────────
+
+# assert_eq <actual> <expected> <message>
+assert_eq() {
+  if [[ "$1" == "$2" ]]; then ok "$3"; else fail "$3 (got '$1', want '$2')"; fi
+}
+
+# assert_not_empty <value> <message>
+assert_not_empty() {
+  if [[ -n "$1" ]]; then ok "$2"; else fail "$2 (empty)"; fi
+}
+
+# assert_gt <actual_number> <threshold> <message>
+# Both arguments are treated as integers (wei-scale is fine).
+assert_gt() {
+  if python3 -c "import sys; sys.exit(0 if int('$1') > int('$2') else 1)" 2>/dev/null; then
+    ok "$3"
+  else
+    fail "$3 (got '$1', want > '$2')"
+  fi
+}
+
+# assert_ok <command ...> <message (last arg)>
+# Runs the command; passes if exit-code == 0.
+assert_ok() {
+  local msg="${*: -1}"
+  local cmd=("${@:1:$#-1}")
+  if "${cmd[@]}" >/dev/null 2>&1; then ok "$msg"; else fail "$msg"; fi
+}
+
+# ── Scenario lifecycle ───────────────────────────────────────
+
+scenario_start() {
+  _SCENARIO_NAME="$1"
+  _PASS=0
+  _FAIL=0
+  info "━━━ START: ${_SCENARIO_NAME} ━━━"
+}
+
+scenario_end() {
+  local total=$((_PASS + _FAIL))
+  local status="PASS"
+  [[ $_FAIL -gt 0 ]] && status="FAIL"
+  info "━━━ END: ${_SCENARIO_NAME}  ${_PASS}/${total} passed  [${status}] ━━━"
+  # Write machine-readable result line for the runner
+  mkdir -p "$REPORT_DIR"
+  echo "${status}|${_SCENARIO_NAME}|${_PASS}|${_FAIL}" >>"${REPORT_DIR}/results.csv"
+  [[ $_FAIL -eq 0 ]]
+}

scenarios/order.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────
+# order.sh — Declares the scenario execution order.
+#
+# Each entry is the basename of a script under scenarios/.
+# Scenarios share the same running devnet and execute serially
+# in the order listed here.
+# ─────────────────────────────────────────────────────────────
+
+# shellcheck disable=SC2034  # SCENARIOS is used by run.sh which sources this file
+SCENARIOS=(
+  test_containers
+  test_basic_balances
+)