[TRACIER - DO NO MERGE] feat: scenario tests for nightly CI runs

.githooks/pre-commit

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FIX="${FIX:-1}"
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+
+$REPO_ROOT/scripts/lint.sh

.github/workflows/ci.yml

@@ -3,12 +3,30 @@ name: CI
 
 on:
   push:
-    branches: ['*']
+    branches: ['**']
   pull_request:
-    branches: [main]
+    branches: ['**']
+  schedule:
+    # Nightly at 03:00 UTC
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+    inputs:
+      reporting:
+        description: 'Create GitHub issue with scenario report'
+        type: boolean
+        default: false
+      skip_report_on_pass:
+        description: 'Skip filing issue when all scenarios pass'
+        type: boolean
+        default: true
+
+env:
+  # When true, file a GitHub issue even when all scenarios pass.
+  # Flip this to 'false' once the suite is stable to only file on failures.
+  REPORT_ON_SUCCESS: 'true'
 
 jobs:
-  fmt-clippy:
+  lint:
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
@@ -20,15 +38,32 @@ jobs:
       with:
         components: rustfmt, clippy
 
-    - name: Check formatting
-      run: cargo fmt --all -- --check
-
-    - name: Run clippy
-      run: cargo clippy --all-targets --all-features -- -D warnings
-
-  foc-start-test:
+    - name: Setup Python tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pipx
+        pipx install black
+        pipx install ruff
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - name: Run linting (check mode)
+      run: FIX=0 ./scripts/lint.sh
+
+  foc-devnet-test:
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          - name: latesttag
+            init_flags: "--lotus latestTag --curio latestTag --filecoin-services latestTag"
+          - name: latestcommit
+            init_flags: "--lotus latestTag --curio gitbranch:pdpv0 --filecoin-services gitbranch:main"
     runs-on: ["self-hosted", "linux", "x64", "16xlarge+gpu"]
-    timeout-minutes: 60
+    timeout-minutes: 100
+    permissions:
+      contents: read
+      issues: write
 
     steps:
     - uses: actions/checkout@v4
@@ -91,7 +126,7 @@ jobs:
           ~/.cargo/git/db/
           target/
         key: ${{ runner.os }}-rust-build-${{ hashFiles('**/Cargo.lock') }}
-    
+
     # Copy binary and clean up Rust artifacts to save disk space
     - name: "EXEC: {Copy binary and clean cache}, DEP: {C-rust-cache}"
       run: |
@@ -151,21 +186,21 @@ jobs:
       if: steps.cache-docker-images.outputs.cache-hit == 'true'
       run: |
         rm -rf ~/.foc-devnet
-        ./foc-devnet init --no-docker-build
+        ./foc-devnet init --no-docker-build ${{ matrix.init_flags }}
 
     # If Docker images are not cached, do full init (downloads YugabyteDB and builds all images)
     - name: "EXEC: {Initialize without cache}, independent"
       if: steps.cache-docker-images.outputs.cache-hit != 'true'
       run: |
         rm -rf ~/.foc-devnet
-        ./foc-devnet init
+        ./foc-devnet init ${{ matrix.init_flags }}
 
-    # CACHE-DOCKER: Build Docker images if not cached
-    - name: "EXEC: {Build Docker images}, DEP: {C-docker-images-cache}"
+    # CACHE-DOCKER: Save Docker images as tarballs for caching
+    - name: "EXEC: {Save Docker images for cache}, DEP: {C-docker-images-cache}"
       if: steps.cache-docker-images.outputs.cache-hit != 'true'
       run: |-
         mkdir -p ~/.docker-images-cache
-        echo "Building Docker images for cache..."
+        echo "Saving Docker images for cache..."
         docker save foc-lotus -o ~/.docker-images-cache/foc-lotus.tar
         docker save foc-lotus-miner -o ~/.docker-images-cache/foc-lotus-miner.tar
         docker save foc-builder -o ~/.docker-images-cache/foc-builder.tar
@@ -189,7 +224,7 @@ jobs:
       uses: actions/cache/restore@v4
       with:
         path: ~/.foc-devnet/bin
-        key: ${{ runner.os }}-binaries-${{ steps.version-hashes.outputs.code-hash }}
+        key: ${{ runner.os }}-binaries-${{ matrix.name }}-${{ steps.version-hashes.outputs.code-hash }}
 
     - name: "EXEC: {Ensure permissions on binaries}, DEP: {C-build-artifacts-cache}"
       if: steps.cache-binaries.outputs.cache-hit == 'true'
@@ -202,9 +237,9 @@ jobs:
       uses: actions/cache/restore@v4
       with:
         path: ~/.foc-devnet/docker/volumes/cache/foc-builder
-        key: ${{ runner.os }}-foc-builder-cache-${{ hashFiles('docker/**') }}-${{ hashFiles('src/config.rs') }}
+        key: ${{ runner.os }}-foc-builder-cache-${{ matrix.name }}-${{ hashFiles('docker/**') }}-${{ hashFiles('src/config.rs') }}
         restore-keys: |
-          ${{ runner.os }}-foc-builder-cache-
+          ${{ runner.os }}-foc-builder-cache-${{ matrix.name }}-
 
     - name: "EXEC: {Ensure permissions}, DEP: {C-foc-builder-cache}"
       if: steps.cache-binaries.outputs.cache-hit != 'true' &&
@@ -230,15 +265,15 @@ jobs:
       uses: actions/cache/save@v4
       with:
         path: ~/.foc-devnet/docker/volumes/cache/foc-builder
-        key: ${{ runner.os }}-foc-builder-cache-${{ hashFiles('docker/**') }}-${{ hashFiles('src/config.rs') }}
+        key: ${{ runner.os }}-foc-builder-cache-${{ matrix.name }}-${{ hashFiles('docker/**') }}-${{ hashFiles('src/config.rs') }}
 
     # CACHE-BINARIES: Save built Lotus/Curio binaries for future runs
     - name: "CACHE_SAVE: {C-build-artifacts-cache}"
       if: steps.cache-binaries.outputs.cache-hit != 'true'
       uses: actions/cache/save@v4
       with:
         path: ~/.foc-devnet/bin
-        key: ${{ runner.os }}-binaries-${{ steps.version-hashes.outputs.code-hash }}
+        key: ${{ runner.os }}-binaries-${{ matrix.name }}-${{ steps.version-hashes.outputs.code-hash }}
 
     # Disk free-up
     - name: "EXEC: {Clean up Go modules}, DEP: {C-build-artifacts-cache}"
@@ -273,8 +308,9 @@ jobs:
       continue-on-error: true
       run: ./foc-devnet start --parallel
 
-    # On failure, collect and print Docker container logs for debugging
-    - name: "EXEC: {Collect Docker logs on failure}, independent"
+    # Collect and print Docker container logs for debugging (always runs for diagnostics)
+    - name: "EXEC: {Collect Docker logs}, independent"
+      if: always()
       run: |
         RUN_DIR="$HOME/.foc-devnet/state/latest"
 
@@ -311,9 +347,11 @@ jobs:
 
     # Verify cluster is running correctly
     - name: "EXEC: {Check cluster status}, independent"
+      if: always()
       run: ./foc-devnet status
 
     - name: "EXEC: {List foc-* containers}, independent"
+      if: always()
       run: |
         echo "Containers using foc-* images (running or exited):"
         docker ps -a --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'
@@ -334,6 +372,13 @@ jobs:
       with:
         node-version: '20'
 
+    # Setup pnpm (required by scenario tests)
+    - name: "EXEC: {Setup pnpm}, independent"
+      if: steps.start_cluster.outcome == 'success'
+      uses: pnpm/action-setup@v4
+      with:
+        version: latest
+
     # Validate schema using zod
     - name: "CHECK: {Validate devnet-info.json schema}"
       if: steps.start_cluster.outcome == 'success'
@@ -346,13 +391,131 @@ jobs:
         node check-balances.js "$DEVNET_INFO"
         echo "✓ All examples ran well"
 
-    # Clean shutdown
+    # Run scenario tests against the live devnet
+    - name: "TEST: {Run scenario tests}"
+      if: steps.start_cluster.outcome == 'success'
+      env:
+        # Enable reporting for nightly schedule or when explicitly requested
+        REPORTING: ${{ github.event_name == 'schedule' || inputs.reporting == true }}
+        # By default, don't file an issue if everything passes
+        SKIP_REPORT_ON_PASS: ${{ inputs.skip_report_on_pass != false }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: python3 scenarios/run.py
+
+    # Ensure scenario report exists even if tests didn't run (for issue reporting)
+    - name: "EXEC: {Ensure scenario report exists}"
+      if: always()
+      run: |
+        REPORT="$HOME/.foc-devnet/state/latest/scenario_report.md"
+        if [ ! -f "$REPORT" ]; then
+          mkdir -p "$(dirname "$REPORT")"
+          {
+            echo "# Scenario Test Report"
+            echo ""
+            echo "**No scenario tests were executed.**"
+            echo ""
+            echo "**Start cluster outcome**: ${{ steps.start_cluster.outcome }}"
+            echo ""
+            echo "## foc-devnet version"
+            echo '```'
+            ./foc-devnet version 2>&1 || echo "version command failed"
+            echo '```'
+          } > "$REPORT"
+        fi
+
+    # Upload scenario report as artifact (name includes strategy to avoid collisions in matrix)
+    - name: "EXEC: {Upload scenario report}"
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: scenario-report-${{ matrix.name }}
+        path: ~/.foc-devnet/state/latest/scenario_*.md
+        if-no-files-found: ignore
+
+    # Clean shutdown (always runs to avoid leaving containers behind)
     - name: "EXEC: {Stop cluster}, independent"
+      if: always()
       run: ./foc-devnet stop
 
     # Mark job as failed if the start step failed, but only after all steps
     - name: "CHECK: {Fail job if start failed}"
-      if: ${{ always() && steps.start_cluster.outcome == 'failure' }}
+      if: always() && steps.start_cluster.outcome == 'failure'
       run: |
         echo "Start cluster failed earlier; marking job as failed." >&2
         exit 1
+
+  issue-reporting:
+    name: Issue Reporting (${{ matrix.name }})
+    if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+    needs: [foc-devnet-test]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: latesttag
+            issue_label: scenarios-run-latesttag
+            issue_title: "FOC Devnet scenarios run report (latestTag)"
+          - name: latestcommit
+            issue_label: scenarios-run-latestcommit
+            issue_title: "FOC Devnet scenarios run report (latestCommit)"
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: "CHECK: {Determine if issue should be filed}"
+        id: should_file
+        run: |
+          FOC_DEVNET_TEST_STEP_RESULT="${{ needs.foc-devnet-test.result }}"
+          if [[ "$FOC_DEVNET_TEST_STEP_RESULT" == "success" ]]; then
+            TEST_PASSED="true"
+          else
+            TEST_PASSED="false"
+          fi
+          echo "passed=$TEST_PASSED" >> $GITHUB_OUTPUT
+          if [[ "$TEST_PASSED" == "true" && "$REPORT_ON_SUCCESS" != "true" ]]; then
+            echo "file=false" >> $GITHUB_OUTPUT
+            echo "Skipping issue: tests passed and REPORT_ON_SUCCESS is not 'true'"
+          else
+            echo "file=true" >> $GITHUB_OUTPUT
+            echo "Filing issue (${{ matrix.name }}): test result was $FOC_DEVNET_TEST_STEP_RESULT"
+          fi
+
+      - name: "EXEC: {Download scenario report for ${{ matrix.name }}}"
+        if: steps.should_file.outputs.file == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          name: scenario-report-${{ matrix.name }}
+          path: /tmp/scenario-report
+        continue-on-error: true
+
+      - name: "EXEC: {Read report content}"
+        if: steps.should_file.outputs.file == 'true'
+        id: report
+        run: |
+          CONTENT=""
+          for f in /tmp/scenario-report/*.md; do
+            if [ -f "$f" ]; then
+              CONTENT+=$(cat "$f")
+              CONTENT+=$'\n\n'
+            fi
+          done
+          if [[ -z "$CONTENT" ]]; then
+            CONTENT="No scenario report available for **${{ matrix.name }}** strategy."
+          fi
+          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
+          echo "content<<$EOF" >> $GITHUB_OUTPUT
+          echo "$CONTENT" >> $GITHUB_OUTPUT
+          echo "$EOF" >> $GITHUB_OUTPUT
+
+      - name: "EXEC: {Create or update issue}"
+        if: steps.should_file.outputs.file == 'true'
+        uses: ipdxco/create-or-update-issue@v1
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          title: ${{ matrix.issue_title }}
+          body: |
+            The **${{ matrix.name }}** scenarios run **${{ needs.foc-devnet-test.result == 'success' && 'passed ✅' || 'failed ❌' }}**.
+            See [the workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+
+            ${{ steps.report.outputs.content }}
+          label: ${{ matrix.issue_label }}

.gitignore

@@ -2,4 +2,6 @@ target/
 contracts/MockUSDFC/lib/
 contracts/MockUSDFC/broadcast/
 artifacts/
-.vscode/
+.vscode/
+*__pycache__/
+.githooks

Cargo.toml

@@ -28,6 +28,7 @@ base32 = "0.4"
 crc32fast = "1.3"
 bip32 = "0.5"
 rand = "0.8"
+semver = "1.0"
 bls-signatures = "0.15"
 names = { version = "0.14", default-features = false }
 shellexpand = "3"

README.md

@@ -31,6 +31,18 @@ For GitHub Actions, add this step before running foc-devnet:
 - run: echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts
 ```
 
+(Optional) Additionally, you may want to get linters for python scenarios, and install pre-commit hooks for development:
+```sh
+sudo apt install pipx
+pipx ensurepath
+
+# Install linting tools
+pipx install black
+
+# Install pre-commit hooks
+./scripts/install_precommit_hooks.sh
+```
+
 ### Step 1: Initialize
 
 ```bash