From 07a4e814f3a81f19500392d2d963f189c997e335 Mon Sep 17 00:00:00 2001 From: "baogorek@gmail.com" Date: Wed, 25 Mar 2026 20:23:03 -0400 Subject: [PATCH 1/2] Use modal deploy + spawn to decouple pipeline from GitHub runner modal run --detach streams logs and dies when the runner timeout fires, killing the pipeline. Instead, deploy the app to Modal (making it persistent) and spawn run_pipeline as a fire-and-forget call. The GitHub Actions step finishes in under a minute while the pipeline runs for 10+ hours on Modal untethered. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pipeline.yaml | 34 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index cec5a748..62ee8ed6 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -40,19 +40,29 @@ jobs: - name: Install Modal run: pip install modal - - name: Launch pipeline on Modal + - name: Deploy and launch pipeline on Modal env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} run: | - ARGS="--action run --branch main" - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - ARGS="$ARGS --gpu ${{ inputs.gpu }}" - ARGS="$ARGS --epochs ${{ inputs.epochs }}" - ARGS="$ARGS --national-epochs ${{ inputs.national_epochs }}" - ARGS="$ARGS --num-workers ${{ inputs.num_workers }}" - if [ "${{ inputs.skip_national }}" = "true" ]; then - ARGS="$ARGS --skip-national" - fi - fi - modal run --detach modal_app/pipeline.py::main $ARGS + modal deploy modal_app/pipeline.py + + GPU="${{ inputs.gpu || 'T4' }}" + EPOCHS="${{ inputs.epochs || '1000' }}" + NATIONAL_EPOCHS="${{ inputs.national_epochs || '4000' }}" + NUM_WORKERS="${{ inputs.num_workers || '8' }}" + SKIP_NATIONAL="${{ inputs.skip_national || 'false' }}" + + python -c " + import modal + run_pipeline = modal.Function.from_name('policyengine-us-data-pipeline', 'run_pipeline') + fc = run_pipeline.spawn( + branch='main', + gpu='${GPU}', + epochs=int('${EPOCHS}'), + national_epochs=int('${NATIONAL_EPOCHS}'), + num_workers=int('${NUM_WORKERS}'), + skip_national='${SKIP_NATIONAL}' == 'true', + ) + print(f'Pipeline spawned. Monitor on the Modal dashboard.') + " From 194972eafb8199a0d810d103e559d4c532931ee1 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Thu, 26 Mar 2026 14:10:10 +0530 Subject: [PATCH 2/2] increasing timeout to ensure deployment - local test took ~22mins to deploy, should be subsequently faster but want to have some headroom --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 62ee8ed6..0c71a9d5 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -29,7 +29,7 @@ on: jobs: pipeline: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 30 steps: - uses: actions/checkout@v4