From 2dd0f47442634f5c0e2dcc71fd9c5438cfc3fda4 Mon Sep 17 00:00:00 2001 From: Suguru Inatomi Date: Tue, 24 Feb 2026 10:55:09 +0900 Subject: [PATCH 1/2] fix: prevent Bazel OOM crash on CI runners - Add resource limits to all CI workflows' bazelrc configuration: - Limit RAM to 6GB (GitHub Actions ubuntu-latest has 7GB) - Limit CPUs to 2 - Limit parallel jobs to 4 - Reduce JVM heap to 2GB max - Discard analysis cache after build to reduce memory pressure - Add build output verification in adev.ts to detect silent failures when Bazel server is killed mid-build --- .github/workflows/adev-preview-build.yml | 18 ++++++++---------- .github/workflows/adev-production-deploy.yml | 18 ++++++++---------- .github/workflows/ci.yml | 9 +++++++++ tools/lib/adev.ts | 11 ++++++++++- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/.github/workflows/adev-preview-build.yml b/.github/workflows/adev-preview-build.yml index 89be429c88..d3bd0e31fa 100644 --- a/.github/workflows/adev-preview-build.yml +++ b/.github/workflows/adev-preview-build.yml @@ -35,16 +35,14 @@ jobs: disk-cache: ${{ github.workflow }} repository-cache: true bazelrc: | - # Print all the options that apply to the build. - # This helps us diagnose which options override others - # (e.g. /etc/bazel.bazelrc vs. tools/bazel.rc) - build --announce_rc - - # More details on failures - build --verbose_failures=true - - # CI supports colors but Bazel does not detect it. - common --color=yes + # Limit resources for CI runners (ubuntu-latest: 7GB RAM, 2 CPUs) + build --local_ram_resources=6144 + build --local_cpu_resources=2 + build --jobs=4 + startup --host_jvm_args=-Xms512m + startup --host_jvm_args=-Xmx2g + build --discard_analysis_cache + build --nokeep_state_after_build - run: pnpm install --frozen-lockfile - run: pnpm run build - uses: angular/dev-infra/github-actions/previews/pack-and-upload-artifact@0512a5b9381ccff00c278d7b4b6ee38e5c09654d diff --git a/.github/workflows/adev-production-deploy.yml b/.github/workflows/adev-production-deploy.yml index f0606fd7bd..90064f8ebe 100644 --- a/.github/workflows/adev-production-deploy.yml +++ b/.github/workflows/adev-production-deploy.yml @@ -27,16 +27,14 @@ jobs: disk-cache: ${{ github.workflow }} repository-cache: true bazelrc: | - # Print all the options that apply to the build. - # This helps us diagnose which options override others - # (e.g. /etc/bazel.bazelrc vs. tools/bazel.rc) - build --announce_rc - - # More details on failures - build --verbose_failures=true - - # CI supports colors but Bazel does not detect it. - common --color=yes + # Limit resources for CI runners (ubuntu-latest: 7GB RAM, 2 CPUs) + build --local_ram_resources=6144 + build --local_cpu_resources=2 + build --jobs=4 + startup --host_jvm_args=-Xms512m + startup --host_jvm_args=-Xmx2g + build --discard_analysis_cache + build --nokeep_state_after_build - run: pnpm install --frozen-lockfile - run: pnpm run build - name: Deploy to Firebase Hosting diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78c661a4e5..418093fdc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,15 @@ jobs: bazelisk-cache: true disk-cache: ${{ github.workflow }} repository-cache: true + bazelrc: | + # Limit resources for CI runners (ubuntu-latest: 7GB RAM, 2 CPUs) + build --local_ram_resources=6144 + build --local_cpu_resources=2 + build --jobs=4 + startup --host_jvm_args=-Xms512m + startup --host_jvm_args=-Xmx2g + build --discard_analysis_cache + build --nokeep_state_after_build - run: pnpm install - run: pnpm run build # build-windows: diff --git a/tools/lib/adev.ts b/tools/lib/adev.ts index 78357c8fba..a443fe3fba 100644 --- a/tools/lib/adev.ts +++ b/tools/lib/adev.ts @@ -1,6 +1,7 @@ import { consola } from 'consola'; import { $ } from 'execa'; -import { buildDir } from './workspace'; +import { existsSync } from 'node:fs'; +import { buildDir, buildOutputDir } from './workspace'; const $$ = $({ stdin: 'inherit', @@ -13,6 +14,14 @@ export async function buildAdev() { const sh = $$({ cwd: buildDir }); await sh`pnpm install --frozen-lockfile`; await sh`pnpm bazel build //adev:build.production --config=release`; + + // Verify build output exists - Bazel may exit with code 0 even when interrupted + if (!existsSync(buildOutputDir)) { + throw new Error( + `Build output directory not found: ${buildOutputDir}\n` + + 'Bazel build may have been interrupted or failed silently.' + ); + } } export function serveAdev() { From 22316dc98539e122085db37a7b7814680a262234 Mon Sep 17 00:00:00 2001 From: Suguru Inatomi Date: Tue, 24 Feb 2026 12:45:08 +0900 Subject: [PATCH 2/2] fix: limit env scope to build step only - Move _JAVA_OPTIONS and NODE_OPTIONS to build step env - Remove startup options from bazelrc (use _JAVA_OPTIONS instead) - Reduce local_ram_resources to 4096MB - Reduce jobs to 2 --- .github/workflows/adev-preview-build.yml | 12 +++++++----- .github/workflows/adev-production-deploy.yml | 12 +++++++----- .github/workflows/ci.yml | 12 +++++++----- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/adev-preview-build.yml b/.github/workflows/adev-preview-build.yml index d3bd0e31fa..daa1321f33 100644 --- a/.github/workflows/adev-preview-build.yml +++ b/.github/workflows/adev-preview-build.yml @@ -36,15 +36,17 @@ jobs: repository-cache: true bazelrc: | # Limit resources for CI runners (ubuntu-latest: 7GB RAM, 2 CPUs) - build --local_ram_resources=6144 + build --local_ram_resources=4096 build --local_cpu_resources=2 - build --jobs=4 - startup --host_jvm_args=-Xms512m - startup --host_jvm_args=-Xmx2g + build --jobs=2 build --discard_analysis_cache build --nokeep_state_after_build - run: pnpm install --frozen-lockfile - - run: pnpm run build + - name: Build + run: pnpm run build + env: + _JAVA_OPTIONS: -Xms512m -Xmx2g + NODE_OPTIONS: --max-old-space-size=4096 - uses: angular/dev-infra/github-actions/previews/pack-and-upload-artifact@0512a5b9381ccff00c278d7b4b6ee38e5c09654d with: workflow-artifact-name: 'adev-preview' diff --git a/.github/workflows/adev-production-deploy.yml b/.github/workflows/adev-production-deploy.yml index 90064f8ebe..0323d43281 100644 --- a/.github/workflows/adev-production-deploy.yml +++ b/.github/workflows/adev-production-deploy.yml @@ -28,15 +28,17 @@ jobs: repository-cache: true bazelrc: | # Limit resources for CI runners (ubuntu-latest: 7GB RAM, 2 CPUs) - build --local_ram_resources=6144 + build --local_ram_resources=4096 build --local_cpu_resources=2 - build --jobs=4 - startup --host_jvm_args=-Xms512m - startup --host_jvm_args=-Xmx2g + build --jobs=2 build --discard_analysis_cache build --nokeep_state_after_build - run: pnpm install --frozen-lockfile - - run: pnpm run build + - name: Build + run: pnpm run build + env: + _JAVA_OPTIONS: -Xms512m -Xmx2g + NODE_OPTIONS: --max-old-space-size=4096 - name: Deploy to Firebase Hosting uses: FirebaseExtended/action-hosting-deploy@0cbcac4740c2bfb00d632f0b863b57713124eb5a # v0.9.0 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 418093fdc7..a77c7e5abc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,15 +40,17 @@ jobs: repository-cache: true bazelrc: | # Limit resources for CI runners (ubuntu-latest: 7GB RAM, 2 CPUs) - build --local_ram_resources=6144 + build --local_ram_resources=4096 build --local_cpu_resources=2 - build --jobs=4 - startup --host_jvm_args=-Xms512m - startup --host_jvm_args=-Xmx2g + build --jobs=2 build --discard_analysis_cache build --nokeep_state_after_build - run: pnpm install - - run: pnpm run build + - name: Build + run: pnpm run build + env: + _JAVA_OPTIONS: -Xms512m -Xmx2g + NODE_OPTIONS: --max-old-space-size=4096 # build-windows: # runs-on: windows-latest # steps: