Merge branch 'bp/case-when' of github.com:vortex-data/vortex into bp/case-when

Author: palaska

.github/runs-on.yml

@@ -3,9 +3,9 @@ images:
     platform: "linux"
     arch: "x64"
     name: "vortex-ci-*"
-    owner: "375504701696"
+    owner: "245040174862"
   vortex-ci-arm64:
     platform: "linux"
     arch: "arm64"
     name: "vortex-ci-*"
-    owner: "375504701696"
+    owner: "245040174862"

.github/workflows/bench-pr.yml

@@ -94,7 +94,7 @@ jobs:
         if: github.event.pull_request.head.repo.fork == false
         uses: aws-actions/configure-aws-credentials@v5
         with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
           aws-region: us-east-1
 
       - name: Install uv
@@ -115,7 +115,7 @@ jobs:
             | jq -r '.workflow_runs[].head_sha' \
           )
 
-          python3 scripts/s3-download.py s3://vortex-benchmark-results-database/data.json.gz data.json.gz --no-sign-request
+          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz | grep $base_commit_sha > base.json
 
           echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md

.github/workflows/bench.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v5
         with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
           aws-region: us-east-1
       - name: Upload Commit Metadata
         shell: bash
@@ -29,7 +29,7 @@ jobs:
 
           sudo apt-get update && sudo apt-get install -y jq
           bash scripts/commit-json.sh > new-commit.json
-          bash scripts/cat-s3.sh vortex-benchmark-results-database commits.json new-commit.json
+          bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json
 
   bench:
     timeout-minutes: 120
@@ -94,13 +94,13 @@ jobs:
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v5
         with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
           aws-region: us-east-1
 
       - name: Upload Benchmark Results
         shell: bash
         run: |
-          bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz results.json
+          bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json
 
       - name: Alert incident.io
         if: failure()
@@ -137,7 +137,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H SF=1 on S3",
             "local_dir": "vortex-bench/data/tpch/1.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
+            "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "1.0"
           },
@@ -154,7 +154,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H SF=10 on S3",
             "local_dir": "vortex-bench/data/tpch/10.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
+            "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "10.0"
           },
@@ -184,7 +184,7 @@ jobs:
             "subcommand": "fineweb",
             "name": "FineWeb S3",
             "local_dir": "vortex-bench/data/fineweb",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
+            "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
           },
           {

.github/workflows/ci.yml

@@ -50,8 +50,8 @@ jobs:
     name: "Python (lint)"
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=2
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=python-lint
@@ -84,8 +84,8 @@ jobs:
     name: "Python (test)"
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=12
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=python-test
@@ -175,8 +175,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - family=m8azn
+      - cpu=2
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-docs
@@ -199,8 +199,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - family=m8azn
+      - cpu=${{ matrix.config.cpu }}
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=${{ matrix.config.name }}
@@ -212,13 +212,17 @@ jobs:
       matrix:
         config:
           - name: "all-features"
+            cpu: 4
             args: "--all-features --all-targets"
           - name: "default features"
+            cpu: 4
             args: "--all-targets"
           - name: "with tokio dispatcher"
+            cpu: 2
             # Only build the crates that have the tokio features, not re-building other crates with no-default-features
             args: "--no-default-features --features tokio --all-targets -p vortex -p vortex-io -p vortex-file -p vortex-layout"
           - name: "wasm32 with default features"
+            cpu: 4
             target: wasm32-unknown-unknown
             env:
               rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'"
@@ -249,8 +253,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - family=m8azn
+      - cpu=1
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-min-deps
@@ -271,8 +275,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=12
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-lint
@@ -316,8 +320,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=4
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-lint-no-default
@@ -342,8 +346,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=1
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=public-api
@@ -383,10 +387,10 @@ jobs:
       matrix:
         include:
           - suite: tests
-            cpu: 32
+            cpu: 24
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
+      - family=m8azn
       - cpu=${{ matrix.cpu }}
       - image=ubuntu24-full-x64
       - extras=s3-cache
@@ -443,8 +447,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=12
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-test-sanitizer
@@ -508,6 +512,7 @@ jobs:
       - family=g5+g4dn+g6
       - cpu=8
       - image=ubuntu24-gpu-x64
+      - spot=true
       - extras=s3-cache
       - tag=cuda-build
     steps:
@@ -535,6 +540,7 @@ jobs:
       - family=g5+g4dn+g6
       - cpu=8
       - image=ubuntu24-gpu-x64
+      - spot=true
       - extras=s3-cache
       - tag=cuda-lint
     steps:
@@ -563,6 +569,7 @@ jobs:
       - family=g5+g4dn+g6
       - cpu=8
       - image=ubuntu24-gpu-x64
+      - spot=true
       - extras=s3-cache
       - tag=cuda-tests
     steps:
@@ -606,6 +613,7 @@ jobs:
       - family=g5+g4dn+g6
       - cpu=8
       - image=ubuntu24-gpu-x64
+      - spot=true
       - extras=s3-cache
       - tag=cuda-test-sanitizer
     steps:
@@ -647,6 +655,7 @@ jobs:
       - family=g5+g4dn+g6
       - cpu=8
       - image=ubuntu24-gpu-x64
+      - spot=true
       - extras=s3-cache
       - tag=cuda-test-cudf
     steps:
@@ -682,7 +691,7 @@ jobs:
           - os: linux-arm64
             runner:
               - runs-on=${{ github.run_id }}
-              - family=m7g
+              - family=m8g
               - cpu=8
               - image=ubuntu24-full-arm64
               - extras=s3-cache
@@ -740,8 +749,8 @@ jobs:
     name: "Java"
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=16
+      - family=m8azn
+      - cpu=4
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=java
@@ -765,19 +774,20 @@ jobs:
     strategy:
       matrix:
         include:
-          - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" }
-          - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" }
-          - { shard: 3, name: "Main library", packages: "vortex" }
-          - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" }
-          - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
-          - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }
-          - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" }
-          - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" }
+          # - { shard: 1, cpu: 8, name: "Core foundation", packages: "vortex-buffer vortex-error" }
+          - { shard: 2, cpu: 16, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" }
+          # - { shard: 3, cpu: 16, name: "Main library", packages: "vortex" }
+          # - { shard: 4, cpu: 8, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" }
+          # - { shard: 5, cpu: 16, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
+          # - { shard: 6, cpu: 8, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }
+          # - { shard: 7, cpu: 8, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" }
+          # - { shard: 8, cpu: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" }
     name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})"
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=c7i.8xlarge
+      - family=m8i
+      - cpu=${{ matrix.cpu }}
       - extras=s3-cache
       - image=ubuntu24-full-x64
       - tag=bench-codspeed-${{ matrix.shard }}
@@ -827,8 +837,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - family=m8azn
+      - cpu=1
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=cxx-build
@@ -857,8 +867,8 @@ jobs:
     name: "SQL logic tests"
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - family=m8azn
+      - cpu=1
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=sql-logic-test
@@ -981,8 +991,8 @@ jobs:
     timeout-minutes: 120
     runs-on:
       - runs-on=${{ github.run_id }}
-      - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - family=m8azn
+      - cpu=1
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-publish-dry-run

.github/workflows/close-fixed-fuzzer-issues.yml

@@ -2,7 +2,7 @@ name: Close Fixed Fuzzer Issues
 
 on:
   schedule:
-    - cron: "0 0 * * 0"  # every Sunday at midnight UTC
+    - cron: "0 0 * * *"  # every night at midnight UTC
   workflow_dispatch:
     inputs:
       dry_run:

.github/workflows/nightly-bench.yml

@@ -46,7 +46,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H on S3",
             "local_dir": "vortex-bench/data/tpch/10.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
+            "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
             "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex",
             "scale_factor": "10.0"
           },
@@ -62,7 +62,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H on S3",
             "local_dir": "vortex-bench/data/tpch/100.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
+            "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
             "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
             "scale_factor": "100.0"
           },