From 55c0719e60b787657188d560a8d954dfccb396c5 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 5 May 2026 14:56:51 +0200 Subject: [PATCH 1/7] Attempt fix by pinning --- .github/workflows/cpp.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 51fb5ead5d75..f4943c419d42 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -363,6 +363,35 @@ jobs: with: msystem: ${{ matrix.msystem_upper }} update: true + - name: Pin MSYS2 packages + # Pin upstream packages whose latest versions break this job: + # - mingw-w64-x86_64-gcc 16.1.0 ships a libstdc++ regression that + # breaks std::shared_ptr/threading tests on MINGW64 (bad_weak_ptr, + # heap corruption 0xc0000374, segfaults, hangs). + # - mingw-w64-{x86_64,clang-x86_64}-aws-sdk-cpp 1.11.801 stopped + # sending the legacy Content-Md5 header on DeleteObjects, which the + # bundled MinIO test server (2024-09-13) still requires. + shell: msys2 {0} + run: | + set -ex + case "${{ matrix.msystem_lower }}" in + mingw64) + urls=( + "https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-libs-15.2.0-14-any.pkg.tar.zst" + "https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-15.2.0-14-any.pkg.tar.zst" + "https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" + ) + ignore="mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-aws-sdk-cpp" + ;; + clang64) + urls=( + "https://repo.msys2.org/mingw/clang64/mingw-w64-clang-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" + ) + ignore="mingw-w64-clang-x86_64-aws-sdk-cpp" + ;; + esac + pacman -U --noconfirm "${urls[@]}" + printf '\nIgnorePkg = %s\n' "$ignore" >> /etc/pacman.conf - name: Setup MSYS2 shell: msys2 {0} run: ci/scripts/msys2_setup.sh cpp From 6c527909d10693d461d299f45c193318de866a63 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 5 May 2026 16:12:10 +0200 Subject: [PATCH 2/7] Attempt fix 2 --- .github/workflows/cpp.yml | 46 ++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index f4943c419d42..aeacbba3d4f8 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -363,38 +363,50 @@ jobs: with: msystem: ${{ matrix.msystem_upper }} update: true + - name: Setup MSYS2 + shell: msys2 {0} + run: ci/scripts/msys2_setup.sh cpp - name: Pin MSYS2 packages - # Pin upstream packages whose latest versions break this job: - # - mingw-w64-x86_64-gcc 16.1.0 ships a libstdc++ regression that - # breaks std::shared_ptr/threading tests on MINGW64 (bad_weak_ptr, - # heap corruption 0xc0000374, segfaults, hangs). - # - mingw-w64-{x86_64,clang-x86_64}-aws-sdk-cpp 1.11.801 stopped - # sending the legacy Content-Md5 header on DeleteObjects, which the - # bundled MinIO test server (2024-09-13) still requires. + # Workaround for upstream regressions: + # - gcc 16.1.0 has a libstdc++ regression that breaks threading + # tests on MINGW64. We also pin C++ packages rebuilt against + # the new libstdc++ since they reference symbols missing from 15.2. + # - aws-sdk-cpp 1.11.801 drops Content-Md5 on DeleteObjects which + # our bundled MinIO (RELEASE.2024-09-13) still requires. shell: msys2 {0} run: | set -ex + base="https://repo.msys2.org/mingw" case "${{ matrix.msystem_lower }}" in mingw64) urls=( - "https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-libs-15.2.0-14-any.pkg.tar.zst" - "https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-15.2.0-14-any.pkg.tar.zst" - "https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-gcc-libs-15.2.0-14-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-gcc-15.2.0-14-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-aws-crt-cpp-0.38.4-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-boost-libs-1.91.0-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-boost-1.91.0-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-ccache-4.13.2-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-clang-libs-22.1.4-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-clang-22.1.4-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-cmake-4.3.2-2-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-icu-78.3-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-llvm-libs-22.1.4-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-llvm-tools-22.1.4-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-llvm-22.1.4-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-tbb-2022.3.0-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-thrift-0.22.0-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-zstd-1.5.7-1-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-gflags-2.2.2-7-any.pkg.tar.zst" + "$base/mingw64/mingw-w64-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" ) - ignore="mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-aws-sdk-cpp" ;; clang64) urls=( - "https://repo.msys2.org/mingw/clang64/mingw-w64-clang-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" + "$base/clang64/mingw-w64-clang-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" ) - ignore="mingw-w64-clang-x86_64-aws-sdk-cpp" ;; esac pacman -U --noconfirm "${urls[@]}" - printf '\nIgnorePkg = %s\n' "$ignore" >> /etc/pacman.conf - - name: Setup MSYS2 - shell: msys2 {0} - run: ci/scripts/msys2_setup.sh cpp - name: Cache ccache uses: actions/cache@v5 with: From 11765298fd37d8c892ee28a5fd7f458f08a43b00 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 5 May 2026 19:52:46 +0200 Subject: [PATCH 3/7] Update comments --- .github/workflows/cpp.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index aeacbba3d4f8..e4a3bdc7b8aa 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -367,12 +367,16 @@ jobs: shell: msys2 {0} run: ci/scripts/msys2_setup.sh cpp - name: Pin MSYS2 packages - # Workaround for upstream regressions: - # - gcc 16.1.0 has a libstdc++ regression that breaks threading - # tests on MINGW64. We also pin C++ packages rebuilt against - # the new libstdc++ since they reference symbols missing from 15.2. - # - aws-sdk-cpp 1.11.801 drops Content-Md5 on DeleteObjects which - # our bundled MinIO (RELEASE.2024-09-13) still requires. + # Temporary workarounds for upstream issues, both with concrete exit + # criteria. + # - MINGW64: gcc 16 makes the intermittent __emutls race in ThreadPool's + # `thread_local current_thread_pool_` (#49272) crash on every run. The + # list also pins C++ packages rebuilt at pkgrel 2 since they reference + # libstdc++ symbols missing from 15.2. Remove once #49462 lands. CLANG64 + # uses libc++ and is unaffected. + # - MINGW64 + CLANG64 jobs: aws-sdk-cpp 1.11.801 dropped Content-Md5 on + # DeleteObjects, which the bundled MinIO (RELEASE.2024-09-13) still + # requires. Remove once MinIO is bumped or the SDK keeps sending MD5. shell: msys2 {0} run: | set -ex @@ -427,6 +431,8 @@ jobs: # https://github.com/apache/arrow/issues/48593 ci/scripts/download_tz_database.sh - name: Download MinIO + # Pinned in tandem with aws-sdk-cpp 1.11.479 (see "Pin MSYS2 + # packages" step). Bump this version and remove the SDK pin together. shell: msys2 {0} run: | mkdir -p /usr/local/bin From 2c7f8c3baec3a17e2359654901c8ada49527a954 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 6 May 2026 13:01:32 +0200 Subject: [PATCH 4/7] Bump MinIO and drop SDK pin per raulcd review --- .github/workflows/cpp.yml | 72 ++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index e4a3bdc7b8aa..bc9d09e5151e 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -367,49 +367,36 @@ jobs: shell: msys2 {0} run: ci/scripts/msys2_setup.sh cpp - name: Pin MSYS2 packages - # Temporary workarounds for upstream issues, both with concrete exit - # criteria. - # - MINGW64: gcc 16 makes the intermittent __emutls race in ThreadPool's - # `thread_local current_thread_pool_` (#49272) crash on every run. The - # list also pins C++ packages rebuilt at pkgrel 2 since they reference - # libstdc++ symbols missing from 15.2. Remove once #49462 lands. CLANG64 - # uses libc++ and is unaffected. - # - MINGW64 + CLANG64 jobs: aws-sdk-cpp 1.11.801 dropped Content-Md5 on - # DeleteObjects, which the bundled MinIO (RELEASE.2024-09-13) still - # requires. Remove once MinIO is bumped or the SDK keeps sending MD5. + # Temporary workaround for #49272: gcc 16 makes the intermittent + # __emutls race in ThreadPool crash on every run. + # Also pins C++ packages whose latest MSYS2 build is against + # gcc-libs 16.1, reverting each to a build against gcc-libs 15.2 for + # ABI compat. Remove once #49462 is solved. + if: matrix.msystem_upper == 'MINGW64' shell: msys2 {0} run: | set -ex - base="https://repo.msys2.org/mingw" - case "${{ matrix.msystem_lower }}" in - mingw64) - urls=( - "$base/mingw64/mingw-w64-x86_64-gcc-libs-15.2.0-14-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-gcc-15.2.0-14-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-aws-crt-cpp-0.38.4-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-boost-libs-1.91.0-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-boost-1.91.0-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-ccache-4.13.2-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-clang-libs-22.1.4-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-clang-22.1.4-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-cmake-4.3.2-2-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-icu-78.3-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-llvm-libs-22.1.4-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-llvm-tools-22.1.4-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-llvm-22.1.4-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-tbb-2022.3.0-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-thrift-0.22.0-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-zstd-1.5.7-1-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-gflags-2.2.2-7-any.pkg.tar.zst" - "$base/mingw64/mingw-w64-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" - ) - ;; - clang64) - urls=( - "$base/clang64/mingw-w64-clang-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" - ) - ;; - esac + base="https://repo.msys2.org/mingw/mingw64" + urls=( + "$base/mingw-w64-x86_64-gcc-libs-15.2.0-14-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-gcc-15.2.0-14-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-aws-crt-cpp-0.38.4-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-boost-libs-1.91.0-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-boost-1.91.0-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-ccache-4.13.2-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-clang-libs-22.1.4-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-clang-22.1.4-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-cmake-4.3.2-2-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-icu-78.3-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-llvm-libs-22.1.4-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-llvm-tools-22.1.4-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-llvm-22.1.4-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-tbb-2022.3.0-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-thrift-0.22.0-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-zstd-1.5.7-1-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-gflags-2.2.2-7-any.pkg.tar.zst" + "$base/mingw-w64-x86_64-aws-sdk-cpp-1.11.479-1-any.pkg.tar.zst" + ) pacman -U --noconfirm "${urls[@]}" - name: Cache ccache uses: actions/cache@v5 @@ -431,14 +418,13 @@ jobs: # https://github.com/apache/arrow/issues/48593 ci/scripts/download_tz_database.sh - name: Download MinIO - # Pinned in tandem with aws-sdk-cpp 1.11.479 (see "Pin MSYS2 - # packages" step). Bump this version and remove the SDK pin together. + # Match the version pinned in ci/scripts/install_minio.sh. shell: msys2 {0} run: | mkdir -p /usr/local/bin wget \ --output-document /usr/local/bin/minio.exe \ - https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z + https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z chmod +x /usr/local/bin/minio.exe - name: Set up Python uses: actions/setup-python@v6 From fb4d7a038d3076e8401ddd307838fa97f507ba2f Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 6 May 2026 16:47:10 +0200 Subject: [PATCH 5/7] Verify removing the pins per @raulcd review --- .github/workflows/cpp.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index bc9d09e5151e..85b014b41a46 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -367,12 +367,7 @@ jobs: shell: msys2 {0} run: ci/scripts/msys2_setup.sh cpp - name: Pin MSYS2 packages - # Temporary workaround for #49272: gcc 16 makes the intermittent - # __emutls race in ThreadPool crash on every run. - # Also pins C++ packages whose latest MSYS2 build is against - # gcc-libs 16.1, reverting each to a build against gcc-libs 15.2 for - # ABI compat. Remove once #49462 is solved. - if: matrix.msystem_upper == 'MINGW64' + if: false shell: msys2 {0} run: | set -ex From 44931abf5060b5b4eaefbb0c87edb22587af9395 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 7 May 2026 08:17:52 +0200 Subject: [PATCH 6/7] Restore MSYS2 pins. #49462 covers arrow-json-test only --- .github/workflows/cpp.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 85b014b41a46..b2acdab3fd2a 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -367,7 +367,14 @@ jobs: shell: msys2 {0} run: ci/scripts/msys2_setup.sh cpp - name: Pin MSYS2 packages - if: false + # Temporary workaround for #49930: gcc 16 surfaces a cluster of 5 + # MINGW64 test failures. Pinning gcc-libs to 15.2 (and C++ packages + # rebuilt against gcc-libs 16.1, for ABI compatibility) avoids all + # of them. #49272/#49462 cover one (arrow-json-test); the other 4 + # (async-utility-test, threading-utility-test, dataset-writer-test + # `bad_weak_ptr`, dataset-file-test) need separate fixes — see #49930. + # Remove once all 5 pass on current upstream MSYS2 without these pins + if: matrix.msystem_upper == 'MINGW64' shell: msys2 {0} run: | set -ex From d0b3cc864421ccfd8064b8c628113b52d0c50645 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 7 May 2026 15:14:16 +0200 Subject: [PATCH 7/7] Re-trigger CI