[SUPERSEDED - use joblib PR] CI fix macOS FlexiBLAS build

.github/workflows/test.yml

@@ -221,6 +221,15 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Restore ccache
+        if: matrix.INSTALL_BLAS == 'flexiblas'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ccache
+          key: ccache-flexiblas-${{ matrix.os }}-${{ hashFiles('continuous_integration/install_flexiblas.sh') }}
+          restore-keys: |
+            ccache-flexiblas-${{ matrix.os }}-
+
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v3
         with:

continuous_integration/check_no_test_skipped.py

@@ -36,7 +36,11 @@
 # List of tests that we don't want to fail the CI if they are skipped in
 # every job. This is useful for tests that depend on specific versions of
 # numpy or scipy and we don't want to pin old versions of these libraries.
-SAFE_SKIPPED_TESTS = ["test_multiple_shipped_openblas"]
+SAFE_SKIPPED_TESTS = [
+    "test_multiple_shipped_openblas",
+    "test_threadpool_limits_by_prefix[1-libblas]",
+    "test_threadpool_limits_by_prefix[3-libblas]",
+]
 
 fail = False
 for test, skipped in always_skipped.items():

continuous_integration/install.sh

@@ -106,7 +106,7 @@ elif [[ "$INSTALL_BLAS" == "blis" ]]; then
     source ./continuous_integration/install_blis.sh
 
 elif [[ "$INSTALL_BLAS" == "flexiblas" ]]; then
-    TO_INSTALL="cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers"
+    TO_INSTALL="ccache cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers"
     make_conda "conda-forge" "$TO_INSTALL"
     source ./continuous_integration/install_flexiblas.sh
 

continuous_integration/install_flexiblas.sh

@@ -11,8 +11,9 @@ mkdir flexiblas_install
 git clone https://github.com/mpimd-csc/flexiblas.git
 pushd flexiblas
 
-# Temporary ping Flexiblas commit to avoid openmp symbols not found at link time
-git checkout v3.4.2 
+export CCACHE_DIR="${CCACHE_DIR:-$HOME/.cache/ccache}"
+mkdir -p "$CCACHE_DIR"
+ccache --max-size=500M
 
 mkdir build
 pushd build
@@ -27,14 +28,26 @@ fi
 # provided backends such as macOS' Apple/Accelerate/vecLib nor plaftorm
 # specific BLAS implementations such as MKL that cannot be installed on
 # arm64 hardware.
+FLEXIBLAS_LIB=$ABS_PATH/flexiblas_install/lib
 cmake ../ -DCMAKE_INSTALL_PREFIX=$ABS_PATH"/flexiblas_install" \
+    -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
+    -DCMAKE_INSTALL_RPATH=$FLEXIBLAS_LIB \
+    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_Fortran_COMPILER_LAUNCHER=ccache \
     -DBLAS_AUTO_DETECT="OFF" \
     -DEXTRA="OPENBLAS_CONDA" \
     -DFLEXIBLAS_DEFAULT="OPENBLAS_CONDA" \
     -DOPENBLAS_CONDA_LIBRARY=$CONDA_PREFIX"/lib/libopenblas"$EXTENSION \
 make
 make install
 
+export LD_LIBRARY_PATH=$FLEXIBLAS_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
+if [[ $(uname) == "Darwin" ]]; then
+    export DYLD_LIBRARY_PATH=$FLEXIBLAS_LIB${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}
+fi
+
 # Check that all 3 BLAS are listed in FlexiBLAS configuration
 $ABS_PATH/flexiblas_install/bin/flexiblas list
 popd
@@ -58,8 +71,10 @@ Cflags: -I\${includedir}" > flexiblas.pc
 
 PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=flexiblas -Csetup-args=-Dlapack=flexiblas
 
-export CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \
-export LDFLAGS="-L$ABS_PATH/flexiblas_install/lib -Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \
+ccache -s || true
+
+export CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas
+export LDFLAGS="-L$FLEXIBLAS_LIB -Wl,-rpath,$FLEXIBLAS_LIB"
 
 popd
 

tests/test_threadpoolctl.py

@@ -613,6 +613,7 @@ def test_architecture():
     expected_openblas_architectures = (
         # XXX: add more as needed by CI or developer laptops
         "armv8",
+        "cooperlake",
         "haswell",
         "neoversen1",
         "prescott",  # see: https://github.com/xianyi/OpenBLAS/pull/3485

threadpoolctl.py

@@ -165,7 +165,11 @@ class OpenBLASController(LibController):
 
     user_api = "blas"
     internal_api = "openblas"
-    filename_prefixes = ("libopenblas", "libblas", "libscipy_openblas")
+    filename_prefixes = (
+        "libopenblas",
+        "libblas",  # legacy conda-forge Windows shim, see _make_controller_from_path
+        "libscipy_openblas",
+    )
 
     _symbol_prefixes = ("", "scipy_")
     _symbol_suffixes = ("", "64_", "_64")
@@ -237,7 +241,10 @@ class BLISController(LibController):
 
     user_api = "blas"
     internal_api = "blis"
-    filename_prefixes = ("libblis", "libblas")
+    filename_prefixes = (
+        "libblis",
+        "libblas",
+    )  # libblas: legacy conda-forge Windows shim
     check_symbols = (
         "bli_thread_get_num_threads",
         "bli_thread_set_num_threads",
@@ -428,7 +435,11 @@ class MKLController(LibController):
 
     user_api = "blas"
     internal_api = "mkl"
-    filename_prefixes = ("libmkl_rt", "mkl_rt", "libblas")
+    filename_prefixes = (
+        "libmkl_rt",
+        "mkl_rt",
+        "libblas",  # legacy conda-forge Windows shim, see _make_controller_from_path
+    )
     check_symbols = (
         "MKL_Get_Max_Threads",
         "MKL_Set_Num_Threads",
@@ -1164,10 +1175,11 @@ def _make_controller_from_path(self, filepath):
             if prefix is None:
                 continue
 
-            # workaround for BLAS libraries packaged by conda-forge on windows, which
-            # are all renamed "libblas.dll". We thus have to check to which BLAS
-            # implementation it actually corresponds looking for implementation
-            # specific symbols.
+            # Legacy workaround for BLAS libraries that conda-forge used to expose
+            # on Windows as libblas.dll, disambiguated via implementation-specific
+            # symbols. Current conda-forge stacks no longer load libblas.dll (e.g.
+            # MKL is exposed as mkl_rt.<version>.dll instead), so this path is
+            # kept for older installs but cannot be exercised in today's CI.
             if prefix == "libblas":
                 if filename.endswith(".dll"):
                     libblas = ctypes.CDLL(filepath, _RTLD_NOLOAD)
@@ -1177,11 +1189,8 @@ def _make_controller_from_path(self, filepath):
                     ):
                         continue
                 else:
-                    # We ignore libblas on other platforms than windows because there
-                    # might be a libblas dso comming with openblas for instance that
-                    # can't be used to instantiate a pertinent LibController (many
-                    # symbols are missing) and would create confusion by making a
-                    # duplicate entry in threadpool_info.
+                    # Non-Windows libblas DSOs (e.g. from openblas) lack the symbols
+                    # needed to instantiate a controller and would duplicate entries.
                     continue
 
             # filename matches a prefix. Now we check if the library has the symbols we