MINIFICPP-2859 Extend Conan usage and integrate it in Windows and MacOS CI jobs

.github/conan/compatibility.py

@@ -0,0 +1,95 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from conan.tools.build import supported_cppstd, supported_cstd
+
+
+def cppstd_compat(conanfile):
+    # It will try to find packages with all the cppstd versions
+    extension_properties = getattr(conanfile, "extension_properties", {})
+    compiler = conanfile.settings.get_safe("compiler")
+    compiler_version = conanfile.settings.get_safe("compiler.version")
+    cppstd = conanfile.settings.get_safe("compiler.cppstd")
+    if not compiler or not compiler_version:
+        return []
+    factors = []  # List of list, each sublist is a potential combination
+    if cppstd is not None and extension_properties.get("compatibility_cppstd") is not False:
+        cppstd_possible_values = supported_cppstd(conanfile)
+        if cppstd_possible_values is None:
+            conanfile.output.warning(f'No cppstd compatibility defined for compiler "{compiler}"')
+        else:  # The current cppst must be included in case there is other factor
+            factors.append([{"compiler.cppstd": v} for v in cppstd_possible_values])
+
+    cstd = conanfile.settings.get_safe("compiler.cstd")
+    if cstd is not None and extension_properties.get("compatibility_cstd") is not False:
+        cstd_possible_values = supported_cstd(conanfile)
+        if cstd_possible_values is None:
+            conanfile.output.warning(f'No cstd compatibility defined for compiler "{compiler}"')
+        else:
+            factors.append([{"compiler.cstd": v} for v in cstd_possible_values if v != cstd])
+    return factors
+
+
+def compatibility(conanfile):
+    # By default, different compiler.cppstd are compatible
+    # factors is a list of lists
+    factors = cppstd_compat(conanfile)
+
+    # MSVC fallback compatibility
+    compiler = conanfile.settings.get_safe("compiler")
+    compiler_version = conanfile.settings.get_safe("compiler.version")
+    if compiler == "msvc":
+        msvc_fallbacks = {
+            "195": ["194", "193"],
+            "194": ["193"],
+        }.get(compiler_version, [])
+
+        if msvc_fallbacks:
+            factors.append([{"compiler.version": v} for v in msvc_fallbacks])
+
+    # macOS / apple-clang: accept any compiler.version >= 13 as compatible
+    os_ = conanfile.settings.get_safe("os")
+    if os_ == "Macos" and compiler == "apple-clang" and compiler_version is not None:
+        try:
+            current_version = int(str(compiler_version).split(".")[0])
+        except ValueError:
+            current_version = None
+        if current_version is not None:
+            min_version = 13
+            max_version = 21  # support up to Xcode 27 with apple-clang 21 for now
+            candidate_versions = [
+                str(v) for v in range(min_version, max_version + 1)
+                if v != current_version
+            ]
+            factors.append([{"compiler.version": v} for v in candidate_versions])
+
+    combinations = _factors_combinations(factors)
+    return [{"settings": [(k, v) for k, v in comb.items()]} for comb in combinations]
+
+
+def _factors_combinations(factors):
+    combinations = []
+    for factor in factors:
+        if not combinations:
+            combinations = list(factor)
+            continue
+        new_combinations = []
+        for comb in combinations:
+            for f in factor:
+                new_comb = comb.copy()
+                new_comb.update(f)
+                new_combinations.append(new_comb)
+        combinations.extend(new_combinations)
+    return combinations

.github/workflows/ci.yml

@@ -13,6 +13,8 @@ env:
     -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
     -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON -DMINIFI_RUST=OFF
   CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
+  CONAN_LOGIN_USERNAME_NIFI_CONAN: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_NIFI_CONAN: ${{ secrets.CONAN_ACCESS_TOKEN }}
 jobs:
   macos_xcode:
     name: "macOS 26 aarch64"
@@ -64,6 +66,7 @@ jobs:
         -DSKIP_TESTS=OFF
         -DUSE_SHARED_LIBS=ON
         -DMINIFI_PERFORMANCE_TESTS=ON
+        -DUSE_CONAN=ON
     steps:
       - id: checkout
         uses: actions/checkout@v6
@@ -85,9 +88,17 @@ jobs:
           echo "PATH=/opt/homebrew/opt/ccache:/opt/homebrew/opt/ccache/bin:/opt/homebrew/opt/ccache/libexec:$PATH" >> $GITHUB_ENV
           echo "DYLD_LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV
           echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
+          mkdir -p ~/.conan2/extensions/plugins/compatibility
+          cp .github/conan/compatibility.py ~/.conan2/extensions/plugins/compatibility/compatibility.py
       - name: build
         run: |
-          python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && python main.py --noninteractive --skip-compiler-install --cmake-options="-DCMAKE_C_FLAGS=${CPPFLAGS} ${CFLAGS} -DCMAKE_CXX_FLAGS=${CPPFLAGS} ${CXXFLAGS}" --minifi-options="${MACOS_MINIFI_OPTIONS}"
+          python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && \
+          python main.py --noninteractive --skip-compiler-install --cmake-options="-DCMAKE_C_FLAGS=${CPPFLAGS} ${CFLAGS} -DCMAKE_CXX_FLAGS=${CPPFLAGS} ${CXXFLAGS}" --minifi-options="${MACOS_MINIFI_OPTIONS}"
+        working-directory: bootstrap
+      - name: Upload conan packages
+        if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          source venv/bin/activate && conan remote login nifi-conan && conan upload "*" -r nifi-conan --confirm
         working-directory: bootstrap
       - name: cache save
         uses: actions/cache/save@v5
@@ -173,6 +184,7 @@ jobs:
         -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=ON
         -DUSE_SHARED_LIBS=OFF
         -DMINIFI_PERFORMANCE_TESTS=ON
+        -DUSE_CONAN=ON
     steps:
       - name: Support longpaths
         run: git config --system core.longpaths true
@@ -194,14 +206,27 @@ jobs:
           if ((Get-FileHash 'sqliteodbc_w64.exe').Hash -ne "a4804e4f54f42c721df1323c5fcac101a8c7a577e7f20979227324ceab572d51") {Write "Hash mismatch"; Exit 1}
           Start-Process -FilePath ".\sqliteodbc_w64.exe" -ArgumentList "/S" -Wait
         shell: powershell
+      - name: Copy conan compatibility.py
+        shell: powershell
+        run: |
+          $compatDir = Join-Path $env:USERPROFILE ".conan2\extensions\plugins\compatibility"
+          New-Item -ItemType Directory -Force -Path $compatDir | Out-Null
+          Copy-Item ".github\conan\compatibility.py" (Join-Path $compatDir "compatibility.py")
       - name: Zero sccache stats
         run: sccache --zero-stats
         shell: bash
       - name: build
+        working-directory: bootstrap
         run: |
-          python -m venv venv & venv\Scripts\activate & pip install -r requirements.txt & python main.py --noninteractive --skip-compiler-install --minifi-options="%WINDOWS_MINIFI_OPTIONS%" --cmake-options="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
+          python -m venv venv && venv\Scripts\activate && pip install -r requirements.txt && ^
+          python main.py --noninteractive --skip-compiler-install --minifi-options="%WINDOWS_MINIFI_OPTIONS%" --cmake-options="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
         shell: cmd
+      - name: Upload conan packages
+        if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
         working-directory: bootstrap
+        run: |
+          venv\Scripts\activate && conan remote login nifi-conan && conan upload "*" -r nifi-conan --confirm
+        shell: cmd
       - name: Save cache
         uses: actions/cache/save@v5
         if: always()

CMakeLists.txt

@@ -87,8 +87,12 @@ endif()
 # Use ccache if present
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+    if(NOT CMAKE_C_COMPILER_LAUNCHER)
+        set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_FOUND}")
+    endif()
+    if(NOT CMAKE_CXX_COMPILER_LAUNCHER)
+        set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_FOUND}")
+    endif()
     message("-- Found ccache: ${CCACHE_FOUND}")
 endif(CCACHE_FOUND)
 
@@ -197,6 +201,7 @@ set(PASSTHROUGH_CMAKE_ARGS -DANDROID_ABI=${ANDROID_ABI}
     -DANDROID_NDK=${ANDROID_NDK}
     -DCMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
     -DCMAKE_POLICY_VERSION_MINIMUM=${CMAKE_POLICY_VERSION_MINIMUM}
+    -DCMAKE_POLICY_DEFAULT_CMP0091=NEW
     -DCMAKE_C_FLAGS=${PASSTHROUGH_CMAKE_C_FLAGS}
     -DCMAKE_CXX_FLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS}
     -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
@@ -221,12 +226,12 @@ set(MINIFI_CPP_COMPILE_DEFINITIONS "")
 
 if(CUSTOM_MALLOC)
     if (CUSTOM_MALLOC STREQUAL jemalloc)
-        include(BundledJemalloc)
-        use_bundled_jemalloc(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-        set(CUSTOM_MALLOC_LIB JeMalloc::JeMalloc)
+        include(GetJemalloc)
+        get_jemalloc(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+        set(CUSTOM_MALLOC_LIB jemalloc::jemalloc)
     elseif (CUSTOM_MALLOC STREQUAL mimalloc)
-        include(MiMalloc)
-        set(CUSTOM_MALLOC_LIB mimalloc)
+        include(GetMiMalloc)
+        set(CUSTOM_MALLOC_LIB mimalloc-static)
     elseif (CUSTOM_MALLOC STREQUAL rpmalloc)
         include(RpMalloc)
         set(CUSTOM_MALLOC_LIB rpmalloc)
@@ -243,7 +248,6 @@ include(GetOpenSSL)
 # BZip2
 if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32)))
     include(GetBZip2)
-    get_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/bzip2/dummy")
 endif()
 
@@ -259,8 +263,7 @@ if(NOT WIN32)
 endif()
 
 # libsodium
-include(BundledLibSodium)
-use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
+include(GetLibSodium)
 
 list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS SODIUM_STATIC=1)
 list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS "RPM_CONFIG_DIR=\"/${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME}\"")
@@ -270,20 +273,17 @@ list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS "RPM_LIB_DIR=\"/usr/${CMAKE_INSTALL_L
 
 # zlib
 include(GetZLIB)
-get_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zlib/dummy")
 
 # cURL
 include(GetLibCURL)
-get_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/curl/dummy")
 
 # spdlog
 include(GetSpdlog)
-get_spdlog()
 
 # yaml-cpp
-include(YamlCpp)
+include(GetYamlCpp)
 
 # concurrentqueue
 add_library(concurrentqueue INTERFACE)
@@ -295,7 +295,7 @@ target_include_directories(RapidJSON SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DI
 target_compile_definitions(RapidJSON INTERFACE RAPIDJSON_HAS_STDSTRING)
 
 # gsl-lite
-include(GslLite)
+include(GetGslLite)
 
 # Add necessary definitions based on the value of STRICT_GSL_CHECKS, see gsl-lite README for more details
 list(APPEND GslDefinitions gsl_CONFIG_DEFAULTS_VERSION=1)
@@ -314,13 +314,17 @@ endif()
 if (STRICT_GSL_CHECKS STREQUAL "DEBUG_ONLY")
     list(APPEND GslDefinitions $<$<NOT:$<CONFIG:Debug>>:${GslDefinitionsNonStrict}>)
 endif()
-target_compile_definitions(gsl-lite INTERFACE ${GslDefinitions})
 
-# date
-include(Date)
+get_target_property(_gsl_lite_real_target gsl-lite::gsl-lite ALIASED_TARGET)
+if (NOT _gsl_lite_real_target)
+    set(_gsl_lite_real_target gsl-lite::gsl-lite)
+endif()
+target_compile_definitions(${_gsl_lite_real_target} INTERFACE ${GslDefinitions})
 
-# magic_enum
-include(MagicEnum)
+include(Date)
+include(GetMagicEnum)
+include(GetRangeV3)
+include(GetAsio)
 
 # Setup warning flags
 if(MSVC)
@@ -364,14 +368,13 @@ add_subdirectory(libminifi)
 
 if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_GRAFANA_LOKI OR ENABLE_CIVET)
     include(GetCivetWeb)
-    get_civetweb()
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/civetweb/dummy")
 endif()
 
 ## Add extensions
 
 # PugiXML required for standard processors and WEL extension
-include(PugiXml)
+include(GetPugiXml)
 
 file(GLOB extension-directories "extensions/*")
 foreach(extension-dir ${extension-directories})

CONAN.md

@@ -36,7 +36,11 @@ source, it uses CMake to install the third party external libraries that are ena
 
 ## Build MiNiFi C++ with Conan
 
-### Install Conan
+### Using the bootstrap python script
+
+The easiest way to build MiNiFi C++ using Conan is by using the the bootstrap script. Run the bootstrap script either with `bootstrap/py_bootstrap.sh` or `bootstrap/py_bootstrap.bat` and the bootstrap installs the conan package manager automatically. In the bootstrap script under Build options menu select the `USE_CONAN` option. Running the one click build with the option selected, it will download the available thirdparty dependencies from the conan repository, otherwise it will build the thirdparty dependency from source (either with the available conan recipe or using the bundled cmake script).
+
+### Install Conan manually
 
 ~~~bash
 sudo pip install --force-reinstall -v "conan==2.28.1"
@@ -45,15 +49,29 @@ sudo pip install --force-reinstall -v "conan==2.28.1"
 conan profile detect
 ~~~
 
-### Create Custom RocksDB Conan Package
-The default RocksDB conan package is built with -fno-rtti, which makes it incompatible with MiNiFi. So we need to create a custom RocksDB conan package that is built with -frtti.
+### Create Custom Conan Package
+Some packages need custom versions built for MiNiFi specifically. This could be due patched source code, different compile options, or different dependency library versions.
+
+For example the default RocksDB conan package is built with -fno-rtti, which makes it incompatible with MiNiFi. So we need to create a custom RocksDB conan package that is built with -frtti.
 
 ~~~bash
 cd $HOME/nifi-minifi-cpp/thirdparty/rocksdb/all
 
 conan create . --user=minifi --channel=develop --test-folder="" --version=11.1.1 --profile=../../../etc/conan/profiles/release-linux
 ~~~
 
+If you want to use conan to build these custom libraries as part of the MiNiFi build you only need to export these libraries instead of creating them.
+
+~~~bash
+conan export $HOME/nifi-minifi-cpp/thirdparty/rocksdb/all --version=11.1.1 --user=minifi --channel=develop
+~~~
+
+You can add the official NiFi remote Conan repository to download the custom Conan packages and recipes manually:
+
+~~~bash
+conan remote add nifi-conan https://apache.jfrog.io/artifactory/api/conan/nifi-conan
+~~~
+
 ### Build MiNiFi
 
 ~~~bash
@@ -68,6 +86,12 @@ conan install . --build=missing --output-folder=build_conan --profile=etc/conan/
 conan build . --output-folder=build_conan --profile=etc/conan/profiles/release-linux
 ~~~
 
+After `conan install` command you can also use the conan generated `conan_toolchain.cmake` file with the CMake command to build with Conan packages:
+
+~~~bash
+cmake -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake .. && ninja
+~~~
+
 - **NOTE**: After building MiNiFi, we must have the MINIFI_HOME environment variable created in order to successfully run the minifi binary executable.
 
 - **NOTE**: When we install the prebuilt conan package representing the MiNiFi third party libraries, we add the `--build=missing` in case some of the prebuilt missing conan packages are not found from conancenter, jfrog, or one of our conan repositories, then conan will build the conan packages from source. We can upload new prebuilt conan packages by **[conan upload](https://docs.conan.io/2/reference/commands/upload.html)**.