fix: check avro-cpp version instead of code pattern, make Avro optional

- Extract version from library soname on conda-forge
- Skip Kafka adapter on Windows if avro-cpp < 1.12.1 or version unknown
- Make Avro compilation conditional with CSP_USE_AVRO define
- Add vcpkg.json override to use avro-cpp 1.12.1 without full submodule update
- Rename CSP_AVRO_TARGET to AVRO_LIBRARIES for consistency

Signed-off-by: Krzysztof Milde <Krzysztof.Milde@Point72.com>

Author: mildek

cpp/cmake/modules/FindDepsKafkaAdapter.cmake

@@ -1,22 +1,48 @@
 cmake_minimum_required(VERSION 3.7.2)
 
+# RdKafka is required for Kafka adapter
+find_package(RdKafka REQUIRED)
+
+# On Windows with vcpkg, librdkafka with SSL requires explicit OpenSSL linking
+# because vcpkg's static library transitive dependencies don't propagate properly
+if(WIN32 AND CSP_USE_VCPKG)
+  find_package(OpenSSL REQUIRED)
+  set(KAFKA_OPENSSL_LIBRARIES OpenSSL::SSL OpenSSL::Crypto CACHE INTERNAL "")
+else()
+  set(KAFKA_OPENSSL_LIBRARIES "" CACHE INTERNAL "")
+endif()
+
+# Avro is optional - Kafka adapter works with JSON/raw bytes without it
+set(CSP_KAFKA_ENABLE_AVRO FALSE)
+
 if (CSP_USE_VCPKG)
-  find_package(RdKafka CONFIG REQUIRED)
-  find_package(unofficial-avro-cpp CONFIG REQUIRED)
-  if(NOT WIN32)
+  find_package(unofficial-avro-cpp CONFIG)
+
+  if(unofficial-avro-cpp_FOUND)
+    # Check avro-cpp version on Windows - require >= 1.12.1 for fmt compatibility
+    if(WIN32 AND unofficial-avro-cpp_VERSION VERSION_LESS "1.12.1")
+      message(WARNING
+        "avro-cpp ${unofficial-avro-cpp_VERSION} has incompatible fmt::formatter on Windows. "
+        "Kafka Avro support will be disabled. Update vcpkg to get avro-cpp >= 1.12.1.")
+    else()
+      set(CSP_KAFKA_ENABLE_AVRO TRUE)
+      set(AVRO_LIBRARIES unofficial::avro-cpp::avrocpp CACHE INTERNAL "")
+    endif()
+  endif()
+
+  if(NOT WIN32 AND CSP_KAFKA_ENABLE_AVRO)
     # Bad, but a temporary workaround for
     # https://github.com/microsoft/vcpkg/issues/40320
     link_directories(${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib)
   endif()
-  set(AVRO_LIBRARIES unofficial::avro-cpp::avrocpp CACHE INTERNAL "")
-  set(DepsKafkaAdapter_FOUND TRUE)
 else()
-  find_package(RdKafka REQUIRED)
   find_package(Avro)
-  if(NOT Avro_FOUND)
-    set(DepsKafkaAdapter_FOUND FALSE)
-  else()
+
+  if(Avro_FOUND)
+    set(CSP_KAFKA_ENABLE_AVRO TRUE)
     set(AVRO_LIBRARIES Avro::avrocpp CACHE INTERNAL "")
-    set(DepsKafkaAdapter_FOUND TRUE)
   endif()
 endif()
+
+# Kafka adapter is available if RdKafka is found
+set(DepsKafkaAdapter_FOUND TRUE)

cpp/csp/adapters/kafka/CMakeLists.txt

@@ -22,9 +22,11 @@ set_target_properties(csp_kafka_adapter PROPERTIES PUBLIC_HEADER "${KAFKA_HEADER
 
 find_package(DepsKafkaAdapter REQUIRED)
 
-target_link_libraries(csp_kafka_adapter PUBLIC csp_adapter_utils RdKafka::rdkafka RdKafka::rdkafka++)
+target_link_libraries(csp_kafka_adapter PUBLIC csp_adapter_utils RdKafka::rdkafka RdKafka::rdkafka++ ${KAFKA_OPENSSL_LIBRARIES})
 
-target_link_libraries(csp_kafka_adapter PUBLIC ${AVRO_LIBRARIES})
+if(CSP_KAFKA_ENABLE_AVRO)
+    target_link_libraries(csp_kafka_adapter PUBLIC ${AVRO_LIBRARIES})
+endif()
 
 install(TARGETS csp_kafka_adapter
         PUBLIC_HEADER DESTINATION include/csp/adapters/kafka

cpp/csp/adapters/kafka/KafkaPublisher.cpp

@@ -3,7 +3,9 @@
 #include <csp/adapters/kafka/KafkaPublisher.h>
 #include <csp/adapters/utils/MessageWriter.h>
 #include <csp/adapters/utils/JSONMessageWriter.h>
+#ifdef CSP_USE_AVRO
 #include <csp/adapters/utils/AvroMessageWriter.h>
+#endif
 
 #include <librdkafka/rdkafkacpp.h>
 
@@ -18,8 +20,10 @@ KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & pr
     auto protocol = properties.get<std::string>( "protocol" );
     if( protocol == "JSON" )
         m_msgWriter = std::make_shared<utils::JSONMessageWriter>( properties );
+#ifdef CSP_USE_AVRO
     else if( protocol == "AVRO" )
         m_msgWriter = std::make_shared<utils::AvroMessageWriter>( properties );
+#endif
     else if( protocol != "RAW_BYTES" )
         CSP_THROW( NotImplemented, "msg protocol " << protocol << " not currently supported for kafka output adapters" );
 }

cpp/csp/adapters/utils/CMakeLists.txt

@@ -12,20 +12,26 @@ set(ADAPTER_UTILS_PUBLIC_HEADERS
         )
 
 set(ADAPTER_UTILS_FILES
-        AvroMessageStructConverter.cpp
         JSONMessageStructConverter.cpp
         MessageWriter.cpp
         MessageEnums.cpp
         MessageStructConverter.cpp
         RawBytesMessageStructConverter.cpp
         )
 
+find_package(DepsKafkaAdapter REQUIRED)
+
+if(CSP_KAFKA_ENABLE_AVRO)
+    list(APPEND ADAPTER_UTILS_FILES AvroMessageStructConverter.cpp)
+endif()
+
 add_library(csp_adapter_utils STATIC ${ADAPTER_UTILS_FILES})
 set_target_properties(csp_adapter_utils PROPERTIES PUBLIC_HEADER "${ADAPTER_UTILS_PUBLIC_HEADERS}" PREFIX lib)
 
-find_package(DepsKafkaAdapter REQUIRED)
-
-target_link_libraries(csp_adapter_utils PUBLIC ${AVRO_LIBRARIES})
+if(CSP_KAFKA_ENABLE_AVRO)
+    target_compile_definitions(csp_adapter_utils PUBLIC CSP_USE_AVRO)
+    target_link_libraries(csp_adapter_utils PUBLIC ${AVRO_LIBRARIES})
+endif()
 
 install(TARGETS csp_adapter_utils
         PUBLIC_HEADER DESTINATION include/csp/adapters/utils

cpp/csp/adapters/utils/MessageStructConverter.cpp

@@ -1,5 +1,7 @@
 #include <csp/adapters/utils/MessageStructConverter.h>
+#ifdef CSP_USE_AVRO
 #include <csp/adapters/utils/AvroMessageStructConverter.h>
+#endif
 #include <csp/adapters/utils/JSONMessageStructConverter.h>
 #include <csp/adapters/utils/RawBytesMessageStructConverter.h>
 
@@ -16,7 +18,9 @@ MessageStructConverterCache::MessageStructConverterCache()
 {
     registerConverter( "RAW_BYTES", &RawBytesMessageStructConverter::create );
     registerConverter( "JSON",      &JSONMessageStructConverter::create );
+#ifdef CSP_USE_AVRO
     registerConverter( "AVRO",      &AvroMessageStructConverter::create );
+#endif
 }
 
 bool MessageStructConverterCache::registerConverter( std::string protocol, Creator creator )

setup.py

@@ -38,7 +38,7 @@
 else:
     VCPKG_TRIPLET = None
 
-VCPKG_SHA = "f9d8eecab99cb23e4befb7ea90b2f9504d311b54"
+VCPKG_SHA = "9c5c2a0ab75aff5bcd08142525f6ff7f6f7ddeee"
 
 # This will be used for e.g. the sdist
 if CSP_USE_VCPKG: