Prototype 2: Evaluating alternatives to YAJL for JSON processing

.gitmodules

@@ -10,3 +10,9 @@
 [submodule "others/mbedtls"]
 	path = others/mbedtls
 	url = https://github.com/Mbed-TLS/mbedtls.git
+[submodule "others/simdjson"]
+	path = others/simdjson
+	url = https://github.com/simdjson/simdjson.git
+[submodule "others/jsoncons"]
+	path = others/jsoncons
+	url = https://github.com/danielaparker/jsoncons.git

README.md

@@ -253,6 +253,16 @@ It is recommended to use assertions where applicable, and to enable them with
 
 The source tree includes a Benchmark tool that can help measure library performance. The tool is located in the `test/benchmark/` directory. The build process also creates the binary here, so you will have the tool after the compilation is finished.
 
+Detailed benchmark test documentation is available in:
+* `docs/benchmark-tests.en.md`
+* `docs/benchmark-tests.de.md`
+
+To compare the JSON regression suite across both supported JSON backends with separate build directories, run:
+
+```shell
+$ ./test/run-json-backend-matrix.sh --jobs 4
+```
+
 To run, just type:
 
 ```shell
@@ -319,10 +329,12 @@ The tool is a straightforward wrapper application that utilizes the library. It
 
 Each transaction is an HTTP/1.1 GET request with some GET parameters. Common headers are added, followed by the response headers and an XML body. Between phases, the tool checks whether an intervention has occurred. All transactions are created with the same data.
 
-Note that the tool does not call the last phase (logging).
+Note that the tool calls the logging phase (`processLogging()`) for each transaction.
 
 Please remember to reset `basic_rules.conf` if you want to try with a different ruleset.
 
+The benchmark directory also includes `json_benchmark`, which targets JSON request-body processing with fixed scenario classes such as large objects, deep nesting, numeric payloads and UTF-8-heavy strings.
+
 ## Reporting Issues
 
 If you are facing a configuration issue or something is not working as you

build/msc_find_lib.m4

@@ -10,11 +10,11 @@ dnl Sets and AC_SUBSTs:
 dnl   ${NAME}_CFLAGS, ${NAME}_LDADD, ${NAME}_LDFLAGS,
 dnl   ${NAME}_VERSION, ${NAME}_DISPLAY, ${NAME}_FOUND (0/1/2)
 dnl
-dnl NAME         - Variable prefix (e.g., YAJL, CURL, LIBXML2)
+dnl NAME         - Variable prefix (e.g., CURL, LIBXML2, LMDB)
 dnl PKG_NAMES    - Space-separated pkg-config names to try
-dnl HEADER       - Header file to look for (e.g., yajl/yajl_parse.h)
+dnl HEADER       - Header file to look for (e.g., libxml/parser.h)
 dnl LIB_NAMES    - Space-separated library names for -l flags
-dnl EXTRA_CFLAGS - Additional CFLAGS when found (e.g., -DWITH_YAJL)
+dnl EXTRA_CFLAGS - Additional CFLAGS when found (e.g., -DWITH_LIBXML2)
 dnl MIN_VERSION  - Optional minimum version for pkg-config check
 dnl WITH_NAME    - Optional --with-X name if different from lowercased NAME
 
@@ -208,7 +208,7 @@ if test "${_msc_header_dir}" = "."; then
         _msc_check_inc_path="$4"
     fi
 else
-    # Header with subdirectory (e.g., "yajl/yajl_parse.h")
+    # Header with subdirectory (e.g., "libxml/parser.h")
     if test -e "$4/include/$2"; then
         _msc_check_inc_path="$4/include"
     elif test -e "$4/$2"; then

build/win32/CMakeLists.txt

@@ -7,6 +7,8 @@ option(WITH_LUA     "Include LUA support"     ON)
 option(WITH_LIBXML2 "Include LibXML2 support" ON)
 option(WITH_MAXMIND "Include MaxMind support" ON)
 option(WITH_CURL    "Include CURL support"    ON)
+set(JSON_BACKEND "simdjson" CACHE STRING "Select internal JSON backend (simdjson or jsoncons)")
+set_property(CACHE JSON_BACKEND PROPERTY STRINGS simdjson jsoncons)
 
 option(USE_ASAN     "Build with Address Sanitizer" OFF)
 
@@ -51,6 +53,8 @@ target_compile_definitions(libinjection PRIVATE LIBINJECTION_VERSION="${LIBINJEC
 project(mbedcrypto C)
 
 set(MBEDTLS_DIR ${BASE_DIR}/others/mbedtls)
+set(SIMDJSON_DIR ${BASE_DIR}/others/simdjson/singleheader)
+set(JSONCONS_DIR ${BASE_DIR}/others/jsoncons/include)
 
 add_library(mbedcrypto STATIC ${MBEDTLS_DIR}/library/base64.c ${MBEDTLS_DIR}/library/sha1.c ${MBEDTLS_DIR}/library/md5.c ${MBEDTLS_DIR}/library/platform_util.c ${MBEDTLS_DIR}/library/constant_time.c)
 
@@ -87,7 +91,25 @@ set(PACKAGE_VERSION "${PROJECT_VERSION}")
 set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}")
 set(PACKAGE_TARNAME "${PACKAGE_NAME}")
 
-set(HAVE_YAJL     1)    # should always be one, mandatory dependency
+if(NOT JSON_BACKEND STREQUAL "simdjson" AND NOT JSON_BACKEND STREQUAL "jsoncons")
+  message(FATAL_ERROR "Unsupported JSON_BACKEND '${JSON_BACKEND}'. Use simdjson or jsoncons.")
+endif()
+
+unset(MSC_JSON_BACKEND_SIMDJSON)
+unset(MSC_JSON_BACKEND_JSONCONS)
+set(JSON_BACKEND_SIMDJSON_SOURCE ${BASE_DIR}/src/request_body_processor/json_backend_simdjson.cc)
+set(JSON_BACKEND_JSONCONS_SOURCE ${BASE_DIR}/src/request_body_processor/json_backend_jsoncons.cc)
+
+if(JSON_BACKEND STREQUAL "simdjson")
+  set(MSC_JSON_BACKEND_SIMDJSON 1)
+  set(JSON_BACKEND_SOURCES ${JSON_BACKEND_SIMDJSON_SOURCE} ${SIMDJSON_DIR}/simdjson.cpp)
+  set(JSON_BACKEND_INCLUDE_DIR ${SIMDJSON_DIR})
+else()
+  set(MSC_JSON_BACKEND_JSONCONS 1)
+  set(JSON_BACKEND_SOURCES ${JSON_BACKEND_JSONCONS_SOURCE})
+  set(JSON_BACKEND_INCLUDE_DIR ${JSONCONS_DIR})
+endif()
+
 set(HAVE_GEOIP    0)    # should always be zero, no conan package available
 set(HAVE_SSDEEP   0)    # should always be zero, no conan package available
 
@@ -119,7 +141,6 @@ macro(include_package package flag)
   endif()
 endmacro()
 
-include_package(yajl HAVE_YAJL)
 include_package(libxml2 HAVE_LIBXML2)
 include_package(lua HAVE_LUA)
 include_package(CURL HAVE_CURL)
@@ -133,11 +154,13 @@ include_package(maxminddb HAVE_MAXMIND)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 file(GLOB_RECURSE libModSecuritySources ${BASE_DIR}/src/*.cc)
+list(REMOVE_ITEM libModSecuritySources ${JSON_BACKEND_SIMDJSON_SOURCE} ${JSON_BACKEND_JSONCONS_SOURCE})
+list(APPEND libModSecuritySources ${JSON_BACKEND_SOURCES})
 
 add_library(libModSecurity SHARED ${libModSecuritySources})
 
 target_compile_definitions(libModSecurity PRIVATE WITH_PCRE2)
-target_include_directories(libModSecurity PRIVATE ${BASE_DIR} ${BASE_DIR}/headers ${BASE_DIR}/others ${MBEDTLS_DIR}/include)
+target_include_directories(libModSecurity PRIVATE ${BASE_DIR} ${BASE_DIR}/headers ${BASE_DIR}/others ${MBEDTLS_DIR}/include ${JSON_BACKEND_INCLUDE_DIR})
 target_link_libraries(libModSecurity PRIVATE pcre2::pcre2 libinjection mbedcrypto Poco::Poco Iphlpapi.lib)
 
 macro(add_package_dependency project compile_definition link_library flag)
@@ -147,7 +170,6 @@ macro(add_package_dependency project compile_definition link_library flag)
   endif()
 endmacro()
 
-add_package_dependency(libModSecurity WITH_YAJL yajl::yajl HAVE_YAJL)
 add_package_dependency(libModSecurity WITH_LIBXML2 LibXml2::LibXml2 HAVE_LIBXML2)
 add_package_dependency(libModSecurity WITH_LUA lua::lua HAVE_LUA)
 if(HAVE_LUA)
@@ -164,17 +186,28 @@ project(libModSecurityTests)
 
 function(setTestTargetProperties executable)
   target_compile_definitions(${executable} PRIVATE WITH_PCRE2)
-  target_include_directories(${executable} PRIVATE ${BASE_DIR} ${BASE_DIR}/headers)
+  target_include_directories(${executable} PRIVATE ${BASE_DIR} ${BASE_DIR}/headers ${JSONCONS_DIR})
   target_link_libraries(${executable} PRIVATE libModSecurity pcre2::pcre2 dirent::dirent)
-  add_package_dependency(${executable} WITH_YAJL yajl::yajl HAVE_YAJL)
 endfunction()
 
 # unit tests
 file(GLOB unitTestSources ${BASE_DIR}/test/unit/*.cc)
+
+list(REMOVE_ITEM unitTestSources
+  ${BASE_DIR}/test/unit/json_backend_depth_tests.cc
+)
+
 add_executable(unit_tests ${unitTestSources} ${BASE_DIR}/test/common/custom_debug_log.cc)
 setTestTargetProperties(unit_tests)
 target_compile_options(unit_tests PRIVATE /wd4805)
 
+add_executable(json_backend_depth_tests
+  ${BASE_DIR}/test/unit/json_backend_depth_tests.cc
+  ${BASE_DIR}/test/common/custom_debug_log.cc
+)
+setTestTargetProperties(json_backend_depth_tests)
+target_compile_options(json_backend_depth_tests PRIVATE /wd4805)
+
 # regression tests
 file(GLOB regressionTestsSources ${BASE_DIR}/test/regression/*.cc)
 add_executable(regression_tests ${regressionTestsSources} ${BASE_DIR}/test/common/custom_debug_log.cc)
@@ -193,6 +226,12 @@ add_regression_test_capability(WITH_MAXMIND HAVE_MAXMIND)
 
 enable_testing()
 
+add_test(
+  NAME json_backend_depth_tests
+  COMMAND json_backend_depth_tests
+  WORKING_DIRECTORY ${BASE_DIR}/test
+)
+
 file(READ ${BASE_DIR}/test/test-suite.in TEST_FILES_RAW)
 string(REPLACE "\n" ";" TEST_FILES ${TEST_FILES_RAW})
 
@@ -212,15 +251,25 @@ foreach(TEST_FILE ${TEST_FILES})
     # test name
     get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE)
 
+    # json_backend_depth_tests is built as a standalone executable,
+    # so skip automatic registration through unit_tests
+    if(TEST_NAME STREQUAL "json_backend_depth_tests")
+      continue()
+    endif()
+
     # determine test runner based on test path prefix
     string(FIND ${TEST_FILE} "test-cases/regression/" is_regression_test)
     if(is_regression_test EQUAL 0)
-        set(TEST_RUNNER "regression_tests")
+      set(TEST_RUNNER "regression_tests")
     else()
-        set(TEST_RUNNER "unit_tests")
+      set(TEST_RUNNER "unit_tests")
     endif()
 
-    add_test(NAME ${TEST_NAME} COMMAND ${TEST_RUNNER} ${TEST_FILE} WORKING_DIRECTORY ${BASE_DIR}/test)
+    add_test(
+      NAME ${TEST_NAME}
+      COMMAND ${TEST_RUNNER} ${TEST_FILE}
+      WORKING_DIRECTORY ${BASE_DIR}/test
+    )
   endif()
 endforeach()
 
@@ -232,7 +281,6 @@ setTestTargetProperties(benchmark)
 add_executable(rules_optimization ${BASE_DIR}/test/optimization/optimization.cc)
 setTestTargetProperties(rules_optimization)
 
-
 # examples
 #
 

build/win32/conanfile.txt

@@ -1,5 +1,4 @@
 [requires]
-yajl/2.1.0
 pcre2/10.42
 libxml2/2.12.6
 lua/5.4.6

build/win32/config.h.cmake

@@ -57,12 +57,15 @@
 /* Define if SSDEEP is available */
 #cmakedefine HAVE_SSDEEP
 
-/* Define if YAJL is available */
-#cmakedefine HAVE_YAJL
-
 /* Define if libcurl is available */
 #cmakedefine HAVE_CURL
 
+/* Define if jsoncons is the selected internal JSON backend */
+#cmakedefine MSC_JSON_BACKEND_JSONCONS
+
+/* Define if simdjson is the selected internal JSON backend */
+#cmakedefine MSC_JSON_BACKEND_SIMDJSON
+
 /* Name of package */
 #define PACKAGE "@PACKAGE_NAME@"
 
@@ -89,4 +92,4 @@
 #cmakedefine STDC_HEADERS
 #endif
 
-#endif // ndef MODSECURITY_CONFIG_H
+#endif // ndef MODSECURITY_CONFIG_H