v2: Allow partial processing of XML, JSON, and multipart request body

apache2/apache2_io.c

@@ -17,6 +17,7 @@
 #include "modsecurity.h"
 #include "apache2.h"
 #include "msc_crypt.h"
+#include "msc_reqbody.h"
 
 #ifdef APLOG_USE_MODULE
     APLOG_USE_MODULE(security2);
@@ -178,7 +179,7 @@
 /**
  * Reads request body from a client.
  */
 apr_status_t read_request_body(modsec_rec *msr, char **error_msg) {
     assert(msr != NULL);
     assert(error_msg!= NULL);
     request_rec *r = msr->r;
@@ -299,38 +300,41 @@
 #endif
             }
 
-            msr->reqbody_length += buflen;
-
-            if (buflen != 0) {
-                int rcbs = modsecurity_request_body_store(msr, buf, buflen, error_msg);
+            if (msr->reqbody_length + buflen > (apr_size_t)msr->txcfg->reqbody_limit) {
+                msr->reqbody_length_limit_exceeded = 1;
+                if (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL) {
+                    buflen = (apr_size_t)msr->txcfg->reqbody_limit - msr->reqbody_length;
+                    if (msr->txcfg->debuglog_level >= 9) {
+                        msr_log(msr, 9, "Input filter: Bucket type %s shortened by %" APR_SIZE_T_FMT " bytes because of reqbody_limit and ProcessPartial.",
+                                bucket->type->name, (apr_size_t)msr->txcfg->reqbody_limit - msr->reqbody_length);
+                    }
 
-                if (msr->reqbody_length > (apr_size_t)msr->txcfg->reqbody_limit && msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL) {
                     finished_reading = 1;
+                    modsecurity_request_body_do_enable_partial_processing(msr);
                 }
+            }
+            if (msr->reqbody_no_files_length_limit_exceeded) {
+                if (msr->txcfg->debuglog_level >= 9) {
+                    msr_log(msr, 9, "Input filter: Bucket type %s skip storing because of no_files_limit and ProcessPartial.",
+                            bucket->type->name);
+                }
+                buflen = 0;
+                finished_reading = 1;
+            }
 
+            msr->reqbody_length += buflen;
+
+            if (buflen != 0) {
+                int rcbs = modsecurity_request_body_store(msr, buf, buflen, error_msg);
                 if (rcbs < 0) {
                     if (rcbs == -5) {
-                        if((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                            return HTTP_REQUEST_ENTITY_TOO_LARGE;
-                        } else if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL)) {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                        } else if ((msr->txcfg->is_enabled == MODSEC_DETECTION_ONLY) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL)) {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                        } else {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                            return HTTP_REQUEST_ENTITY_TOO_LARGE;
-                        }
+                        return HTTP_REQUEST_ENTITY_TOO_LARGE;
                     }
 
-                    if((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT))
+                    if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
                         return HTTP_INTERNAL_SERVER_ERROR;
+                    }
                 }
-
             }
 
             if (APR_BUCKET_IS_EOS(bucket)) {
@@ -351,11 +355,14 @@
 
     msr->if_status = IF_STATUS_WANTS_TO_RUN;
 
-    if (rcbe == -5) {
-        return HTTP_REQUEST_ENTITY_TOO_LARGE;
-    }
     if (rcbe < 0) {
-        return HTTP_INTERNAL_SERVER_ERROR;
+        if (rcbe == -5) {
+            return HTTP_REQUEST_ENTITY_TOO_LARGE;
+        }
+
+        if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
+            return HTTP_INTERNAL_SERVER_ERROR;
+        }
     }
     return APR_SUCCESS;
 }

apache2/modsecurity.h

@@ -279,6 +279,9 @@ struct modsec_rec {
     unsigned int         if_started_forwarding;
 
     apr_size_t           reqbody_length;
+    unsigned int         reqbody_partial_processing_enabled;
+    unsigned int         reqbody_length_limit_exceeded;
+    unsigned int         reqbody_no_files_length_limit_exceeded;
 
     apr_bucket_brigade  *of_brigade;
     unsigned int         of_status;

apache2/msc_json.c

@@ -187,7 +187,7 @@ static int yajl_start_array(void *ctx) {
     msr->json->current_depth++;
     if (msr->json->current_depth > msr->txcfg->reqbody_json_depth_limit) {
         msr->json->depth_limit_exceeded = 1;
-	return 0;
+        return 0;
     }
 
     if (msr->txcfg->debuglog_level >= 9) {
@@ -262,7 +262,7 @@ static int yajl_start_map(void *ctx)
     msr->json->current_depth++;
     if (msr->json->current_depth > msr->txcfg->reqbody_json_depth_limit) {
         msr->json->depth_limit_exceeded = 1;
-	return 0;
+        return 0;
     }
 
     if (msr->txcfg->debuglog_level >= 9) {
@@ -367,6 +367,10 @@ int json_init(modsec_rec *msr, char **error_msg) {
     return 1;
 }
 
+void json_allow_partial_values(modsec_rec *msr) {
+    (void)yajl_config(msr->json->handle, yajl_allow_partial_values, 1);
+}
+
 /**
  * Feed one chunk of data to the JSON parser.
  */
@@ -380,16 +384,16 @@ int json_process_chunk(modsec_rec *msr, const char *buf, unsigned int size, char
     /* Feed our parser and catch any errors */
     msr->json->status = yajl_parse(msr->json->handle, buf, size);
     if (msr->json->status != yajl_status_ok) {
-	if (msr->json->depth_limit_exceeded) {
-           *error_msg = "JSON depth limit exceeded";
-	} else {
-        if (msr->json->yajl_error) *error_msg = msr->json->yajl_error;
-        else {
-             char* yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
-             *error_msg = apr_pstrdup(msr->mp, yajl_err);
-             yajl_free_error(msr->json->handle, yajl_err);
+        if (msr->json->depth_limit_exceeded) {
+             *error_msg = "JSON depth limit exceeded";
+        } else {
+            if (msr->json->yajl_error) *error_msg = msr->json->yajl_error;
+            else {
+                 char* yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
+                 *error_msg = apr_pstrdup(msr->mp, yajl_err);
+                 yajl_free_error(msr->json->handle, yajl_err);
+            }
         }
-	}
         return -1;
     }
 
@@ -409,13 +413,13 @@ int json_complete(modsec_rec *msr, char **error_msg) {
     /* Wrap up the parsing process */
     msr->json->status = yajl_complete_parse(msr->json->handle);
     if (msr->json->status != yajl_status_ok) {
-	if (msr->json->depth_limit_exceeded) {
-           *error_msg = "JSON depth limit exceeded";
-	} else {
-           char *yajl_err = yajl_get_error(msr->json->handle, 0, NULL, 0);
-           *error_msg = apr_pstrdup(msr->mp, yajl_err);
-           yajl_free_error(msr->json->handle, yajl_err);
-	}
+        if (msr->json->depth_limit_exceeded) {
+             *error_msg = "JSON depth limit exceeded";
+        } else {
+             char *yajl_err = yajl_get_error(msr->json->handle, 0, NULL, 0);
+             *error_msg = apr_pstrdup(msr->mp, yajl_err);
+             yajl_free_error(msr->json->handle, yajl_err);
+        }
 
         return -1;
     }

apache2/msc_json.h

@@ -48,6 +48,8 @@ struct json_data {
 
 int DSOLOCAL json_init(modsec_rec *msr, char **error_msg);
 
+void DSOLOCAL json_allow_partial_values(modsec_rec *msr);
+
 int DSOLOCAL json_process(modsec_rec *msr, const char *buf,
     unsigned int size, char **error_msg);
 

apache2/msc_multipart.c

@@ -19,6 +19,9 @@
 #include "msc_multipart.h"
 #include "msc_util.h"
 #include "msc_parsers.h"
+#include "msc_reqbody.h"
+
+#define CONTENT_TYPE_MAX_LENGTH 1024
 
 void validate_quotes(modsec_rec *msr, char *data, char quote)  {
     assert(msr != NULL);
@@ -271,7 +274,8 @@
     }
 
     /* The buffer is data so increase the data length counter. */
-    msr->msc_reqbody_no_files_length += (MULTIPART_BUF_SIZE - msr->mpd->bufleft);
+    len = modsecurity_request_body_may_enable_partial_processing_for_no_files_length(msr, len, "MULTIPART");
+    msr->msc_reqbody_no_files_length += len;
 
     if (len > 1) {
         if (msr->mpd->buf[len - 2] == '\r') {
@@ -421,7 +425,7 @@
             if (data == msr->mpd->buf) {
                 *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid part header (header name missing).");
 
-                 return -1;
+                return -1;
             }
 
             /* check if multipart header contains any invalid characters */
@@ -469,7 +473,7 @@
     assert(error_msg != NULL);
     char *p = msr->mpd->buf + (MULTIPART_BUF_SIZE - msr->mpd->bufleft);
     char localreserve[2] = { '\0', '\0' }; /* initialized to quiet warning */
-    int bytes_reserved = 0;
+    int bytes_reserved = 0, len;
 
     *error_msg = NULL;
 
@@ -585,7 +589,9 @@
         value_part_t *value_part = apr_pcalloc(msr->mp, sizeof(value_part_t));
 
         /* The buffer contains data so increase the data length counter. */
-        msr->msc_reqbody_no_files_length += (MULTIPART_BUF_SIZE - msr->mpd->bufleft) + msr->mpd->reserve[0];
+        len = modsecurity_request_body_may_enable_partial_processing_for_no_files_length(msr,
+            (MULTIPART_BUF_SIZE - msr->mpd->bufleft) + msr->mpd->reserve[0], "MULTIPART");
+        msr->msc_reqbody_no_files_length += len;
 
         /* add this part to the list of parts */
 
@@ -595,14 +601,14 @@
         }
 
         if (msr->mpd->reserve[0] != 0) {
-            value_part->data = apr_palloc(msr->mp, (MULTIPART_BUF_SIZE - msr->mpd->bufleft) + msr->mpd->reserve[0]);
+            value_part->data = apr_palloc(msr->mp, len);
             memcpy(value_part->data, &(msr->mpd->reserve[1]), msr->mpd->reserve[0]);
-            memcpy(value_part->data + msr->mpd->reserve[0], msr->mpd->buf, (MULTIPART_BUF_SIZE - msr->mpd->bufleft));
+            memcpy(value_part->data + msr->mpd->reserve[0], msr->mpd->buf, len - msr->mpd->reserve[0]);
 
-            value_part->length = (MULTIPART_BUF_SIZE - msr->mpd->bufleft) + msr->mpd->reserve[0];
+            value_part->length = len;
             msr->mpd->mpp->length += value_part->length;
         } else {
-            value_part->length = (MULTIPART_BUF_SIZE - msr->mpd->bufleft);
+            value_part->length = len;
             value_part->data = apr_pstrmemdup(msr->mp, msr->mpd->buf, value_part->length);
             msr->mpd->mpp->length += value_part->length;
         }
@@ -802,7 +808,7 @@
 /**
  *
  */
 int multipart_init(modsec_rec *msr, char **error_msg) {
     assert(msr != NULL);
     assert(error_msg != NULL);
     *error_msg = NULL;
@@ -822,7 +828,7 @@
         return -1;
     }
 
-    if (strlen(msr->request_content_type) > 1024) {
+    if (strlen(msr->request_content_type) > CONTENT_TYPE_MAX_LENGTH) {
         msr->mpd->flag_error = 1;
         *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid boundary in C-T (length).");
         return -1;
@@ -967,7 +973,7 @@
  * Finalise multipart processing. This method is invoked at the end, when it
  * is clear that there is no more data to be processed.
  */
 int multipart_complete(modsec_rec *msr, char **error_msg) {
     assert(msr != NULL);
     assert(error_msg != NULL);
     if (msr->mpd == NULL) return 1;
@@ -1023,38 +1029,94 @@
              * processed yet) in the buffer.
              */
             if (msr->mpd->buf_contains_line) {
-                if ( ((unsigned int)(MULTIPART_BUF_SIZE - msr->mpd->bufleft) == (4 + strlen(msr->mpd->boundary)))
+                /*
+                 * Note that the buffer may end with the final boundary followed by only CR,
+                 * coming from the [CRLF epilogue], when allow_process_partial == 1 (which is
+                 * set when SecRequestBodyLimitAction is ProcessPartial and the request body
+                 * length exceeds SecRequestBodyLimit).
+                 *
+                 * The following definitions are copied from RFC 2046:
+                 *
+                 * dash-boundary := "--" boundary
+                 *
+                 * delimiter := CRLF dash-boundary
+                 *
+                 * close-delimiter := delimiter "--"
+                 *
+                 * multipart-body := [preamble CRLF]
+                 *                   dash-boundary transport-padding CRLF
+                 *                   body-part *encapsulation
+                 *                   close-delimiter transport-padding
+                 *                   [CRLF epilogue]
+                 */
+                unsigned int buf_data_len = (unsigned int)(MULTIPART_BUF_SIZE - msr->mpd->bufleft);
+                size_t boundary_len = strnlen(msr->mpd->boundary, CONTENT_TYPE_MAX_LENGTH);
+                if ( (buf_data_len >= 2 + boundary_len)
                     && (*(msr->mpd->buf) == '-')
                     && (*(msr->mpd->buf + 1) == '-')
-                    && (strncmp(msr->mpd->buf + 2, msr->mpd->boundary, strlen(msr->mpd->boundary)) == 0)
-                    && (*(msr->mpd->buf + 2 + strlen(msr->mpd->boundary)) == '-')
-                    && (*(msr->mpd->buf + 2 + strlen(msr->mpd->boundary) + 1) == '-') )
+                    && (strncmp(msr->mpd->buf + 2, msr->mpd->boundary, boundary_len) == 0) )
                 {
-                    if ((msr->mpd->crlf_state_buf_end == 2) && (msr->mpd->flag_lf_line != 1)) {
-                        msr->mpd->flag_lf_line = 1;
-                        if (msr->mpd->flag_crlf_line) {
-                            msr_log(msr, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
-                        } else {
-                            msr_log(msr, 4, "Multipart: Warning: incorrect line endings used (LF).");
+                    if ( (buf_data_len >= 2 + boundary_len + 2)
+                        && (*(msr->mpd->buf + 2 + boundary_len) == '-')
+                        && (*(msr->mpd->buf + 2 + boundary_len + 1) == '-') )
+                    {
+                        /* If body fits in limit and ends with final boundary plus just CR, reject it. */
+                        if ( (msr->mpd->allow_process_partial == 0)
+                            && (buf_data_len == 2 + boundary_len + 2 + 1)
+                            && (*(msr->mpd->buf + 2 + boundary_len + 2) == '\r') )
+                        {
+                            *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid epilogue after final boundary.");
+                            return -1;
                         }
+
+                        if ((msr->mpd->crlf_state_buf_end == 2) && (msr->mpd->flag_lf_line != 1)) {
+                            msr->mpd->flag_lf_line = 1;
+                            if (msr->mpd->flag_crlf_line) {
+                                msr_log(msr, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
+                            } else {
+                                msr_log(msr, 4, "Multipart: Warning: incorrect line endings used (LF).");
+                            }
+                        }
+                        if (msr->mpd->mpp_substate_part_data_read == 0) {
+                            /* it looks like the final boundary, but it's where part data should begin */
+                            msr->mpd->flag_invalid_part = 1;
+                            msr_log(msr, 4, "Multipart: Warning: Invalid part (data contains final boundary)");
+                        }
+                        /* Looks like the final boundary - process it. */
+                        if (multipart_process_boundary(msr, 1 /* final */, error_msg) < 0) {
+                            msr->mpd->flag_error = 1;
+                            return -1;
+                        }
+
+                        /* The payload is complete after all. */
+                        msr->mpd->is_complete = 1;
                     }
-                    if (msr->mpd->mpp_substate_part_data_read == 0) {
-                        /* it looks like the final boundary, but it's where part data should begin */
-                        msr->mpd->flag_invalid_part = 1;
-                        msr_log(msr, 4, "Multipart: Warning: Invalid part (data contains final boundary)");
-                    }
-                    /* Looks like the final boundary - process it. */
-                    if (multipart_process_boundary(msr, 1 /* final */, error_msg) < 0) {
-                        msr->mpd->flag_error = 1;
-                        return -1;
+                    else if (msr->mpd->allow_process_partial == 1) {
+                        if (buf_data_len >= 2 + boundary_len + 1) {
+                            if (*(msr->mpd->buf + 2 + boundary_len) == '-') {
+                                if ( (buf_data_len >= 2 + boundary_len + 2)
+                                    && (*(msr->mpd->buf + 2 + boundary_len + 1) != '-') ) {
+                                    *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid final boundary.");
+                                    return -1;
+                                }
+                            }
+                            else if ( (*(msr->mpd->buf + 2 + boundary_len) != '\r')
+                                || ((buf_data_len >= 2 + boundary_len + 2)
+                                    && (*(msr->mpd->buf + 2 + boundary_len + 1) != '\n')) ) {
+                                *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid boundary.");
+                                return -1;
+                            }
+                        }
+                        /* process it as a non-final boundary to avoid building a new part. */
+                        if (multipart_process_boundary(msr, 0, error_msg) < 0) {
+                            msr->mpd->flag_error = 1;
+                            return -1;
+                        }
                     }
-
-                    /* The payload is complete after all. */
-                    msr->mpd->is_complete = 1;
                 }
             }
 
-            if (msr->mpd->is_complete == 0) {
+            if (msr->mpd->is_complete == 0 && msr->mpd->allow_process_partial == 0) {
                 *error_msg = apr_psprintf(msr->mp, "Multipart: Final boundary missing.");
                 return -1;
             }
@@ -1296,10 +1358,10 @@
             if (c == 0x0a) {
                 if (msr->mpd->crlf_state == 1) {
                     msr->mpd->crlf_state = 3;
-	        } else {
+                } else {
                     msr->mpd->crlf_state = 2;
-	        }
-	    }
+                }
+            }
             msr->mpd->crlf_state_buf_end = msr->mpd->crlf_state;
         }