diff --git a/CHANGES b/CHANGES index 2344887d8c8..0828db3fde0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ -*- coding: utf-8 -*- Changes with Apache 2.5.1 + *) mpm_event, mpm_worker, mpm_winnt, mpm_prefork: Add the optional functions + ap_mpm_note_extra_connection_added() and + ap_mpm_note_extra_connection_removed() to account for connections managed + outside the MPM accept loop (e.g., UDP). On graceful stop a child waits + for those connections no longer than max(Timeout, + GracefulShutdownTimeout), then warns and exits. + [Tarek Ibrahim ] + * mod_ssl: Add support for OpenSSL provider based certificate stores. [Graham Leggett] diff --git a/docs/log-message-tags/next-number b/docs/log-message-tags/next-number index 6c6f5d54af5..29cdba2b98b 100644 --- a/docs/log-message-tags/next-number +++ b/docs/log-message-tags/next-number @@ -1 +1 @@ -10619 +10623 diff --git a/include/ap_mmn.h b/include/ap_mmn.h index ccd504d8e82..3bc5f4de9fc 100644 --- a/include/ap_mmn.h +++ b/include/ap_mmn.h @@ -737,6 +737,9 @@ * 20211221.29 (2.5.1-dev) Add ap_set_time_process_request() to scoreboard.h * 20211221.30 (2.5.1-dev) Add ap_stat_check() to httpd.h * 20211221.31 (2.5.1-dev) Add ap_*_timingsafe() to httpd.h + * 20211221.32 (2.5.1-dev) Add the optional functions ap_mpm_note_extra_ + * connection_added() and ap_mpm_note_extra_ + * connection_removed() to mpm_common.h */ #define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */ @@ -744,7 +747,7 @@ #ifndef MODULE_MAGIC_NUMBER_MAJOR #define MODULE_MAGIC_NUMBER_MAJOR 20211221 #endif -#define MODULE_MAGIC_NUMBER_MINOR 31 /* 0...n */ +#define MODULE_MAGIC_NUMBER_MINOR 32 /* 0...n */ /** * Determine if the server's current MODULE_MAGIC_NUMBER is at least a diff --git a/include/mpm_common.h b/include/mpm_common.h index f60c473d537..cca0e74e72a 100644 --- a/include/mpm_common.h +++ b/include/mpm_common.h @@ -40,6 +40,7 @@ #include "ap_config.h" #include "ap_mpm.h" #include "scoreboard.h" +#include "apr_optional.h" #if APR_HAVE_NETINET_TCP_H #include /* for TCP_NODELAY */ @@ -560,6 +561,26 @@ AP_DECLARE_HOOK(void, child_stopped, */ void mpm_common_pre_config(apr_pool_t *pconf); +/** + * Hooks for modules to report connections the MPM did not accept itself. + * + * MPMs that wait for their connection count to drain before stopping a child + * need this so externally accepted connections keep the child alive until + * they finish. + * + * Call ap_mpm_note_extra_connection_added() when such a connection starts, + * and ap_mpm_note_extra_connection_removed() when it ends. These functions + * may be NULL if the active MPM does not implement them. + * + * A module using them is expected to notice that the child is stopping (e.g. + * with the child_stopping hook) and to end the connections it noted in a + * timely manner, gracefully or not. The MPM waits for them no longer than + * max(Timeout, GracefulShutdownTimeout), then logs a warning and exits + * anyway, possibly cutting those connections short. + */ +APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_added, (void)); +APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_removed, (void)); + #ifdef __cplusplus } #endif diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index 385532d03b4..0eff784fdda 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -188,6 +188,7 @@ static int num_listensocks = 0; static apr_int32_t conns_this_child; /* MaxConnectionsPerChild, only access in listener thread */ static apr_uint32_t connection_count = 0; /* Number of open connections */ +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static apr_uint32_t lingering_count = 0; /* Number of connections in lingering close */ static apr_uint32_t suspended_count = 0; /* Number of suspended connections */ static apr_uint32_t clogged_count = 0; /* Number of threads processing ssl conns */ @@ -871,6 +872,44 @@ static apr_status_t decrement_connection_count(void *cs_) return APR_SUCCESS; } +static void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +static void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, deadline; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + deadline = apr_time_now() + timeout; + + do { + apr_sleep(apr_time_from_msec(100)); + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && apr_time_now() < deadline); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + APLOGNO(10619) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + static void notify_suspend(event_conn_state_t *cs) { ap_run_suspend_connection(cs->c, cs->r); @@ -3139,6 +3178,10 @@ static void child_main(int child_num_arg, int child_bucket) rv == AP_MPM_PODX_GRACEFUL ? "graceful" : "ungraceful"); } + if (terminate_mode == ST_GRACEFUL) { + wait_for_extra_connections(); + } + free(threads); clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); @@ -3876,6 +3919,10 @@ static void setup_slave_conn(conn_rec *c, void *csd) event_conn_state_t *cs; mcs = ap_get_module_config(c->master->conn_config, &mpm_event_module); + if (!mcs) { + /* Master connection is not managed by this MPM; nothing to inherit. */ + return; + } cs = apr_pcalloc(c->pool, sizeof(*cs)); cs->c = c; @@ -3954,6 +4001,9 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, const char *userdata_key = "mpm_event_module"; int test_atomics = 0; + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + debug = ap_exists_config_define("DEBUG"); if (debug) { diff --git a/server/mpm/prefork/prefork.c b/server/mpm/prefork/prefork.c index 3de062f367a..2a82e09507f 100644 --- a/server/mpm/prefork/prefork.c +++ b/server/mpm/prefork/prefork.c @@ -18,6 +18,7 @@ #include "apr_portable.h" #include "apr_strings.h" #include "apr_thread_proc.h" +#include "apr_atomic.h" #include "apr_signal.h" #define APR_WANT_STDIO @@ -89,6 +90,7 @@ /* config globals */ +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static int ap_daemons_to_start=0; static int ap_daemons_min_free=0; static int ap_daemons_max_free=0; @@ -217,6 +219,44 @@ static void prefork_note_child_started(int slot, pid_t pid) ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED); } +static void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +static void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, deadline; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + deadline = apr_time_now() + timeout; + + do { + apr_sleep(apr_time_from_msec(100)); + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && apr_time_now() < deadline); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + APLOGNO(10620) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + /* a clean exit from a child with proper cleanup */ static void clean_child_exit_ex(int code, int from_signal) __attribute__ ((noreturn)); static void clean_child_exit_ex(int code, int from_signal) @@ -229,6 +269,9 @@ static void clean_child_exit_ex(int code, int from_signal) if (pchild) { if (!code && !from_signal) { ap_run_child_stopping(pchild, !retained->mpm->is_ungraceful); + if (!retained->mpm->is_ungraceful) { + wait_for_extra_connections(); + } ap_run_child_stopped(pchild, !retained->mpm->is_ungraceful); } apr_pool_destroy(pchild); @@ -1321,6 +1364,9 @@ static int prefork_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp apr_status_t rv; const char *userdata_key = "mpm_prefork_module"; + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + debug = ap_exists_config_define("DEBUG"); if (debug) { diff --git a/server/mpm/winnt/child.c b/server/mpm/winnt/child.c index 1ad8df0374b..faf90da69b1 100644 --- a/server/mpm/winnt/child.c +++ b/server/mpm/winnt/child.c @@ -131,9 +131,49 @@ static apr_thread_mutex_t *ctxpool_lock; static winnt_conn_ctx_t *ctxpool_head = NULL; static apr_uint32_t num_completion_contexts = 0; static apr_uint32_t max_num_completion_contexts = 0; +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static HANDLE ThreadDispatchIOCP = NULL; static HANDLE ctxpool_wait_event = NULL; +void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, time_remains; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + time_remains = timeout / APR_TIME_C(1000); + + do { + Sleep(100); + time_remains -= 100; + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && time_remains > 0); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, APR_SUCCESS, ap_server_conf, + APLOGNO(10622) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + static void mpm_recycle_completion_context(winnt_conn_ctx_t *context) { /* Recycle the completion context. @@ -1262,6 +1302,10 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid) ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364) "Child: All worker threads have exited."); + if (graceful_shutdown) { + wait_for_extra_connections(); + } + ap_run_child_stopped(pchild, graceful_shutdown); apr_thread_mutex_destroy(child_lock); diff --git a/server/mpm/winnt/mpm_winnt.c b/server/mpm/winnt/mpm_winnt.c index cbc090fb662..818eb276187 100644 --- a/server/mpm/winnt/mpm_winnt.c +++ b/server/mpm/winnt/mpm_winnt.c @@ -1368,6 +1368,9 @@ static int winnt_pre_config(apr_pool_t *pconf_, apr_pool_t *plog, apr_pool_t *pt * -k runservice [WinNT errors logged from rewrite_args] */ + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + /* Initialize shared static objects. * TODO: Put config related statics into an sconf structure. */ diff --git a/server/mpm/winnt/mpm_winnt.h b/server/mpm/winnt/mpm_winnt.h index 7d71a28837e..e9398e48dd2 100644 --- a/server/mpm/winnt/mpm_winnt.h +++ b/server/mpm/winnt/mpm_winnt.h @@ -93,6 +93,8 @@ void hold_console_open_on_error(void); /* From child.c: */ void child_main(apr_pool_t *pconf, DWORD parent_pid); +void ap_mpm_note_extra_connection_added(void); +void ap_mpm_note_extra_connection_removed(void); #endif /* APACHE_MPM_WINNT_H */ /** @} */ diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 42b81a8ed1b..252f732645b 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -30,6 +30,7 @@ #include "apr_thread_mutex.h" #include "apr_proc_mutex.h" #include "apr_poll.h" +#include "apr_atomic.h" #include @@ -117,6 +118,7 @@ * Actual definitions of config globals */ +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static int threads_per_child = 0; /* Worker threads per child */ static int ap_daemons_to_start = 0; static int min_spare_threads = 0; @@ -511,6 +513,44 @@ static void check_infinite_requests(void) } } +static void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +static void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, deadline; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + deadline = apr_time_now() + timeout; + + do { + apr_sleep(apr_time_from_msec(100)); + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && apr_time_now() < deadline); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + APLOGNO(10621) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + static void unblock_signal(int sig) { sigset_t sig_mask; @@ -1328,6 +1368,10 @@ static void child_main(int child_num_arg, int child_bucket) rv == AP_MPM_PODX_GRACEFUL ? ST_GRACEFUL : ST_UNGRACEFUL); } + if (terminate_mode == ST_GRACEFUL) { + wait_for_extra_connections(); + } + free(threads); clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); @@ -2103,6 +2147,9 @@ static int worker_pre_config(apr_pool_t *pconf, apr_pool_t *plog, apr_status_t rv; const char *userdata_key = "mpm_worker_module"; + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + debug = ap_exists_config_define("DEBUG"); if (debug) {