allow mpm to be note connections that are not TCP based.

CHANGES

@@ -1,6 +1,14 @@
                                                          -*- coding: utf-8 -*-
 Changes with Apache 2.5.1
 
+  *) mpm_event, mpm_worker, mpm_winnt, mpm_prefork: Add the optional functions
+     ap_mpm_note_extra_connection_added() and
+     ap_mpm_note_extra_connection_removed() to account for connections managed
+     outside the MPM accept loop (e.g., UDP).  On graceful stop a child waits
+     for those connections no longer than max(Timeout,
+     GracefulShutdownTimeout), then warns and exits.
+     [Tarek Ibrahim <tareki@pulsarxtech.com> <t1br4h1m@gmail.com>]
+
   * mod_ssl: Add support for OpenSSL provider based certificate
     stores. [Graham Leggett]
 

docs/log-message-tags/next-number

@@ -1 +1 @@
-10619
+10623

include/ap_mmn.h

@@ -737,14 +737,17 @@
  * 20211221.29 (2.5.1-dev) Add ap_set_time_process_request() to scoreboard.h
  * 20211221.30 (2.5.1-dev) Add ap_stat_check() to httpd.h
  * 20211221.31 (2.5.1-dev) Add ap_*_timingsafe() to httpd.h
+ * 20211221.32 (2.5.1-dev) Add the optional functions ap_mpm_note_extra_
+ *                         connection_added() and ap_mpm_note_extra_
+ *                         connection_removed() to mpm_common.h
  */
 
 #define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */
 
 #ifndef MODULE_MAGIC_NUMBER_MAJOR
 #define MODULE_MAGIC_NUMBER_MAJOR 20211221
 #endif
-#define MODULE_MAGIC_NUMBER_MINOR 31             /* 0...n */
+#define MODULE_MAGIC_NUMBER_MINOR 32            /* 0...n */
 
 /**
  * Determine if the server's current MODULE_MAGIC_NUMBER is at least a

include/mpm_common.h

@@ -40,6 +40,7 @@
 #include "ap_config.h"
 #include "ap_mpm.h"
 #include "scoreboard.h"
+#include "apr_optional.h"
 
 #if APR_HAVE_NETINET_TCP_H
 #include <netinet/tcp.h>    /* for TCP_NODELAY */
@@ -560,6 +561,26 @@ AP_DECLARE_HOOK(void, child_stopped,
  */
 void mpm_common_pre_config(apr_pool_t *pconf);
 
+/**
+ * Hooks for modules to report connections the MPM did not accept itself.
+ *
+ * MPMs that wait for their connection count to drain before stopping a child
+ * need this so externally accepted connections keep the child alive until
+ * they finish.
+ *
+ * Call ap_mpm_note_extra_connection_added() when such a connection starts,
+ * and ap_mpm_note_extra_connection_removed() when it ends. These functions
+ * may be NULL if the active MPM does not implement them.
+ *
+ * A module using them is expected to notice that the child is stopping (e.g.
+ * with the child_stopping hook) and to end the connections it noted in a
+ * timely manner, gracefully or not.  The MPM waits for them no longer than
+ * max(Timeout, GracefulShutdownTimeout), then logs a warning and exits
+ * anyway, possibly cutting those connections short.
+ */
+APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_added, (void));
+APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_removed, (void));
+
 #ifdef __cplusplus
 }
 #endif

server/mpm/event/event.c

@@ -188,6 +188,7 @@ static int num_listensocks = 0;
 static apr_int32_t conns_this_child;        /* MaxConnectionsPerChild, only access
                                                in listener thread */
 static apr_uint32_t connection_count = 0;   /* Number of open connections */
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static apr_uint32_t lingering_count = 0;    /* Number of connections in lingering close */
 static apr_uint32_t suspended_count = 0;    /* Number of suspended connections */
 static apr_uint32_t clogged_count = 0;      /* Number of threads processing ssl conns */
@@ -871,6 +872,44 @@ static apr_status_t decrement_connection_count(void *cs_)
     return APR_SUCCESS;
 }
 
+static void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+static void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10619)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void notify_suspend(event_conn_state_t *cs)
 {
     ap_run_suspend_connection(cs->c, cs->r);
@@ -3139,6 +3178,10 @@ static void child_main(int child_num_arg, int child_bucket)
                      rv == AP_MPM_PODX_GRACEFUL ? "graceful" : "ungraceful");
     }
 
+    if (terminate_mode == ST_GRACEFUL) {
+        wait_for_extra_connections();
+    }
+
     free(threads);
 
     clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0);
@@ -3876,6 +3919,10 @@ static void setup_slave_conn(conn_rec *c, void *csd)
     event_conn_state_t *cs;
 
     mcs = ap_get_module_config(c->master->conn_config, &mpm_event_module);
+    if (!mcs) {
+        /* Master connection is not managed by this MPM; nothing to inherit. */
+        return;
+    }
 
     cs = apr_pcalloc(c->pool, sizeof(*cs));
     cs->c = c;
@@ -3954,6 +4001,9 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog,
     const char *userdata_key = "mpm_event_module";
     int test_atomics = 0;
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     debug = ap_exists_config_define("DEBUG");
 
     if (debug) {

server/mpm/prefork/prefork.c

@@ -18,6 +18,7 @@
 #include "apr_portable.h"
 #include "apr_strings.h"
 #include "apr_thread_proc.h"
+#include "apr_atomic.h"
 #include "apr_signal.h"
 
 #define APR_WANT_STDIO
@@ -89,6 +90,7 @@
 
 /* config globals */
 
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static int ap_daemons_to_start=0;
 static int ap_daemons_min_free=0;
 static int ap_daemons_max_free=0;
@@ -217,6 +219,44 @@ static void prefork_note_child_started(int slot, pid_t pid)
     ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
 }
 
+static void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+static void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10620)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 /* a clean exit from a child with proper cleanup */
 static void clean_child_exit_ex(int code, int from_signal) __attribute__ ((noreturn));
 static void clean_child_exit_ex(int code, int from_signal)
@@ -229,6 +269,9 @@ static void clean_child_exit_ex(int code, int from_signal)
     if (pchild) {
         if (!code && !from_signal) {
             ap_run_child_stopping(pchild, !retained->mpm->is_ungraceful);
+            if (!retained->mpm->is_ungraceful) {
+                wait_for_extra_connections();
+            }
             ap_run_child_stopped(pchild, !retained->mpm->is_ungraceful);
         }
         apr_pool_destroy(pchild);
@@ -1321,6 +1364,9 @@ static int prefork_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp
     apr_status_t rv;
     const char *userdata_key = "mpm_prefork_module";
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     debug = ap_exists_config_define("DEBUG");
 
     if (debug) {

server/mpm/winnt/child.c

@@ -131,9 +131,49 @@ static apr_thread_mutex_t  *ctxpool_lock;
 static winnt_conn_ctx_t *ctxpool_head = NULL;
 static apr_uint32_t num_completion_contexts = 0;
 static apr_uint32_t max_num_completion_contexts = 0;
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static HANDLE ThreadDispatchIOCP = NULL;
 static HANDLE ctxpool_wait_event = NULL;
 
+void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, time_remains;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    time_remains = timeout / APR_TIME_C(1000);
+
+    do {
+        Sleep(100);
+        time_remains -= 100;
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && time_remains > 0);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, APR_SUCCESS, ap_server_conf,
+                     APLOGNO(10622)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void mpm_recycle_completion_context(winnt_conn_ctx_t *context)
 {
     /* Recycle the completion context.
@@ -1262,6 +1302,10 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
     ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364)
                  "Child: All worker threads have exited.");
 
+    if (graceful_shutdown) {
+        wait_for_extra_connections();
+    }
+
     ap_run_child_stopped(pchild, graceful_shutdown);
 
     apr_thread_mutex_destroy(child_lock);

server/mpm/winnt/mpm_winnt.c

@@ -1368,6 +1368,9 @@ static int winnt_pre_config(apr_pool_t *pconf_, apr_pool_t *plog, apr_pool_t *pt
      *   -k runservice [WinNT errors logged from rewrite_args]
      */
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     /* Initialize shared static objects.
      * TODO: Put config related statics into an sconf structure.
      */

server/mpm/winnt/mpm_winnt.h

@@ -93,6 +93,8 @@ void hold_console_open_on_error(void);
 
 /* From child.c: */
 void child_main(apr_pool_t *pconf, DWORD parent_pid);
+void ap_mpm_note_extra_connection_added(void);
+void ap_mpm_note_extra_connection_removed(void);
 
 #endif /* APACHE_MPM_WINNT_H */
 /** @} */

server/mpm/worker/worker.c

@@ -30,6 +30,7 @@
 #include "apr_thread_mutex.h"
 #include "apr_proc_mutex.h"
 #include "apr_poll.h"
+#include "apr_atomic.h"
 
 #include <stdlib.h>
 
@@ -117,6 +118,7 @@
  * Actual definitions of config globals
  */
 
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static int threads_per_child = 0;     /* Worker threads per child */
 static int ap_daemons_to_start = 0;
 static int min_spare_threads = 0;
@@ -511,6 +513,44 @@ static void check_infinite_requests(void)
     }
 }
 
+static void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+static void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10621)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void unblock_signal(int sig)
 {
     sigset_t sig_mask;
@@ -1328,6 +1368,10 @@ static void child_main(int child_num_arg, int child_bucket)
                      rv == AP_MPM_PODX_GRACEFUL ? ST_GRACEFUL : ST_UNGRACEFUL);
     }
 
+    if (terminate_mode == ST_GRACEFUL) {
+        wait_for_extra_connections();
+    }
+
     free(threads);
 
     clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0);
@@ -2103,6 +2147,9 @@ static int worker_pre_config(apr_pool_t *pconf, apr_pool_t *plog,
     apr_status_t rv;
     const char *userdata_key = "mpm_worker_module";
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     debug = ap_exists_config_define("DEBUG");
 
     if (debug) {