-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Expand file tree
/
Copy pathworker.c
More file actions
2555 lines (2292 loc) · 96.3 KB
/
worker.c
File metadata and controls
2555 lines (2292 loc) · 96.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* The purpose of this MPM is to fix the design flaws in the threaded
* model. Because of the way that pthreads and mutex locks interact,
* it is basically impossible to cleanly gracefully shutdown a child
* process if multiple threads are all blocked in accept. This model
* fixes those problems.
*/
#include "apr.h"
#include "apr_portable.h"
#include "apr_strings.h"
#include "apr_file_io.h"
#include "apr_thread_proc.h"
#include "apr_signal.h"
#include "apr_thread_mutex.h"
#include "apr_proc_mutex.h"
#include "apr_poll.h"
#include <stdlib.h>
#define APR_WANT_STRFUNC
#include "apr_want.h"
#if APR_HAVE_UNISTD_H
#include <unistd.h>
#endif
#if APR_HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#if APR_HAVE_SYS_WAIT_H
#include <sys/wait.h>
#endif
#ifdef HAVE_SYS_PROCESSOR_H
#include <sys/processor.h> /* for bindprocessor() */
#endif
#if !APR_HAS_THREADS
#error The Worker MPM requires APR threads, but they are unavailable.
#endif
#include "ap_config.h"
#include "httpd.h"
#include "http_main.h"
#include "http_log.h"
#include "http_config.h" /* for read_config */
#include "http_core.h" /* for get_remote_host */
#include "http_connection.h"
#include "ap_mpm.h"
#include "mpm_common.h"
#include "ap_listen.h"
#include "scoreboard.h"
#include "mpm_fdqueue.h"
#include "mpm_default.h"
#include "util_mutex.h"
#include "unixd.h"
#include "util_time.h"
#include <signal.h>
#include <limits.h> /* for INT_MAX */
/* Limit on the total --- clients will be locked out if more servers than
* this are needed. It is intended solely to keep the server from crashing
* when things get out of hand.
*
* We keep a hard maximum number of servers, for two reasons --- first off,
* in case something goes seriously wrong, we want to stop the fork bomb
* short of actually crashing the machine we're running on by filling some
* kernel table. Secondly, it keeps the size of the scoreboard file small
* enough that we can read the whole thing without worrying too much about
* the overhead.
*/
#ifndef DEFAULT_SERVER_LIMIT
#define DEFAULT_SERVER_LIMIT 16
#endif
/* Admin can't tune ServerLimit beyond MAX_SERVER_LIMIT. We want
* some sort of compile-time limit to help catch typos.
*/
#ifndef MAX_SERVER_LIMIT
#define MAX_SERVER_LIMIT 20000
#endif
/* Limit on the threads per process. Clients will be locked out if more than
* this * server_limit are needed.
*
* We keep this for one reason it keeps the size of the scoreboard file small
* enough that we can read the whole thing without worrying too much about
* the overhead.
*/
#ifndef DEFAULT_THREAD_LIMIT
#define DEFAULT_THREAD_LIMIT 64
#endif
/* Admin can't tune ThreadLimit beyond MAX_THREAD_LIMIT. We want
* some sort of compile-time limit to help catch typos.
*/
#ifndef MAX_THREAD_LIMIT
#define MAX_THREAD_LIMIT 20000
#endif
/*
* Actual definitions of config globals
*/
static int threads_per_child = 0; /* Worker threads per child */
static int ap_daemons_to_start = 0;
static int min_spare_threads = 0;
static int max_spare_threads = 0;
static int ap_daemons_limit = 0;
static int max_workers = 0;
static int server_limit = 0;
static int thread_limit = 0;
static int had_healthy_child = 0;
static volatile int dying = 0;
static int workers_may_exit = 0;
static int start_thread_may_exit = 0;
static int listener_may_exit = 0;
static int listener_is_wakeable = 0; /* Pollset supports APR_POLLSET_WAKEABLE */
static int requests_this_child;
static int num_listensocks = 0;
static int resource_shortage = 0;
static fd_queue_t *worker_queue;
static fd_queue_info_t *worker_queue_info;
static apr_pollset_t *worker_pollset;
static int idle_termination_timeout = -1; /* never terminate by default */
static int idle_termination_remaining;
typedef struct worker_child_bucket {
ap_pod_t *pod;
ap_listen_rec *listeners;
apr_proc_mutex_t *mutex;
} worker_child_bucket;
static worker_child_bucket *my_bucket; /* Current child bucket */
/* data retained by worker across load/unload of the module
* allocated on first call to pre-config hook; located on
* subsequent calls to pre-config hook
*/
typedef struct worker_retained_data {
ap_unixd_mpm_retained_data *mpm;
apr_pool_t *gen_pool; /* generation pool (children start->stop lifetime) */
worker_child_bucket *buckets; /* children buckets (reset per generation) */
int first_server_limit;
int first_thread_limit;
int sick_child_detected;
int maxclients_reported;
int near_maxclients_reported;
/*
* The max child slot ever assigned, preserved across restarts. Necessary
* to deal with MaxRequestWorkers changes across AP_SIG_GRACEFUL restarts.
* We use this value to optimize routines that have to scan the entire
* scoreboard.
*/
int max_daemons_limit;
/*
* idle_spawn_rate is the number of children that will be spawned on the
* next maintenance cycle if there aren't enough idle servers. It is
* maintained per listeners bucket, doubled up to MAX_SPAWN_RATE, and
* reset only when a cycle goes by without the need to spawn.
*/
int *idle_spawn_rate;
int hold_off_on_exponential_spawning;
int idle_timeout; /* did we time out? */
} worker_retained_data;
static worker_retained_data *retained;
#ifndef MAX_SPAWN_RATE
#define MAX_SPAWN_RATE 32
#endif
static int max_spawn_rate_per_bucket = MAX_SPAWN_RATE / 1;
#define MPM_CHILD_PID(i) (ap_scoreboard_image->parent[i].pid)
/* The structure used to pass unique initialization info to each thread */
typedef struct {
int pid;
int tid;
int sd;
} proc_info;
/* Structure used to pass information to the thread responsible for
* creating the rest of the threads.
*/
typedef struct {
apr_thread_t **threads;
apr_thread_t *listener;
int child_num_arg;
apr_threadattr_t *threadattr;
} thread_starter;
#define ID_FROM_CHILD_THREAD(c, t) ((c * thread_limit) + t)
/* The worker MPM respects a couple of runtime flags that can aid
* in debugging. Setting the -DNO_DETACH flag will prevent the root process
* from detaching from its controlling terminal. Additionally, setting
* the -DONE_PROCESS flag (which implies -DNO_DETACH) will get you the
* child_main loop running in the process which originally started up.
* This gives you a pretty nice debugging environment. (You'll get a SIGHUP
* early in standalone_main; just continue through. This is the server
* trying to kill off any child processes which it might have lying
* around --- Apache doesn't keep track of their pids, it just sends
* SIGHUP to the process group, ignoring it in the root process.
* Continue through and you'll be fine.).
*/
static int one_process = 0;
#ifdef DEBUG_SIGSTOP
int raise_sigstop_flags;
#endif
static apr_pool_t *pconf; /* Pool for config stuff */
static apr_pool_t *pchild; /* Pool for httpd child stuff */
static apr_pool_t *pruntime; /* Pool for MPM threads stuff */
static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main
thread. Use this instead */
static pid_t parent_pid;
static apr_os_thread_t *listener_os_thread;
#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
#define SAFE_ACCEPT(stmt) (ap_listeners->next ? (stmt) : APR_SUCCESS)
#else
#define SAFE_ACCEPT(stmt) (stmt)
#endif
/* The LISTENER_SIGNAL signal will be sent from the main thread to the
* listener thread to wake it up for graceful termination (what a child
* process from an old generation does when the admin does "apachectl
* graceful"). This signal will be blocked in all threads of a child
* process except for the listener thread.
*/
#define LISTENER_SIGNAL SIGHUP
/* The WORKER_SIGNAL signal will be sent from the main thread to the
* worker threads during an ungraceful restart or shutdown.
* This ensures that on systems (i.e., Linux) where closing the worker
* socket doesn't awake the worker thread when it is polling on the socket
* (especially in apr_wait_for_io_or_timeout() when handling
* Keep-Alive connections), close_worker_sockets() and join_workers()
* still function in timely manner and allow ungraceful shutdowns to
* proceed to completion. Otherwise join_workers() doesn't return
* before the main process decides the child process is non-responsive
* and sends a SIGKILL.
*/
#define WORKER_SIGNAL AP_SIG_GRACEFUL
/* An array of socket descriptors in use by each thread used to
* perform a non-graceful (forced) shutdown of the server. */
static apr_socket_t **worker_sockets;
static void close_worker_sockets(void)
{
int i;
for (i = 0; i < threads_per_child; i++) {
if (worker_sockets[i]) {
apr_socket_close(worker_sockets[i]);
worker_sockets[i] = NULL;
}
}
}
static void wakeup_listener(void)
{
listener_may_exit = 1;
/* Unblock the listener if it's poll()ing */
if (worker_pollset && listener_is_wakeable) {
apr_pollset_wakeup(worker_pollset);
}
/* unblock the listener if it's waiting for a worker */
ap_queue_info_term(worker_queue_info);
if (!listener_os_thread) {
/* XXX there is an obscure path that this doesn't handle perfectly:
* right after listener thread is created but before
* listener_os_thread is set, the first worker thread hits an
* error and starts graceful termination
*/
return;
}
/*
* we should just be able to "kill(ap_my_pid, LISTENER_SIGNAL)" on all
* platforms and wake up the listener thread since it is the only thread
* with SIGHUP unblocked, but that doesn't work on Linux
*/
#ifdef HAVE_PTHREAD_KILL
pthread_kill(*listener_os_thread, LISTENER_SIGNAL);
#else
kill(ap_my_pid, LISTENER_SIGNAL);
#endif
}
#define ST_INIT 0
#define ST_GRACEFUL 1
#define ST_UNGRACEFUL 2
static int terminate_mode = ST_INIT;
static void signal_threads(int mode)
{
if (terminate_mode == mode) {
return;
}
terminate_mode = mode;
retained->mpm->mpm_state = AP_MPMQ_STOPPING;
/* in case we weren't called from the listener thread, wake up the
* listener thread
*/
wakeup_listener();
/* for ungraceful termination, let the workers exit now;
* for graceful termination, the listener thread will notify the
* workers to exit once it has stopped accepting new connections
*/
if (mode == ST_UNGRACEFUL) {
workers_may_exit = 1;
ap_queue_interrupt_all(worker_queue);
close_worker_sockets(); /* forcefully kill all current connections */
}
ap_run_child_stopping(pchild, mode == ST_GRACEFUL);
}
static int worker_query(int query_code, int *result, apr_status_t *rv)
{
*rv = APR_SUCCESS;
switch (query_code) {
case AP_MPMQ_MAX_DAEMON_USED:
*result = retained->max_daemons_limit;
break;
case AP_MPMQ_IS_THREADED:
*result = AP_MPMQ_STATIC;
break;
case AP_MPMQ_IS_FORKED:
*result = AP_MPMQ_DYNAMIC;
break;
case AP_MPMQ_HARD_LIMIT_DAEMONS:
*result = server_limit;
break;
case AP_MPMQ_HARD_LIMIT_THREADS:
*result = thread_limit;
break;
case AP_MPMQ_MAX_THREADS:
*result = threads_per_child;
break;
case AP_MPMQ_MIN_SPARE_DAEMONS:
*result = 0;
break;
case AP_MPMQ_MIN_SPARE_THREADS:
*result = min_spare_threads;
break;
case AP_MPMQ_MAX_SPARE_DAEMONS:
*result = 0;
break;
case AP_MPMQ_MAX_SPARE_THREADS:
*result = max_spare_threads;
break;
case AP_MPMQ_MAX_REQUESTS_DAEMON:
*result = ap_max_requests_per_child;
break;
case AP_MPMQ_MAX_DAEMONS:
*result = ap_daemons_limit;
break;
case AP_MPMQ_MPM_STATE:
*result = retained->mpm->mpm_state;
break;
case AP_MPMQ_GENERATION:
*result = retained->mpm->my_generation;
break;
default:
*rv = APR_ENOTIMPL;
break;
}
return OK;
}
static void worker_note_child_killed(int childnum, pid_t pid, ap_generation_t gen)
{
if (childnum != -1) { /* child had a scoreboard slot? */
ap_run_child_status(ap_server_conf,
ap_scoreboard_image->parent[childnum].pid,
ap_scoreboard_image->parent[childnum].generation,
childnum, MPM_CHILD_EXITED);
ap_scoreboard_image->parent[childnum].pid = 0;
}
else {
ap_run_child_status(ap_server_conf, pid, gen, -1, MPM_CHILD_EXITED);
}
}
static void worker_note_child_started(int slot, pid_t pid)
{
ap_generation_t gen = retained->mpm->my_generation;
ap_scoreboard_image->parent[slot].pid = pid;
ap_scoreboard_image->parent[slot].generation = gen;
ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
}
static void worker_note_child_lost_slot(int slot, pid_t newpid)
{
ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00263)
"pid %" APR_PID_T_FMT " taking over scoreboard slot from "
"%" APR_PID_T_FMT "%s",
newpid,
ap_scoreboard_image->parent[slot].pid,
ap_scoreboard_image->parent[slot].quiescing ?
" (quiescing)" : "");
ap_run_child_status(ap_server_conf,
ap_scoreboard_image->parent[slot].pid,
ap_scoreboard_image->parent[slot].generation,
slot, MPM_CHILD_LOST_SLOT);
/* Don't forget about this exiting child process, or we
* won't be able to kill it if it doesn't exit by the
* time the server is shut down.
*/
ap_register_extra_mpm_process(ap_scoreboard_image->parent[slot].pid,
ap_scoreboard_image->parent[slot].generation);
}
static const char *worker_get_name(void)
{
return "worker";
}
/* a clean exit from a child with proper cleanup */
static void clean_child_exit(int code) __attribute__ ((noreturn));
static void clean_child_exit(int code)
{
retained->mpm->mpm_state = AP_MPMQ_STOPPING;
if (terminate_mode == ST_INIT) {
ap_run_child_stopping(pchild, 0);
}
if (pchild) {
ap_run_child_stopped(pchild, terminate_mode == ST_GRACEFUL);
apr_pool_destroy(pchild);
}
if (one_process) {
worker_note_child_killed(/* slot */ 0, 0, 0);
}
exit(code);
}
static void just_die(int sig)
{
clean_child_exit(0);
}
/*****************************************************************
* Connection structures and accounting...
*/
static int child_fatal;
/*****************************************************************
* Here follows a long bunch of generic server bookkeeping stuff...
*/
/*****************************************************************
* Child process main loop.
*/
static void process_socket(apr_thread_t *thd, apr_pool_t *p, apr_socket_t *sock,
int my_child_num,
int my_thread_num, apr_bucket_alloc_t *bucket_alloc)
{
conn_rec *current_conn;
long conn_id = ID_FROM_CHILD_THREAD(my_child_num, my_thread_num);
ap_sb_handle_t *sbh;
ap_create_sb_handle(&sbh, p, my_child_num, my_thread_num);
current_conn = ap_run_create_connection(p, ap_server_conf, sock,
conn_id, sbh, bucket_alloc);
if (current_conn) {
current_conn->current_thread = thd;
ap_process_connection(current_conn, sock);
ap_lingering_close(current_conn);
}
}
/* requests_this_child has gone to zero or below. See if the admin coded
"MaxConnectionsPerChild 0", and keep going in that case. Doing it this way
simplifies the hot path in worker_thread */
static void check_infinite_requests(void)
{
if (ap_max_requests_per_child) {
signal_threads(ST_GRACEFUL);
}
else {
requests_this_child = INT_MAX; /* keep going */
}
}
static void unblock_signal(int sig)
{
sigset_t sig_mask;
sigemptyset(&sig_mask);
sigaddset(&sig_mask, sig);
#if defined(SIGPROCMASK_SETS_THREAD_MASK)
sigprocmask(SIG_UNBLOCK, &sig_mask, NULL);
#else
pthread_sigmask(SIG_UNBLOCK, &sig_mask, NULL);
#endif
}
static void dummy_signal_handler(int sig)
{
/* XXX If specifying SIG_IGN is guaranteed to unblock a syscall,
* then we don't need this goofy function.
*/
}
static void accept_mutex_error(const char *func, apr_status_t rv, int process_slot)
{
int level = APLOG_EMERG;
if (ap_scoreboard_image->parent[process_slot].generation !=
ap_scoreboard_image->global->running_generation) {
level = APLOG_DEBUG; /* common to get these at restart time */
}
else if (requests_this_child == INT_MAX
|| ((requests_this_child == ap_max_requests_per_child)
&& ap_max_requests_per_child)) {
ap_log_error(APLOG_MARK, level, rv, ap_server_conf, APLOGNO(00272)
"apr_proc_mutex_%s failed "
"before this child process served any requests.",
func);
clean_child_exit(APEXIT_CHILDSICK);
}
ap_log_error(APLOG_MARK, level, rv, ap_server_conf, APLOGNO(00273)
"apr_proc_mutex_%s failed. Attempting to "
"shutdown process gracefully.", func);
signal_threads(ST_GRACEFUL);
}
static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy)
{
proc_info * ti = dummy;
int process_slot = ti->pid;
void *csd = NULL;
apr_pool_t *ptrans = NULL; /* Pool for per-transaction stuff */
apr_status_t rv;
ap_listen_rec *lr = NULL;
int have_idle_worker = 0;
int last_poll_idx = 0;
free(ti);
/* Unblock the signal used to wake this thread up, and set a handler for
* it.
*/
apr_signal(LISTENER_SIGNAL, dummy_signal_handler);
unblock_signal(LISTENER_SIGNAL);
/* TODO: Switch to a system where threads reuse the results from earlier
poll calls - manoj */
while (1) {
/* TODO: requests_this_child should be synchronized - aaron */
if (requests_this_child <= 0) {
check_infinite_requests();
}
if (listener_may_exit) break;
if (!have_idle_worker) {
rv = ap_queue_info_wait_for_idler(worker_queue_info, NULL);
if (APR_STATUS_IS_EOF(rv)) {
break; /* we've been signaled to die now */
}
else if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03287)
"apr_queue_info_wait failed. Attempting to "
" shutdown process gracefully.");
signal_threads(ST_GRACEFUL);
break;
}
have_idle_worker = 1;
}
/* We've already decremented the idle worker count inside
* ap_queue_info_wait_for_idler. */
if ((rv = SAFE_ACCEPT(apr_proc_mutex_lock(my_bucket->mutex)))
!= APR_SUCCESS) {
if (!listener_may_exit) {
accept_mutex_error("lock", rv, process_slot);
}
break; /* skip the lock release */
}
if (!my_bucket->listeners->next) {
/* Only one listener, so skip the poll */
lr = my_bucket->listeners;
}
else {
while (!listener_may_exit) {
apr_int32_t numdesc;
const apr_pollfd_t *pdesc;
rv = apr_pollset_poll(worker_pollset, -1, &numdesc, &pdesc);
if (rv != APR_SUCCESS) {
if (APR_STATUS_IS_EINTR(rv)) {
continue;
}
/* apr_pollset_poll() will only return errors in catastrophic
* circumstances. Let's try exiting gracefully, for now. */
ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03137)
"apr_pollset_poll: (listen)");
signal_threads(ST_GRACEFUL);
}
if (listener_may_exit) break;
/* We can always use pdesc[0], but sockets at position N
* could end up completely starved of attention in a very
* busy server. Therefore, we round-robin across the
* returned set of descriptors. While it is possible that
* the returned set of descriptors might flip around and
* continue to starve some sockets, we happen to know the
* internal pollset implementation retains ordering
* stability of the sockets. Thus, the round-robin should
* ensure that a socket will eventually be serviced.
*/
if (last_poll_idx >= numdesc)
last_poll_idx = 0;
/* Grab a listener record from the client_data of the poll
* descriptor, and advance our saved index to round-robin
* the next fetch.
*
* ### hmm... this descriptor might have POLLERR rather
* ### than POLLIN
*/
lr = pdesc[last_poll_idx++].client_data;
break;
} /* while */
} /* if/else */
if (!listener_may_exit) {
/* the following pops a recycled ptrans pool off a stack */
ap_queue_info_pop_pool(worker_queue_info, &ptrans);
if (ptrans == NULL) {
/* we can't use a recycled transaction pool this time.
* create a new transaction pool */
apr_allocator_t *allocator;
apr_allocator_create(&allocator);
apr_allocator_max_free_set(allocator, ap_max_mem_free);
apr_pool_create_ex(&ptrans, pconf, NULL, allocator);
apr_allocator_owner_set(allocator, ptrans);
apr_pool_tag(ptrans, "transaction");
}
rv = lr->accept_func(&csd, lr, ptrans);
/* later we trash rv and rely on csd to indicate success/failure */
AP_DEBUG_ASSERT(rv == APR_SUCCESS || !csd);
if (rv == APR_EGENERAL) {
/* E[NM]FILE, ENOMEM, etc */
resource_shortage = 1;
signal_threads(ST_GRACEFUL);
}
else if (ap_accept_error_is_nonfatal(rv)) {
ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, ap_server_conf,
"accept() on client socket failed");
}
if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(my_bucket->mutex)))
!= APR_SUCCESS) {
if (listener_may_exit) {
break;
}
accept_mutex_error("unlock", rv, process_slot);
}
if (csd != NULL) {
rv = ap_queue_push_socket(worker_queue, csd, NULL, ptrans);
if (rv) {
/* trash the connection; we couldn't queue the connected
* socket to a worker
*/
apr_socket_close(csd);
ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf, APLOGNO(03138)
"ap_queue_push_socket failed");
}
else {
have_idle_worker = 0;
}
}
}
else {
if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(my_bucket->mutex)))
!= APR_SUCCESS) {
int level = APLOG_EMERG;
if (ap_scoreboard_image->parent[process_slot].generation !=
ap_scoreboard_image->global->running_generation) {
level = APLOG_DEBUG; /* common to get these at restart time */
}
ap_log_error(APLOG_MARK, level, rv, ap_server_conf, APLOGNO(00274)
"apr_proc_mutex_unlock failed. Attempting to "
"shutdown process gracefully.");
signal_threads(ST_GRACEFUL);
}
break;
}
}
ap_close_listeners_ex(my_bucket->listeners);
ap_queue_info_free_idle_pools(worker_queue_info);
ap_queue_term(worker_queue);
dying = 1;
ap_scoreboard_image->parent[process_slot].quiescing = 1;
/* wake up the main thread */
kill(ap_my_pid, SIGTERM);
apr_thread_exit(thd, APR_SUCCESS);
return NULL;
}
/* XXX For ungraceful termination/restart, we definitely don't want to
* wait for active connections to finish but we may want to wait
* for idle workers to get out of the queue code and release mutexes,
* since those mutexes are cleaned up pretty soon and some systems
* may not react favorably (i.e., segfault) if operations are attempted
* on cleaned-up mutexes.
*/
static void * APR_THREAD_FUNC worker_thread(apr_thread_t *thd, void * dummy)
{
proc_info * ti = dummy;
int process_slot = ti->pid;
int thread_slot = ti->tid;
apr_socket_t *csd = NULL;
apr_bucket_alloc_t *bucket_alloc;
apr_pool_t *last_ptrans = NULL;
apr_pool_t *ptrans; /* Pool for per-transaction stuff */
apr_status_t rv;
int is_idle = 0;
free(ti);
ap_scoreboard_image->servers[process_slot][thread_slot].pid = ap_my_pid;
ap_scoreboard_image->servers[process_slot][thread_slot].tid = apr_os_thread_current();
ap_scoreboard_image->servers[process_slot][thread_slot].generation = retained->mpm->my_generation;
ap_update_child_status_from_indexes(process_slot, thread_slot,
SERVER_STARTING, NULL);
#ifdef HAVE_PTHREAD_KILL
apr_signal(WORKER_SIGNAL, dummy_signal_handler);
unblock_signal(WORKER_SIGNAL);
#endif
while (!workers_may_exit) {
if (!is_idle) {
rv = ap_queue_info_set_idle(worker_queue_info, last_ptrans);
last_ptrans = NULL;
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03288)
"ap_queue_info_set_idle failed. Attempting to "
"shutdown process gracefully.");
signal_threads(ST_GRACEFUL);
break;
}
is_idle = 1;
}
ap_update_child_status_from_indexes(process_slot, thread_slot,
SERVER_READY, NULL);
worker_pop:
if (workers_may_exit) {
break;
}
rv = ap_queue_pop_socket(worker_queue, &csd, &ptrans);
if (rv != APR_SUCCESS) {
/* We get APR_EOF during a graceful shutdown once all the connections
* accepted by this server process have been handled.
*/
if (APR_STATUS_IS_EOF(rv)) {
break;
}
/* We get APR_EINTR whenever ap_queue_pop_*() has been interrupted
* from an explicit call to ap_queue_interrupt_all(). This allows
* us to unblock threads stuck in ap_queue_pop_*() when a shutdown
* is pending.
*
* If workers_may_exit is set and this is ungraceful termination/
* restart, we are bound to get an error on some systems (e.g.,
* AIX, which sanity-checks mutex operations) since the queue
* may have already been cleaned up. Don't log the "error" if
* workers_may_exit is set.
*/
else if (APR_STATUS_IS_EINTR(rv)) {
goto worker_pop;
}
/* We got some other error. */
else if (!workers_may_exit) {
ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf, APLOGNO(03139)
"ap_queue_pop_socket failed");
}
continue;
}
is_idle = 0;
worker_sockets[thread_slot] = csd;
bucket_alloc = apr_bucket_alloc_create(ptrans);
process_socket(thd, ptrans, csd, process_slot, thread_slot, bucket_alloc);
worker_sockets[thread_slot] = NULL;
requests_this_child--;
apr_pool_clear(ptrans);
last_ptrans = ptrans;
}
ap_update_child_status_from_indexes(process_slot, thread_slot,
dying ? SERVER_DEAD
: SERVER_GRACEFUL, NULL);
apr_thread_exit(thd, APR_SUCCESS);
return NULL;
}
static int check_signal(int signum)
{
switch (signum) {
case SIGTERM:
case SIGINT:
return 1;
}
return 0;
}
static void create_listener_thread(thread_starter *ts)
{
int my_child_num = ts->child_num_arg;
apr_threadattr_t *thread_attr = ts->threadattr;
proc_info *my_info;
apr_status_t rv;
my_info = (proc_info *)ap_malloc(sizeof(proc_info));
my_info->pid = my_child_num;
my_info->tid = -1; /* listener thread doesn't have a thread slot */
my_info->sd = 0;
rv = ap_thread_create(&ts->listener, thread_attr, listener_thread,
my_info, pruntime);
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00275)
"ap_thread_create: unable to create listener thread");
/* let the parent decide how bad this really is */
clean_child_exit(APEXIT_CHILDSICK);
}
apr_os_thread_get(&listener_os_thread, ts->listener);
}
static void setup_threads_runtime(void)
{
ap_listen_rec *lr;
int pollset_flags;
apr_status_t rv;
/* All threads (listener, workers) and synchronization objects (queues,
* pollset, mutexes...) created here should have at least the lifetime of
* the connections they handle (i.e. ptrans). We can't use this thread's
* self pool because all these objects survive it, nor use pchild or pconf
* directly because this starter thread races with other modules' runtime,
* nor finally pchild (or subpool thereof) because it is killed explicitly
* before pconf (thus connections/ptrans can live longer, which matters in
* ONE_PROCESS mode). So this leaves us with a subpool of pconf, created
* before any ptrans hence destroyed after.
*/
apr_pool_create(&pruntime, pconf);
apr_pool_tag(pruntime, "mpm_runtime");
/* We must create the fd queues before we start up the listener
* and worker threads. */
rv = ap_queue_create(&worker_queue, threads_per_child, pruntime);
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03140)
"ap_queue_create() failed");
clean_child_exit(APEXIT_CHILDFATAL);
}
rv = ap_queue_info_create(&worker_queue_info, pruntime,
threads_per_child, -1);
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03141)
"ap_queue_info_create() failed");
clean_child_exit(APEXIT_CHILDFATAL);
}
/* Create the main pollset. When APR_POLLSET_WAKEABLE is asked we account
* for the wakeup pipe explicitely with num_listensocks+1 because some
* pollset implementations don't do it implicitely in APR.
*/
pollset_flags = APR_POLLSET_NOCOPY | APR_POLLSET_WAKEABLE;
rv = apr_pollset_create(&worker_pollset, num_listensocks + 1, pruntime,
pollset_flags);
if (rv == APR_SUCCESS) {
listener_is_wakeable = 1;
}
else {
pollset_flags &= ~APR_POLLSET_WAKEABLE;
rv = apr_pollset_create(&worker_pollset, num_listensocks, pruntime,
pollset_flags);
}
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03285)
"Couldn't create pollset in thread;"
" check system or user limits");
/* let the parent decide how bad this really is */
clean_child_exit(APEXIT_CHILDSICK);
}
for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) {
apr_pollfd_t *pfd = apr_pcalloc(pruntime, sizeof *pfd);
pfd->desc_type = APR_POLL_SOCKET;
pfd->desc.s = lr->sd;
pfd->reqevents = APR_POLLIN;
pfd->client_data = lr;
rv = apr_pollset_add(worker_pollset, pfd);
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03286)
"Couldn't create add listener to pollset;"
" check system or user limits");
/* let the parent decide how bad this really is */
clean_child_exit(APEXIT_CHILDSICK);
}
lr->accept_func = ap_unixd_accept;
}
worker_sockets = apr_pcalloc(pruntime, threads_per_child *
sizeof(apr_socket_t *));
}
/* XXX under some circumstances not understood, children can get stuck
* in start_threads forever trying to take over slots which will
* never be cleaned up; for now there is an APLOG_DEBUG message issued
* every so often when this condition occurs
*/
static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
{
thread_starter *ts = dummy;
apr_thread_t **threads = ts->threads;
apr_threadattr_t *thread_attr = ts->threadattr;
int my_child_num = ts->child_num_arg;
proc_info *my_info;
apr_status_t rv;
int threads_created = 0;
int listener_started = 0;
int prev_threads_created;
int loops, i;
loops = prev_threads_created = 0;
while (1) {
/* threads_per_child does not include the listener thread */
for (i = 0; i < threads_per_child; i++) {
int status = ap_scoreboard_image->servers[my_child_num][i].status;
if (status != SERVER_GRACEFUL && status != SERVER_DEAD) {
continue;
}
my_info = (proc_info *)ap_malloc(sizeof(proc_info));
my_info->pid = my_child_num;
my_info->tid = i;
my_info->sd = 0;
/* We are creating threads right now */
ap_update_child_status_from_indexes(my_child_num, i,
SERVER_STARTING, NULL);
/* We let each thread update its own scoreboard entry. This is