Groonga 3.0.9 Source Code Document
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
ngx_epoll_module.c
Go to the documentation of this file.
1 
2 /*
3  * Copyright (C) Igor Sysoev
4  * Copyright (C) Nginx, Inc.
5  */
6 
7 
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_event.h>
11 
12 
13 #if (NGX_TEST_BUILD_EPOLL)
14 
15 /* epoll declarations */
16 
17 #define EPOLLIN 0x001
18 #define EPOLLPRI 0x002
19 #define EPOLLOUT 0x004
20 #define EPOLLRDNORM 0x040
21 #define EPOLLRDBAND 0x080
22 #define EPOLLWRNORM 0x100
23 #define EPOLLWRBAND 0x200
24 #define EPOLLMSG 0x400
25 #define EPOLLERR 0x008
26 #define EPOLLHUP 0x010
27 
28 #define EPOLLET 0x80000000
29 #define EPOLLONESHOT 0x40000000
30 
31 #define EPOLL_CTL_ADD 1
32 #define EPOLL_CTL_DEL 2
33 #define EPOLL_CTL_MOD 3
34 
35 typedef union epoll_data {
36  void *ptr;
37  int fd;
38  uint32_t u32;
39  uint64_t u64;
40 } epoll_data_t;
41 
42 struct epoll_event {
43  uint32_t events;
44  epoll_data_t data;
45 };
46 
47 
48 int epoll_create(int size);
49 
50 int epoll_create(int size)
51 {
52  return -1;
53 }
54 
55 
56 int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
57 
58 int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
59 {
60  return -1;
61 }
62 
63 
64 int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout);
65 
66 int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout)
67 {
68  return -1;
69 }
70 
71 #if (NGX_HAVE_FILE_AIO)
72 
73 #define SYS_io_setup 245
74 #define SYS_io_destroy 246
75 #define SYS_io_getevents 247
76 #define SYS_eventfd 323
77 
78 typedef u_int aio_context_t;
79 
80 struct io_event {
81  uint64_t data; /* the data field from the iocb */
82  uint64_t obj; /* what iocb this event came from */
83  int64_t res; /* result code for this event */
84  int64_t res2; /* secondary result */
85 };
86 
87 
88 #endif
89 #endif
90 
91 
92 typedef struct {
96 
97 
98 static ngx_int_t ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer);
99 static void ngx_epoll_done(ngx_cycle_t *cycle);
100 static ngx_int_t ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event,
101  ngx_uint_t flags);
102 static ngx_int_t ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event,
103  ngx_uint_t flags);
104 static ngx_int_t ngx_epoll_add_connection(ngx_connection_t *c);
105 static ngx_int_t ngx_epoll_del_connection(ngx_connection_t *c,
106  ngx_uint_t flags);
107 static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
108  ngx_uint_t flags);
109 
110 #if (NGX_HAVE_FILE_AIO)
111 static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
112 #endif
113 
114 static void *ngx_epoll_create_conf(ngx_cycle_t *cycle);
115 static char *ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf);
116 
117 static int ep = -1;
118 static struct epoll_event *event_list;
119 static ngx_uint_t nevents;
120 
121 #if (NGX_HAVE_FILE_AIO)
122 
123 int ngx_eventfd = -1;
124 aio_context_t ngx_aio_ctx = 0;
125 
126 static ngx_event_t ngx_eventfd_event;
127 static ngx_connection_t ngx_eventfd_conn;
128 
129 #endif
130 
131 static ngx_str_t epoll_name = ngx_string("epoll");
132 
133 static ngx_command_t ngx_epoll_commands[] = {
134 
135  { ngx_string("epoll_events"),
138  0,
139  offsetof(ngx_epoll_conf_t, events),
140  NULL },
141 
142  { ngx_string("worker_aio_requests"),
145  0,
146  offsetof(ngx_epoll_conf_t, aio_requests),
147  NULL },
148 
150 };
151 
152 
154  &epoll_name,
155  ngx_epoll_create_conf, /* create configuration */
156  ngx_epoll_init_conf, /* init configuration */
157 
158  {
159  ngx_epoll_add_event, /* add an event */
160  ngx_epoll_del_event, /* delete an event */
161  ngx_epoll_add_event, /* enable an event */
162  ngx_epoll_del_event, /* disable an event */
163  ngx_epoll_add_connection, /* add an connection */
164  ngx_epoll_del_connection, /* delete an connection */
165  NULL, /* process the changes */
166  ngx_epoll_process_events, /* process the events */
167  ngx_epoll_init, /* init the events */
168  ngx_epoll_done, /* done the events */
169  }
170 };
171 
174  &ngx_epoll_module_ctx, /* module context */
175  ngx_epoll_commands, /* module directives */
176  NGX_EVENT_MODULE, /* module type */
177  NULL, /* init master */
178  NULL, /* init module */
179  NULL, /* init process */
180  NULL, /* init thread */
181  NULL, /* exit thread */
182  NULL, /* exit process */
183  NULL, /* exit master */
185 };
186 
187 
188 #if (NGX_HAVE_FILE_AIO)
189 
190 /*
191  * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly
192  * as syscalls instead of libaio usage, because the library header file
193  * supports eventfd() since 0.3.107 version only.
194  *
195  * Also we do not use eventfd() in glibc, because glibc supports it
196  * since 2.8 version and glibc maps two syscalls eventfd() and eventfd2()
197  * into single eventfd() function with different number of parameters.
198  */
199 
200 static int
201 io_setup(u_int nr_reqs, aio_context_t *ctx)
202 {
203  return syscall(SYS_io_setup, nr_reqs, ctx);
204 }
205 
206 
207 static int
208 io_destroy(aio_context_t ctx)
209 {
210  return syscall(SYS_io_destroy, ctx);
211 }
212 
213 
214 static int
215 io_getevents(aio_context_t ctx, long min_nr, long nr, struct io_event *events,
216  struct timespec *tmo)
217 {
218  return syscall(SYS_io_getevents, ctx, min_nr, nr, events, tmo);
219 }
220 
221 
222 static void
223 ngx_epoll_aio_init(ngx_cycle_t *cycle, ngx_epoll_conf_t *epcf)
224 {
225  int n;
226  struct epoll_event ee;
227 
228  ngx_eventfd = syscall(SYS_eventfd, 0);
229 
230  if (ngx_eventfd == -1) {
232  "eventfd() failed");
233  ngx_file_aio = 0;
234  return;
235  }
236 
238  "eventfd: %d", ngx_eventfd);
239 
240  n = 1;
241 
242  if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) {
244  "ioctl(eventfd, FIONBIO) failed");
245  goto failed;
246  }
247 
248  if (io_setup(epcf->aio_requests, &ngx_aio_ctx) == -1) {
250  "io_setup() failed");
251  goto failed;
252  }
253 
254  ngx_eventfd_event.data = &ngx_eventfd_conn;
255  ngx_eventfd_event.handler = ngx_epoll_eventfd_handler;
256  ngx_eventfd_event.log = cycle->log;
257  ngx_eventfd_event.active = 1;
258  ngx_eventfd_conn.fd = ngx_eventfd;
259  ngx_eventfd_conn.read = &ngx_eventfd_event;
260  ngx_eventfd_conn.log = cycle->log;
261 
262  ee.events = EPOLLIN|EPOLLET;
263  ee.data.ptr = &ngx_eventfd_conn;
264 
265  if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) {
266  return;
267  }
268 
270  "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
271 
272  if (io_destroy(ngx_aio_ctx) == -1) {
274  "io_destroy() failed");
275  }
276 
277 failed:
278 
279  if (close(ngx_eventfd) == -1) {
281  "eventfd close() failed");
282  }
283 
284  ngx_eventfd = -1;
285  ngx_aio_ctx = 0;
286  ngx_file_aio = 0;
287 }
288 
289 #endif
290 
291 
292 static ngx_int_t
293 ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
294 {
295  ngx_epoll_conf_t *epcf;
296 
297  epcf = ngx_event_get_conf(cycle->conf_ctx, ngx_epoll_module);
298 
299  if (ep == -1) {
300  ep = epoll_create(cycle->connection_n / 2);
301 
302  if (ep == -1) {
304  "epoll_create() failed");
305  return NGX_ERROR;
306  }
307 
308 #if (NGX_HAVE_FILE_AIO)
309 
310  ngx_epoll_aio_init(cycle, epcf);
311 
312 #endif
313  }
314 
315  if (nevents < epcf->events) {
316  if (event_list) {
317  ngx_free(event_list);
318  }
319 
320  event_list = ngx_alloc(sizeof(struct epoll_event) * epcf->events,
321  cycle->log);
322  if (event_list == NULL) {
323  return NGX_ERROR;
324  }
325  }
326 
327  nevents = epcf->events;
328 
329  ngx_io = ngx_os_io;
330 
331  ngx_event_actions = ngx_epoll_module_ctx.actions;
332 
333 #if (NGX_HAVE_CLEAR_EVENT)
335 #else
337 #endif
340 
341  return NGX_OK;
342 }
343 
344 
345 static void
346 ngx_epoll_done(ngx_cycle_t *cycle)
347 {
348  if (close(ep) == -1) {
350  "epoll close() failed");
351  }
352 
353  ep = -1;
354 
355 #if (NGX_HAVE_FILE_AIO)
356 
357  if (ngx_eventfd != -1) {
358 
359  if (io_destroy(ngx_aio_ctx) == -1) {
361  "io_destroy() failed");
362  }
363 
364  if (close(ngx_eventfd) == -1) {
366  "eventfd close() failed");
367  }
368 
369  ngx_eventfd = -1;
370  }
371 
372  ngx_aio_ctx = 0;
373 
374 #endif
375 
376  ngx_free(event_list);
377 
378  event_list = NULL;
379  nevents = 0;
380 }
381 
382 
383 static ngx_int_t
384 ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
385 {
386  int op;
387  uint32_t events, prev;
388  ngx_event_t *e;
389  ngx_connection_t *c;
390  struct epoll_event ee;
391 
392  c = ev->data;
393 
394  events = (uint32_t) event;
395 
396  if (event == NGX_READ_EVENT) {
397  e = c->write;
398  prev = EPOLLOUT;
399 #if (NGX_READ_EVENT != EPOLLIN)
400  events = EPOLLIN;
401 #endif
402 
403  } else {
404  e = c->read;
405  prev = EPOLLIN;
406 #if (NGX_WRITE_EVENT != EPOLLOUT)
407  events = EPOLLOUT;
408 #endif
409  }
410 
411  if (e->active) {
412  op = EPOLL_CTL_MOD;
413  events |= prev;
414 
415  } else {
416  op = EPOLL_CTL_ADD;
417  }
418 
419  ee.events = events | (uint32_t) flags;
420  ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
421 
423  "epoll add event: fd:%d op:%d ev:%08XD",
424  c->fd, op, ee.events);
425 
426  if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
428  "epoll_ctl(%d, %d) failed", op, c->fd);
429  return NGX_ERROR;
430  }
431 
432  ev->active = 1;
433 #if 0
434  ev->oneshot = (flags & NGX_ONESHOT_EVENT) ? 1 : 0;
435 #endif
436 
437  return NGX_OK;
438 }
439 
440 
441 static ngx_int_t
442 ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
443 {
444  int op;
445  uint32_t prev;
446  ngx_event_t *e;
447  ngx_connection_t *c;
448  struct epoll_event ee;
449 
450  /*
451  * when the file descriptor is closed, the epoll automatically deletes
452  * it from its queue, so we do not need to delete explicitly the event
453  * before the closing the file descriptor
454  */
455 
456  if (flags & NGX_CLOSE_EVENT) {
457  ev->active = 0;
458  return NGX_OK;
459  }
460 
461  c = ev->data;
462 
463  if (event == NGX_READ_EVENT) {
464  e = c->write;
465  prev = EPOLLOUT;
466 
467  } else {
468  e = c->read;
469  prev = EPOLLIN;
470  }
471 
472  if (e->active) {
473  op = EPOLL_CTL_MOD;
474  ee.events = prev | (uint32_t) flags;
475  ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
476 
477  } else {
478  op = EPOLL_CTL_DEL;
479  ee.events = 0;
480  ee.data.ptr = NULL;
481  }
482 
484  "epoll del event: fd:%d op:%d ev:%08XD",
485  c->fd, op, ee.events);
486 
487  if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
489  "epoll_ctl(%d, %d) failed", op, c->fd);
490  return NGX_ERROR;
491  }
492 
493  ev->active = 0;
494 
495  return NGX_OK;
496 }
497 
498 
499 static ngx_int_t
500 ngx_epoll_add_connection(ngx_connection_t *c)
501 {
502  struct epoll_event ee;
503 
504  ee.events = EPOLLIN|EPOLLOUT|EPOLLET;
505  ee.data.ptr = (void *) ((uintptr_t) c | c->read->instance);
506 
508  "epoll add connection: fd:%d ev:%08XD", c->fd, ee.events);
509 
510  if (epoll_ctl(ep, EPOLL_CTL_ADD, c->fd, &ee) == -1) {
512  "epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->fd);
513  return NGX_ERROR;
514  }
515 
516  c->read->active = 1;
517  c->write->active = 1;
518 
519  return NGX_OK;
520 }
521 
522 
523 static ngx_int_t
524 ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags)
525 {
526  int op;
527  struct epoll_event ee;
528 
529  /*
530  * when the file descriptor is closed the epoll automatically deletes
531  * it from its queue so we do not need to delete explicitly the event
532  * before the closing the file descriptor
533  */
534 
535  if (flags & NGX_CLOSE_EVENT) {
536  c->read->active = 0;
537  c->write->active = 0;
538  return NGX_OK;
539  }
540 
542  "epoll del connection: fd:%d", c->fd);
543 
544  op = EPOLL_CTL_DEL;
545  ee.events = 0;
546  ee.data.ptr = NULL;
547 
548  if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
550  "epoll_ctl(%d, %d) failed", op, c->fd);
551  return NGX_ERROR;
552  }
553 
554  c->read->active = 0;
555  c->write->active = 0;
556 
557  return NGX_OK;
558 }
559 
560 
561 static ngx_int_t
562 ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
563 {
564  int events;
565  uint32_t revents;
566  ngx_int_t instance, i;
567  ngx_uint_t level;
568  ngx_err_t err;
569  ngx_event_t *rev, *wev, **queue;
570  ngx_connection_t *c;
571 
572  /* NGX_TIMER_INFINITE == INFTIM */
573 
575  "epoll timer: %M", timer);
576 
577  events = epoll_wait(ep, event_list, (int) nevents, timer);
578 
579  err = (events == -1) ? ngx_errno : 0;
580 
581  if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
582  ngx_time_update();
583  }
584 
585  if (err) {
586  if (err == NGX_EINTR) {
587 
588  if (ngx_event_timer_alarm) {
590  return NGX_OK;
591  }
592 
593  level = NGX_LOG_INFO;
594 
595  } else {
596  level = NGX_LOG_ALERT;
597  }
598 
599  ngx_log_error(level, cycle->log, err, "epoll_wait() failed");
600  return NGX_ERROR;
601  }
602 
603  if (events == 0) {
604  if (timer != NGX_TIMER_INFINITE) {
605  return NGX_OK;
606  }
607 
608  ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
609  "epoll_wait() returned no events without timeout");
610  return NGX_ERROR;
611  }
612 
613  ngx_mutex_lock(ngx_posted_events_mutex);
614 
615  for (i = 0; i < events; i++) {
616  c = event_list[i].data.ptr;
617 
618  instance = (uintptr_t) c & 1;
619  c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1);
620 
621  rev = c->read;
622 
623  if (c->fd == -1 || rev->instance != instance) {
624 
625  /*
626  * the stale event from a file descriptor
627  * that was just closed in this iteration
628  */
629 
631  "epoll: stale event %p", c);
632  continue;
633  }
634 
635  revents = event_list[i].events;
636 
638  "epoll: fd:%d ev:%04XD d:%p",
639  c->fd, revents, event_list[i].data.ptr);
640 
641  if (revents & (EPOLLERR|EPOLLHUP)) {
643  "epoll_wait() error on fd:%d ev:%04XD",
644  c->fd, revents);
645  }
646 
647 #if 0
648  if (revents & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) {
649  ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
650  "strange epoll_wait() events fd:%d ev:%04XD",
651  c->fd, revents);
652  }
653 #endif
654 
655  if ((revents & (EPOLLERR|EPOLLHUP))
656  && (revents & (EPOLLIN|EPOLLOUT)) == 0)
657  {
658  /*
659  * if the error events were returned without EPOLLIN or EPOLLOUT,
660  * then add these flags to handle the events at least in one
661  * active handler
662  */
663 
664  revents |= EPOLLIN|EPOLLOUT;
665  }
666 
667  if ((revents & EPOLLIN) && rev->active) {
668 
669  if ((flags & NGX_POST_THREAD_EVENTS) && !rev->accept) {
670  rev->posted_ready = 1;
671 
672  } else {
673  rev->ready = 1;
674  }
675 
676  if (flags & NGX_POST_EVENTS) {
677  queue = (ngx_event_t **) (rev->accept ?
679 
680  ngx_locked_post_event(rev, queue);
681 
682  } else {
683  rev->handler(rev);
684  }
685  }
686 
687  wev = c->write;
688 
689  if ((revents & EPOLLOUT) && wev->active) {
690 
691  if (c->fd == -1 || wev->instance != instance) {
692 
693  /*
694  * the stale event from a file descriptor
695  * that was just closed in this iteration
696  */
697 
699  "epoll: stale event %p", c);
700  continue;
701  }
702 
703  if (flags & NGX_POST_THREAD_EVENTS) {
704  wev->posted_ready = 1;
705 
706  } else {
707  wev->ready = 1;
708  }
709 
710  if (flags & NGX_POST_EVENTS) {
712 
713  } else {
714  wev->handler(wev);
715  }
716  }
717  }
718 
719  ngx_mutex_unlock(ngx_posted_events_mutex);
720 
721  return NGX_OK;
722 }
723 
724 
725 #if (NGX_HAVE_FILE_AIO)
726 
727 static void
728 ngx_epoll_eventfd_handler(ngx_event_t *ev)
729 {
730  int n, events;
731  long i;
732  uint64_t ready;
733  ngx_err_t err;
734  ngx_event_t *e;
735  ngx_event_aio_t *aio;
736  struct io_event event[64];
737  struct timespec ts;
738 
739  ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler");
740 
741  n = read(ngx_eventfd, &ready, 8);
742 
743  err = ngx_errno;
744 
745  ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n);
746 
747  if (n != 8) {
748  if (n == -1) {
749  if (err == NGX_EAGAIN) {
750  return;
751  }
752 
753  ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed");
754  return;
755  }
756 
758  "read(eventfd) returned only %d bytes", n);
759  return;
760  }
761 
762  ts.tv_sec = 0;
763  ts.tv_nsec = 0;
764 
765  while (ready) {
766 
767  events = io_getevents(ngx_aio_ctx, 1, 64, event, &ts);
768 
770  "io_getevents: %l", events);
771 
772  if (events > 0) {
773  ready -= events;
774 
775  for (i = 0; i < events; i++) {
776 
778  "io_event: %uXL %uXL %L %L",
779  event[i].data, event[i].obj,
780  event[i].res, event[i].res2);
781 
782  e = (ngx_event_t *) (uintptr_t) event[i].data;
783 
784  e->complete = 1;
785  e->active = 0;
786  e->ready = 1;
787 
788  aio = e->data;
789  aio->res = event[i].res;
790 
792  }
793 
794  continue;
795  }
796 
797  if (events == 0) {
798  return;
799  }
800 
801  /* events == -1 */
803  "io_getevents() failed");
804  return;
805  }
806 }
807 
808 #endif
809 
810 
811 static void *
812 ngx_epoll_create_conf(ngx_cycle_t *cycle)
813 {
814  ngx_epoll_conf_t *epcf;
815 
816  epcf = ngx_palloc(cycle->pool, sizeof(ngx_epoll_conf_t));
817  if (epcf == NULL) {
818  return NULL;
819  }
820 
821  epcf->events = NGX_CONF_UNSET;
823 
824  return epcf;
825 }
826 
827 
828 static char *
829 ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf)
830 {
831  ngx_epoll_conf_t *epcf = conf;
832 
833  ngx_conf_init_uint_value(epcf->events, 512);
835 
836  return NGX_CONF_OK;
837 }