event.c 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446
  1. /*
  2. * The contents of this file are subject to the Mozilla Public
  3. * License Version 1.1 (the "License"); you may not use this file
  4. * except in compliance with the License. You may obtain a copy of
  5. * the License at http://www.mozilla.org/MPL/
  6. *
  7. * Software distributed under the License is distributed on an "AS
  8. * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
  9. * implied. See the License for the specific language governing
  10. * rights and limitations under the License.
  11. *
  12. * The Original Code is the Netscape Portable Runtime library.
  13. *
  14. * The Initial Developer of the Original Code is Netscape
  15. * Communications Corporation. Portions created by Netscape are
  16. * Copyright (C) 1994-2000 Netscape Communications Corporation. All
  17. * Rights Reserved.
  18. *
  19. * Contributor(s): Silicon Graphics, Inc.
  20. * Yahoo! Inc.
  21. *
  22. * Alternatively, the contents of this file may be used under the
  23. * terms of the GNU General Public License Version 2 or later (the
  24. * "GPL"), in which case the provisions of the GPL are applicable
  25. * instead of those above. If you wish to allow use of your
  26. * version of this file only under the terms of the GPL and not to
  27. * allow others to use your version of this file under the MPL,
  28. * indicate your decision by deleting the provisions above and
  29. * replace them with the notice and other provisions required by
  30. * the GPL. If you do not delete the provisions above, a recipient
  31. * may use your version of this file under either the MPL or the
  32. * GPL.
  33. */
  34. #include <stdlib.h>
  35. #include <unistd.h>
  36. #include <fcntl.h>
  37. #include <string.h>
  38. #include <time.h>
  39. #include <errno.h>
  40. #include "common.h"
  41. #ifdef MD_HAVE_KQUEUE
  42. #include <sys/event.h>
  43. #endif
  44. #ifdef MD_HAVE_EPOLL
  45. #include <sys/epoll.h>
  46. #endif
  47. // Global stat.
  48. #if defined(DEBUG) && defined(DEBUG_STATS)
  49. unsigned long long _st_stat_epoll = 0;
  50. unsigned long long _st_stat_epoll_zero = 0;
  51. unsigned long long _st_stat_epoll_shake = 0;
  52. unsigned long long _st_stat_epoll_spin = 0;
  53. #endif
  54. #if defined(USE_POLL) && !defined(MD_HAVE_POLL)
  55. /* Force poll usage if explicitly asked for it */
  56. #define MD_HAVE_POLL
  57. #endif
  58. static struct _st_seldata {
  59. fd_set fd_read_set, fd_write_set, fd_exception_set;
  60. int fd_ref_cnts[FD_SETSIZE][3];
  61. int maxfd;
  62. } *_st_select_data;
  63. #define _ST_SELECT_MAX_OSFD (_st_select_data->maxfd)
  64. #define _ST_SELECT_READ_SET (_st_select_data->fd_read_set)
  65. #define _ST_SELECT_WRITE_SET (_st_select_data->fd_write_set)
  66. #define _ST_SELECT_EXCEP_SET (_st_select_data->fd_exception_set)
  67. #define _ST_SELECT_READ_CNT(fd) (_st_select_data->fd_ref_cnts[fd][0])
  68. #define _ST_SELECT_WRITE_CNT(fd) (_st_select_data->fd_ref_cnts[fd][1])
  69. #define _ST_SELECT_EXCEP_CNT(fd) (_st_select_data->fd_ref_cnts[fd][2])
  70. #ifdef MD_HAVE_POLL
  71. static struct _st_polldata {
  72. struct pollfd *pollfds;
  73. int pollfds_size;
  74. int fdcnt;
  75. } *_st_poll_data;
  76. #define _ST_POLL_OSFD_CNT (_st_poll_data->fdcnt)
  77. #define _ST_POLLFDS (_st_poll_data->pollfds)
  78. #define _ST_POLLFDS_SIZE (_st_poll_data->pollfds_size)
  79. #endif /* MD_HAVE_POLL */
  80. #ifdef MD_HAVE_KQUEUE
  81. typedef struct _kq_fd_data {
  82. int rd_ref_cnt;
  83. int wr_ref_cnt;
  84. int revents;
  85. } _kq_fd_data_t;
  86. static struct _st_kqdata {
  87. _kq_fd_data_t *fd_data;
  88. struct kevent *evtlist;
  89. struct kevent *addlist;
  90. struct kevent *dellist;
  91. int fd_data_size;
  92. int evtlist_size;
  93. int addlist_size;
  94. int addlist_cnt;
  95. int dellist_size;
  96. int dellist_cnt;
  97. int kq;
  98. pid_t pid;
  99. } *_st_kq_data;
  100. #ifndef ST_KQ_MIN_EVTLIST_SIZE
  101. #define ST_KQ_MIN_EVTLIST_SIZE 64
  102. #endif
  103. #define _ST_KQ_READ_CNT(fd) (_st_kq_data->fd_data[fd].rd_ref_cnt)
  104. #define _ST_KQ_WRITE_CNT(fd) (_st_kq_data->fd_data[fd].wr_ref_cnt)
  105. #define _ST_KQ_REVENTS(fd) (_st_kq_data->fd_data[fd].revents)
  106. #endif /* MD_HAVE_KQUEUE */
  107. #ifdef MD_HAVE_EPOLL
  108. typedef struct _epoll_fd_data {
  109. int rd_ref_cnt;
  110. int wr_ref_cnt;
  111. int ex_ref_cnt;
  112. int revents;
  113. } _epoll_fd_data_t;
  114. static struct _st_epolldata {
  115. _epoll_fd_data_t *fd_data;
  116. struct epoll_event *evtlist;
  117. int fd_data_size;
  118. int evtlist_size;
  119. int evtlist_cnt;
  120. int fd_hint;
  121. int epfd;
  122. pid_t pid;
  123. } *_st_epoll_data;
  124. #ifndef ST_EPOLL_EVTLIST_SIZE
  125. /* Not a limit, just a hint */
  126. #define ST_EPOLL_EVTLIST_SIZE 4096
  127. #endif
  128. #define _ST_EPOLL_READ_CNT(fd) (_st_epoll_data->fd_data[fd].rd_ref_cnt)
  129. #define _ST_EPOLL_WRITE_CNT(fd) (_st_epoll_data->fd_data[fd].wr_ref_cnt)
  130. #define _ST_EPOLL_EXCEP_CNT(fd) (_st_epoll_data->fd_data[fd].ex_ref_cnt)
  131. #define _ST_EPOLL_REVENTS(fd) (_st_epoll_data->fd_data[fd].revents)
  132. #define _ST_EPOLL_READ_BIT(fd) (_ST_EPOLL_READ_CNT(fd) ? EPOLLIN : 0)
  133. #define _ST_EPOLL_WRITE_BIT(fd) (_ST_EPOLL_WRITE_CNT(fd) ? EPOLLOUT : 0)
  134. #define _ST_EPOLL_EXCEP_BIT(fd) (_ST_EPOLL_EXCEP_CNT(fd) ? EPOLLPRI : 0)
  135. #define _ST_EPOLL_EVENTS(fd) \
  136. (_ST_EPOLL_READ_BIT(fd)|_ST_EPOLL_WRITE_BIT(fd)|_ST_EPOLL_EXCEP_BIT(fd))
  137. #endif /* MD_HAVE_EPOLL */
  138. _st_eventsys_t *_st_eventsys = NULL;
  139. /*****************************************
  140. * select event system
  141. */
  142. ST_HIDDEN int _st_select_init(void)
  143. {
  144. _st_select_data = (struct _st_seldata *) malloc(sizeof(*_st_select_data));
  145. if (!_st_select_data)
  146. return -1;
  147. memset(_st_select_data, 0, sizeof(*_st_select_data));
  148. _st_select_data->maxfd = -1;
  149. return 0;
  150. }
  151. ST_HIDDEN int _st_select_pollset_add(struct pollfd *pds, int npds)
  152. {
  153. struct pollfd *pd;
  154. struct pollfd *epd = pds + npds;
  155. /* Do checks up front */
  156. for (pd = pds; pd < epd; pd++) {
  157. if (pd->fd < 0 || pd->fd >= FD_SETSIZE || !pd->events ||
  158. (pd->events & ~(POLLIN | POLLOUT | POLLPRI))) {
  159. errno = EINVAL;
  160. return -1;
  161. }
  162. }
  163. for (pd = pds; pd < epd; pd++) {
  164. if (pd->events & POLLIN) {
  165. FD_SET(pd->fd, &_ST_SELECT_READ_SET);
  166. _ST_SELECT_READ_CNT(pd->fd)++;
  167. }
  168. if (pd->events & POLLOUT) {
  169. FD_SET(pd->fd, &_ST_SELECT_WRITE_SET);
  170. _ST_SELECT_WRITE_CNT(pd->fd)++;
  171. }
  172. if (pd->events & POLLPRI) {
  173. FD_SET(pd->fd, &_ST_SELECT_EXCEP_SET);
  174. _ST_SELECT_EXCEP_CNT(pd->fd)++;
  175. }
  176. if (_ST_SELECT_MAX_OSFD < pd->fd)
  177. _ST_SELECT_MAX_OSFD = pd->fd;
  178. }
  179. return 0;
  180. }
  181. ST_HIDDEN void _st_select_pollset_del(struct pollfd *pds, int npds)
  182. {
  183. struct pollfd *pd;
  184. struct pollfd *epd = pds + npds;
  185. for (pd = pds; pd < epd; pd++) {
  186. if (pd->events & POLLIN) {
  187. if (--_ST_SELECT_READ_CNT(pd->fd) == 0)
  188. FD_CLR(pd->fd, &_ST_SELECT_READ_SET);
  189. }
  190. if (pd->events & POLLOUT) {
  191. if (--_ST_SELECT_WRITE_CNT(pd->fd) == 0)
  192. FD_CLR(pd->fd, &_ST_SELECT_WRITE_SET);
  193. }
  194. if (pd->events & POLLPRI) {
  195. if (--_ST_SELECT_EXCEP_CNT(pd->fd) == 0)
  196. FD_CLR(pd->fd, &_ST_SELECT_EXCEP_SET);
  197. }
  198. }
  199. }
  200. ST_HIDDEN void _st_select_find_bad_fd(void)
  201. {
  202. _st_clist_t *q;
  203. _st_pollq_t *pq;
  204. int notify;
  205. struct pollfd *pds, *epds;
  206. int pq_max_osfd, osfd;
  207. short events;
  208. _ST_SELECT_MAX_OSFD = -1;
  209. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  210. pq = _ST_POLLQUEUE_PTR(q);
  211. notify = 0;
  212. epds = pq->pds + pq->npds;
  213. pq_max_osfd = -1;
  214. for (pds = pq->pds; pds < epds; pds++) {
  215. osfd = pds->fd;
  216. pds->revents = 0;
  217. if (pds->events == 0)
  218. continue;
  219. if (fcntl(osfd, F_GETFL, 0) < 0) {
  220. pds->revents = POLLNVAL;
  221. notify = 1;
  222. }
  223. if (osfd > pq_max_osfd) {
  224. pq_max_osfd = osfd;
  225. }
  226. }
  227. if (notify) {
  228. ST_REMOVE_LINK(&pq->links);
  229. pq->on_ioq = 0;
  230. /*
  231. * Decrement the count of descriptors for each descriptor/event
  232. * because this I/O request is being removed from the ioq
  233. */
  234. for (pds = pq->pds; pds < epds; pds++) {
  235. osfd = pds->fd;
  236. events = pds->events;
  237. if (events & POLLIN) {
  238. if (--_ST_SELECT_READ_CNT(osfd) == 0) {
  239. FD_CLR(osfd, &_ST_SELECT_READ_SET);
  240. }
  241. }
  242. if (events & POLLOUT) {
  243. if (--_ST_SELECT_WRITE_CNT(osfd) == 0) {
  244. FD_CLR(osfd, &_ST_SELECT_WRITE_SET);
  245. }
  246. }
  247. if (events & POLLPRI) {
  248. if (--_ST_SELECT_EXCEP_CNT(osfd) == 0) {
  249. FD_CLR(osfd, &_ST_SELECT_EXCEP_SET);
  250. }
  251. }
  252. }
  253. if (pq->thread->flags & _ST_FL_ON_SLEEPQ)
  254. _ST_DEL_SLEEPQ(pq->thread);
  255. pq->thread->state = _ST_ST_RUNNABLE;
  256. _ST_ADD_RUNQ(pq->thread);
  257. } else {
  258. if (_ST_SELECT_MAX_OSFD < pq_max_osfd)
  259. _ST_SELECT_MAX_OSFD = pq_max_osfd;
  260. }
  261. }
  262. }
  263. ST_HIDDEN void _st_select_dispatch(void)
  264. {
  265. struct timeval timeout, *tvp;
  266. fd_set r, w, e;
  267. fd_set *rp, *wp, *ep;
  268. int nfd, pq_max_osfd, osfd;
  269. _st_clist_t *q;
  270. st_utime_t min_timeout;
  271. _st_pollq_t *pq;
  272. int notify;
  273. struct pollfd *pds, *epds;
  274. short events, revents;
  275. /*
  276. * Assignment of fd_sets
  277. */
  278. r = _ST_SELECT_READ_SET;
  279. w = _ST_SELECT_WRITE_SET;
  280. e = _ST_SELECT_EXCEP_SET;
  281. rp = &r;
  282. wp = &w;
  283. ep = &e;
  284. if (_ST_SLEEPQ == NULL) {
  285. tvp = NULL;
  286. } else {
  287. min_timeout = (_ST_SLEEPQ->due <= _ST_LAST_CLOCK) ? 0 :
  288. (_ST_SLEEPQ->due - _ST_LAST_CLOCK);
  289. timeout.tv_sec = (int) (min_timeout / 1000000);
  290. timeout.tv_usec = (int) (min_timeout % 1000000);
  291. tvp = &timeout;
  292. }
  293. /* Check for I/O operations */
  294. nfd = select(_ST_SELECT_MAX_OSFD + 1, rp, wp, ep, tvp);
  295. /* Notify threads that are associated with the selected descriptors */
  296. if (nfd > 0) {
  297. _ST_SELECT_MAX_OSFD = -1;
  298. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  299. pq = _ST_POLLQUEUE_PTR(q);
  300. notify = 0;
  301. epds = pq->pds + pq->npds;
  302. pq_max_osfd = -1;
  303. for (pds = pq->pds; pds < epds; pds++) {
  304. osfd = pds->fd;
  305. events = pds->events;
  306. revents = 0;
  307. if ((events & POLLIN) && FD_ISSET(osfd, rp)) {
  308. revents |= POLLIN;
  309. }
  310. if ((events & POLLOUT) && FD_ISSET(osfd, wp)) {
  311. revents |= POLLOUT;
  312. }
  313. if ((events & POLLPRI) && FD_ISSET(osfd, ep)) {
  314. revents |= POLLPRI;
  315. }
  316. pds->revents = revents;
  317. if (revents) {
  318. notify = 1;
  319. }
  320. if (osfd > pq_max_osfd) {
  321. pq_max_osfd = osfd;
  322. }
  323. }
  324. if (notify) {
  325. ST_REMOVE_LINK(&pq->links);
  326. pq->on_ioq = 0;
  327. /*
  328. * Decrement the count of descriptors for each descriptor/event
  329. * because this I/O request is being removed from the ioq
  330. */
  331. for (pds = pq->pds; pds < epds; pds++) {
  332. osfd = pds->fd;
  333. events = pds->events;
  334. if (events & POLLIN) {
  335. if (--_ST_SELECT_READ_CNT(osfd) == 0) {
  336. FD_CLR(osfd, &_ST_SELECT_READ_SET);
  337. }
  338. }
  339. if (events & POLLOUT) {
  340. if (--_ST_SELECT_WRITE_CNT(osfd) == 0) {
  341. FD_CLR(osfd, &_ST_SELECT_WRITE_SET);
  342. }
  343. }
  344. if (events & POLLPRI) {
  345. if (--_ST_SELECT_EXCEP_CNT(osfd) == 0) {
  346. FD_CLR(osfd, &_ST_SELECT_EXCEP_SET);
  347. }
  348. }
  349. }
  350. if (pq->thread->flags & _ST_FL_ON_SLEEPQ)
  351. _ST_DEL_SLEEPQ(pq->thread);
  352. pq->thread->state = _ST_ST_RUNNABLE;
  353. _ST_ADD_RUNQ(pq->thread);
  354. } else {
  355. if (_ST_SELECT_MAX_OSFD < pq_max_osfd)
  356. _ST_SELECT_MAX_OSFD = pq_max_osfd;
  357. }
  358. }
  359. } else if (nfd < 0) {
  360. /*
  361. * It can happen when a thread closes file descriptor
  362. * that is being used by some other thread -- BAD!
  363. */
  364. if (errno == EBADF)
  365. _st_select_find_bad_fd();
  366. }
  367. }
  368. ST_HIDDEN int _st_select_fd_new(int osfd)
  369. {
  370. if (osfd >= FD_SETSIZE) {
  371. errno = EMFILE;
  372. return -1;
  373. }
  374. return 0;
  375. }
  376. ST_HIDDEN int _st_select_fd_close(int osfd)
  377. {
  378. if (_ST_SELECT_READ_CNT(osfd) || _ST_SELECT_WRITE_CNT(osfd) ||
  379. _ST_SELECT_EXCEP_CNT(osfd)) {
  380. errno = EBUSY;
  381. return -1;
  382. }
  383. return 0;
  384. }
  385. ST_HIDDEN int _st_select_fd_getlimit(void)
  386. {
  387. return FD_SETSIZE;
  388. }
  389. static _st_eventsys_t _st_select_eventsys = {
  390. "select",
  391. ST_EVENTSYS_SELECT,
  392. _st_select_init,
  393. _st_select_dispatch,
  394. _st_select_pollset_add,
  395. _st_select_pollset_del,
  396. _st_select_fd_new,
  397. _st_select_fd_close,
  398. _st_select_fd_getlimit
  399. };
  400. #ifdef MD_HAVE_POLL
  401. /*****************************************
  402. * poll event system
  403. */
  404. ST_HIDDEN int _st_poll_init(void)
  405. {
  406. _st_poll_data = (struct _st_polldata *) malloc(sizeof(*_st_poll_data));
  407. if (!_st_poll_data)
  408. return -1;
  409. _ST_POLLFDS = (struct pollfd *) malloc(ST_MIN_POLLFDS_SIZE *
  410. sizeof(struct pollfd));
  411. if (!_ST_POLLFDS) {
  412. free(_st_poll_data);
  413. _st_poll_data = NULL;
  414. return -1;
  415. }
  416. _ST_POLLFDS_SIZE = ST_MIN_POLLFDS_SIZE;
  417. _ST_POLL_OSFD_CNT = 0;
  418. return 0;
  419. }
  420. ST_HIDDEN int _st_poll_pollset_add(struct pollfd *pds, int npds)
  421. {
  422. struct pollfd *pd;
  423. struct pollfd *epd = pds + npds;
  424. for (pd = pds; pd < epd; pd++) {
  425. if (pd->fd < 0 || !pd->events) {
  426. errno = EINVAL;
  427. return -1;
  428. }
  429. }
  430. _ST_POLL_OSFD_CNT += npds;
  431. return 0;
  432. }
  433. /* ARGSUSED */
  434. ST_HIDDEN void _st_poll_pollset_del(struct pollfd *pds, int npds)
  435. {
  436. _ST_POLL_OSFD_CNT -= npds;
  437. ST_ASSERT(_ST_POLL_OSFD_CNT >= 0);
  438. }
  439. ST_HIDDEN void _st_poll_dispatch(void)
  440. {
  441. int timeout, nfd;
  442. _st_clist_t *q;
  443. st_utime_t min_timeout;
  444. _st_pollq_t *pq;
  445. struct pollfd *pds, *epds, *pollfds;
  446. /*
  447. * Build up the array of struct pollfd to wait on.
  448. * If existing array is not big enough, release it and allocate a new one.
  449. */
  450. ST_ASSERT(_ST_POLL_OSFD_CNT >= 0);
  451. if (_ST_POLL_OSFD_CNT > _ST_POLLFDS_SIZE) {
  452. free(_ST_POLLFDS);
  453. _ST_POLLFDS = (struct pollfd *) malloc((_ST_POLL_OSFD_CNT + 10) *
  454. sizeof(struct pollfd));
  455. ST_ASSERT(_ST_POLLFDS != NULL);
  456. _ST_POLLFDS_SIZE = _ST_POLL_OSFD_CNT + 10;
  457. }
  458. pollfds = _ST_POLLFDS;
  459. /* Gather all descriptors into one array */
  460. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  461. pq = _ST_POLLQUEUE_PTR(q);
  462. memcpy(pollfds, pq->pds, sizeof(struct pollfd) * pq->npds);
  463. pollfds += pq->npds;
  464. }
  465. ST_ASSERT(pollfds <= _ST_POLLFDS + _ST_POLLFDS_SIZE);
  466. if (_ST_SLEEPQ == NULL) {
  467. timeout = -1;
  468. } else {
  469. min_timeout = (_ST_SLEEPQ->due <= _ST_LAST_CLOCK) ? 0 :
  470. (_ST_SLEEPQ->due - _ST_LAST_CLOCK);
  471. timeout = (int) (min_timeout / 1000);
  472. }
  473. /* Check for I/O operations */
  474. nfd = poll(_ST_POLLFDS, _ST_POLL_OSFD_CNT, timeout);
  475. /* Notify threads that are associated with the selected descriptors */
  476. if (nfd > 0) {
  477. pollfds = _ST_POLLFDS;
  478. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  479. pq = _ST_POLLQUEUE_PTR(q);
  480. epds = pollfds + pq->npds;
  481. for (pds = pollfds; pds < epds; pds++) {
  482. if (pds->revents)
  483. break;
  484. }
  485. if (pds < epds) {
  486. memcpy(pq->pds, pollfds, sizeof(struct pollfd) * pq->npds);
  487. ST_REMOVE_LINK(&pq->links);
  488. pq->on_ioq = 0;
  489. if (pq->thread->flags & _ST_FL_ON_SLEEPQ)
  490. _ST_DEL_SLEEPQ(pq->thread);
  491. pq->thread->state = _ST_ST_RUNNABLE;
  492. _ST_ADD_RUNQ(pq->thread);
  493. _ST_POLL_OSFD_CNT -= pq->npds;
  494. ST_ASSERT(_ST_POLL_OSFD_CNT >= 0);
  495. }
  496. pollfds = epds;
  497. }
  498. }
  499. }
  500. /* ARGSUSED */
  501. ST_HIDDEN int _st_poll_fd_new(int osfd)
  502. {
  503. return 0;
  504. }
  505. /* ARGSUSED */
  506. ST_HIDDEN int _st_poll_fd_close(int osfd)
  507. {
  508. /*
  509. * We don't maintain I/O counts for poll event system
  510. * so nothing to check here.
  511. */
  512. return 0;
  513. }
  514. ST_HIDDEN int _st_poll_fd_getlimit(void)
  515. {
  516. /* zero means no specific limit */
  517. return 0;
  518. }
  519. static _st_eventsys_t _st_poll_eventsys = {
  520. "poll",
  521. ST_EVENTSYS_POLL,
  522. _st_poll_init,
  523. _st_poll_dispatch,
  524. _st_poll_pollset_add,
  525. _st_poll_pollset_del,
  526. _st_poll_fd_new,
  527. _st_poll_fd_close,
  528. _st_poll_fd_getlimit
  529. };
  530. #endif /* MD_HAVE_POLL */
  531. #ifdef MD_HAVE_KQUEUE
  532. /*****************************************
  533. * kqueue event system
  534. */
  535. ST_HIDDEN int _st_kq_init(void)
  536. {
  537. int err = 0;
  538. int rv = 0;
  539. _st_kq_data = (struct _st_kqdata *) calloc(1, sizeof(*_st_kq_data));
  540. if (!_st_kq_data)
  541. return -1;
  542. if ((_st_kq_data->kq = kqueue()) < 0) {
  543. err = errno;
  544. rv = -1;
  545. goto cleanup_kq;
  546. }
  547. fcntl(_st_kq_data->kq, F_SETFD, FD_CLOEXEC);
  548. _st_kq_data->pid = getpid();
  549. /*
  550. * Allocate file descriptor data array.
  551. * FD_SETSIZE looks like good initial size.
  552. */
  553. _st_kq_data->fd_data_size = FD_SETSIZE;
  554. _st_kq_data->fd_data = (_kq_fd_data_t *)calloc(_st_kq_data->fd_data_size, sizeof(_kq_fd_data_t));
  555. if (!_st_kq_data->fd_data) {
  556. err = errno;
  557. rv = -1;
  558. goto cleanup_kq;
  559. }
  560. /* Allocate event lists */
  561. _st_kq_data->evtlist_size = ST_KQ_MIN_EVTLIST_SIZE;
  562. _st_kq_data->evtlist = (struct kevent *)malloc(_st_kq_data->evtlist_size * sizeof(struct kevent));
  563. _st_kq_data->addlist_size = ST_KQ_MIN_EVTLIST_SIZE;
  564. _st_kq_data->addlist = (struct kevent *)malloc(_st_kq_data->addlist_size * sizeof(struct kevent));
  565. _st_kq_data->dellist_size = ST_KQ_MIN_EVTLIST_SIZE;
  566. _st_kq_data->dellist = (struct kevent *)malloc(_st_kq_data->dellist_size * sizeof(struct kevent));
  567. if (!_st_kq_data->evtlist || !_st_kq_data->addlist ||
  568. !_st_kq_data->dellist) {
  569. err = ENOMEM;
  570. rv = -1;
  571. }
  572. cleanup_kq:
  573. if (rv < 0) {
  574. if (_st_kq_data->kq >= 0)
  575. close(_st_kq_data->kq);
  576. free(_st_kq_data->fd_data);
  577. free(_st_kq_data->evtlist);
  578. free(_st_kq_data->addlist);
  579. free(_st_kq_data->dellist);
  580. free(_st_kq_data);
  581. _st_kq_data = NULL;
  582. errno = err;
  583. }
  584. return rv;
  585. }
  586. ST_HIDDEN int _st_kq_fd_data_expand(int maxfd)
  587. {
  588. _kq_fd_data_t *ptr;
  589. int n = _st_kq_data->fd_data_size;
  590. while (maxfd >= n)
  591. n <<= 1;
  592. ptr = (_kq_fd_data_t *)realloc(_st_kq_data->fd_data, n * sizeof(_kq_fd_data_t));
  593. if (!ptr)
  594. return -1;
  595. memset(ptr + _st_kq_data->fd_data_size, 0, (n - _st_kq_data->fd_data_size) * sizeof(_kq_fd_data_t));
  596. _st_kq_data->fd_data = ptr;
  597. _st_kq_data->fd_data_size = n;
  598. return 0;
  599. }
  600. ST_HIDDEN int _st_kq_addlist_expand(int avail)
  601. {
  602. struct kevent *ptr;
  603. int n = _st_kq_data->addlist_size;
  604. while (avail > n - _st_kq_data->addlist_cnt)
  605. n <<= 1;
  606. ptr = (struct kevent *)realloc(_st_kq_data->addlist, n * sizeof(struct kevent));
  607. if (!ptr)
  608. return -1;
  609. _st_kq_data->addlist = ptr;
  610. _st_kq_data->addlist_size = n;
  611. /*
  612. * Try to expand the result event list too
  613. * (although we don't have to do it).
  614. */
  615. ptr = (struct kevent *)realloc(_st_kq_data->evtlist, n * sizeof(struct kevent));
  616. if (ptr) {
  617. _st_kq_data->evtlist = ptr;
  618. _st_kq_data->evtlist_size = n;
  619. }
  620. return 0;
  621. }
  622. ST_HIDDEN void _st_kq_addlist_add(const struct kevent *kev)
  623. {
  624. ST_ASSERT(_st_kq_data->addlist_cnt < _st_kq_data->addlist_size);
  625. memcpy(_st_kq_data->addlist + _st_kq_data->addlist_cnt, kev, sizeof(struct kevent));
  626. _st_kq_data->addlist_cnt++;
  627. }
  628. ST_HIDDEN void _st_kq_dellist_add(const struct kevent *kev)
  629. {
  630. int n = _st_kq_data->dellist_size;
  631. if (_st_kq_data->dellist_cnt >= n) {
  632. struct kevent *ptr;
  633. n <<= 1;
  634. ptr = (struct kevent *)realloc(_st_kq_data->dellist, n * sizeof(struct kevent));
  635. if (!ptr) {
  636. /* See comment in _st_kq_pollset_del() */
  637. return;
  638. }
  639. _st_kq_data->dellist = ptr;
  640. _st_kq_data->dellist_size = n;
  641. }
  642. memcpy(_st_kq_data->dellist + _st_kq_data->dellist_cnt, kev, sizeof(struct kevent));
  643. _st_kq_data->dellist_cnt++;
  644. }
  645. ST_HIDDEN int _st_kq_pollset_add(struct pollfd *pds, int npds)
  646. {
  647. struct kevent kev;
  648. struct pollfd *pd;
  649. struct pollfd *epd = pds + npds;
  650. /*
  651. * Pollset adding is "atomic". That is, either it succeeded for
  652. * all descriptors in the set or it failed. It means that we
  653. * need to do all the checks up front so we don't have to
  654. * "unwind" if adding of one of the descriptors failed.
  655. */
  656. for (pd = pds; pd < epd; pd++) {
  657. /* POLLIN and/or POLLOUT must be set, but nothing else */
  658. if (pd->fd < 0 || !pd->events || (pd->events & ~(POLLIN | POLLOUT))) {
  659. errno = EINVAL;
  660. return -1;
  661. }
  662. if (pd->fd >= _st_kq_data->fd_data_size &&
  663. _st_kq_fd_data_expand(pd->fd) < 0)
  664. return -1;
  665. }
  666. /*
  667. * Make sure we have enough room in the addlist for twice as many
  668. * descriptors as in the pollset (for both READ and WRITE filters).
  669. */
  670. npds <<= 1;
  671. if (npds > _st_kq_data->addlist_size - _st_kq_data->addlist_cnt && _st_kq_addlist_expand(npds) < 0)
  672. return -1;
  673. for (pd = pds; pd < epd; pd++) {
  674. if ((pd->events & POLLIN) && (_ST_KQ_READ_CNT(pd->fd)++ == 0)) {
  675. memset(&kev, 0, sizeof(kev));
  676. kev.ident = pd->fd;
  677. kev.filter = EVFILT_READ;
  678. #ifdef NOTE_EOF
  679. /* Make it behave like select() and poll() */
  680. kev.fflags = NOTE_EOF;
  681. #endif
  682. kev.flags = (EV_ADD | EV_ONESHOT);
  683. _st_kq_addlist_add(&kev);
  684. }
  685. if ((pd->events & POLLOUT) && (_ST_KQ_WRITE_CNT(pd->fd)++ == 0)) {
  686. memset(&kev, 0, sizeof(kev));
  687. kev.ident = pd->fd;
  688. kev.filter = EVFILT_WRITE;
  689. kev.flags = (EV_ADD | EV_ONESHOT);
  690. _st_kq_addlist_add(&kev);
  691. }
  692. }
  693. return 0;
  694. }
  695. ST_HIDDEN void _st_kq_pollset_del(struct pollfd *pds, int npds)
  696. {
  697. struct kevent kev;
  698. struct pollfd *pd;
  699. struct pollfd *epd = pds + npds;
  700. /*
  701. * It's OK if deleting fails because a descriptor will either be
  702. * closed or fire only once (we set EV_ONESHOT flag).
  703. */
  704. _st_kq_data->dellist_cnt = 0;
  705. for (pd = pds; pd < epd; pd++) {
  706. if ((pd->events & POLLIN) && (--_ST_KQ_READ_CNT(pd->fd) == 0)) {
  707. memset(&kev, 0, sizeof(kev));
  708. kev.ident = pd->fd;
  709. kev.filter = EVFILT_READ;
  710. kev.flags = EV_DELETE;
  711. _st_kq_dellist_add(&kev);
  712. }
  713. if ((pd->events & POLLOUT) && (--_ST_KQ_WRITE_CNT(pd->fd) == 0)) {
  714. memset(&kev, 0, sizeof(kev));
  715. kev.ident = pd->fd;
  716. kev.filter = EVFILT_WRITE;
  717. kev.flags = EV_DELETE;
  718. _st_kq_dellist_add(&kev);
  719. }
  720. }
  721. if (_st_kq_data->dellist_cnt > 0) {
  722. /*
  723. * We do "synchronous" kqueue deletes to avoid deleting
  724. * closed descriptors and other possible problems.
  725. */
  726. int rv;
  727. do {
  728. /* This kevent() won't block since result list size is 0 */
  729. rv = kevent(_st_kq_data->kq, _st_kq_data->dellist, _st_kq_data->dellist_cnt, NULL, 0, NULL);
  730. } while (rv < 0 && errno == EINTR);
  731. }
  732. }
  733. ST_HIDDEN void _st_kq_dispatch(void)
  734. {
  735. struct timespec timeout, *tsp;
  736. struct kevent kev;
  737. st_utime_t min_timeout;
  738. _st_clist_t *q;
  739. _st_pollq_t *pq;
  740. struct pollfd *pds, *epds;
  741. int nfd, i, osfd, notify, filter;
  742. short events, revents;
  743. if (_ST_SLEEPQ == NULL) {
  744. tsp = NULL;
  745. } else {
  746. min_timeout = (_ST_SLEEPQ->due <= _ST_LAST_CLOCK) ? 0 : (_ST_SLEEPQ->due - _ST_LAST_CLOCK);
  747. timeout.tv_sec = (time_t) (min_timeout / 1000000);
  748. timeout.tv_nsec = (long) ((min_timeout % 1000000) * 1000);
  749. tsp = &timeout;
  750. }
  751. retry_kevent:
  752. /* Check for I/O operations */
  753. nfd = kevent(_st_kq_data->kq,
  754. _st_kq_data->addlist, _st_kq_data->addlist_cnt,
  755. _st_kq_data->evtlist, _st_kq_data->evtlist_size, tsp);
  756. _st_kq_data->addlist_cnt = 0;
  757. if (nfd > 0) {
  758. for (i = 0; i < nfd; i++) {
  759. osfd = _st_kq_data->evtlist[i].ident;
  760. filter = _st_kq_data->evtlist[i].filter;
  761. if (filter == EVFILT_READ) {
  762. _ST_KQ_REVENTS(osfd) |= POLLIN;
  763. } else if (filter == EVFILT_WRITE) {
  764. _ST_KQ_REVENTS(osfd) |= POLLOUT;
  765. }
  766. if (_st_kq_data->evtlist[i].flags & EV_ERROR) {
  767. if (_st_kq_data->evtlist[i].data == EBADF) {
  768. _ST_KQ_REVENTS(osfd) |= POLLNVAL;
  769. } else {
  770. _ST_KQ_REVENTS(osfd) |= POLLERR;
  771. }
  772. }
  773. }
  774. _st_kq_data->dellist_cnt = 0;
  775. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  776. pq = _ST_POLLQUEUE_PTR(q);
  777. notify = 0;
  778. epds = pq->pds + pq->npds;
  779. for (pds = pq->pds; pds < epds; pds++) {
  780. osfd = pds->fd;
  781. events = pds->events;
  782. revents = (short)(_ST_KQ_REVENTS(osfd) & ~(POLLIN | POLLOUT));
  783. if ((events & POLLIN) && (_ST_KQ_REVENTS(osfd) & POLLIN)) {
  784. revents |= POLLIN;
  785. }
  786. if ((events & POLLOUT) && (_ST_KQ_REVENTS(osfd) & POLLOUT)) {
  787. revents |= POLLOUT;
  788. }
  789. pds->revents = revents;
  790. if (revents) {
  791. notify = 1;
  792. }
  793. }
  794. if (notify) {
  795. ST_REMOVE_LINK(&pq->links);
  796. pq->on_ioq = 0;
  797. for (pds = pq->pds; pds < epds; pds++) {
  798. osfd = pds->fd;
  799. events = pds->events;
  800. /*
  801. * We set EV_ONESHOT flag so we only need to delete
  802. * descriptor if it didn't fire.
  803. */
  804. if ((events & POLLIN) && (--_ST_KQ_READ_CNT(osfd) == 0) && ((_ST_KQ_REVENTS(osfd) & POLLIN) == 0)) {
  805. memset(&kev, 0, sizeof(kev));
  806. kev.ident = osfd;
  807. kev.filter = EVFILT_READ;
  808. kev.flags = EV_DELETE;
  809. _st_kq_dellist_add(&kev);
  810. }
  811. if ((events & POLLOUT) && (--_ST_KQ_WRITE_CNT(osfd) == 0) && ((_ST_KQ_REVENTS(osfd) & POLLOUT) == 0)) {
  812. memset(&kev, 0, sizeof(kev));
  813. kev.ident = osfd;
  814. kev.filter = EVFILT_WRITE;
  815. kev.flags = EV_DELETE;
  816. _st_kq_dellist_add(&kev);
  817. }
  818. }
  819. if (pq->thread->flags & _ST_FL_ON_SLEEPQ)
  820. _ST_DEL_SLEEPQ(pq->thread);
  821. pq->thread->state = _ST_ST_RUNNABLE;
  822. _ST_ADD_RUNQ(pq->thread);
  823. }
  824. }
  825. if (_st_kq_data->dellist_cnt > 0) {
  826. int rv;
  827. do {
  828. /* This kevent() won't block since result list size is 0 */
  829. rv = kevent(_st_kq_data->kq, _st_kq_data->dellist, _st_kq_data->dellist_cnt, NULL, 0, NULL);
  830. } while (rv < 0 && errno == EINTR);
  831. }
  832. for (i = 0; i < nfd; i++) {
  833. osfd = _st_kq_data->evtlist[i].ident;
  834. _ST_KQ_REVENTS(osfd) = 0;
  835. }
  836. } else if (nfd < 0) {
  837. if (errno == EBADF && _st_kq_data->pid != getpid()) {
  838. /* We probably forked, reinitialize kqueue */
  839. if ((_st_kq_data->kq = kqueue()) < 0) {
  840. /* There is nothing we can do here, will retry later */
  841. return;
  842. }
  843. fcntl(_st_kq_data->kq, F_SETFD, FD_CLOEXEC);
  844. _st_kq_data->pid = getpid();
  845. /* Re-register all descriptors on ioq with new kqueue */
  846. memset(_st_kq_data->fd_data, 0, _st_kq_data->fd_data_size * sizeof(_kq_fd_data_t));
  847. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  848. pq = _ST_POLLQUEUE_PTR(q);
  849. _st_kq_pollset_add(pq->pds, pq->npds);
  850. }
  851. goto retry_kevent;
  852. }
  853. }
  854. }
  855. ST_HIDDEN int _st_kq_fd_new(int osfd)
  856. {
  857. if (osfd >= _st_kq_data->fd_data_size && _st_kq_fd_data_expand(osfd) < 0)
  858. return -1;
  859. return 0;
  860. }
  861. ST_HIDDEN int _st_kq_fd_close(int osfd)
  862. {
  863. if (_ST_KQ_READ_CNT(osfd) || _ST_KQ_WRITE_CNT(osfd)) {
  864. errno = EBUSY;
  865. return -1;
  866. }
  867. return 0;
  868. }
  869. ST_HIDDEN int _st_kq_fd_getlimit(void)
  870. {
  871. /* zero means no specific limit */
  872. return 0;
  873. }
  874. static _st_eventsys_t _st_kq_eventsys = {
  875. "kqueue",
  876. ST_EVENTSYS_ALT,
  877. _st_kq_init,
  878. _st_kq_dispatch,
  879. _st_kq_pollset_add,
  880. _st_kq_pollset_del,
  881. _st_kq_fd_new,
  882. _st_kq_fd_close,
  883. _st_kq_fd_getlimit
  884. };
  885. #endif /* MD_HAVE_KQUEUE */
  886. #ifdef MD_HAVE_EPOLL
  887. /*****************************************
  888. * epoll event system
  889. */
  890. ST_HIDDEN int _st_epoll_init(void)
  891. {
  892. int fdlim;
  893. int err = 0;
  894. int rv = 0;
  895. _st_epoll_data = (struct _st_epolldata *) calloc(1, sizeof(*_st_epoll_data));
  896. if (!_st_epoll_data)
  897. return -1;
  898. fdlim = st_getfdlimit();
  899. _st_epoll_data->fd_hint = (fdlim > 0 && fdlim < ST_EPOLL_EVTLIST_SIZE) ? fdlim : ST_EPOLL_EVTLIST_SIZE;
  900. if ((_st_epoll_data->epfd = epoll_create(_st_epoll_data->fd_hint)) < 0) {
  901. err = errno;
  902. rv = -1;
  903. goto cleanup_epoll;
  904. }
  905. fcntl(_st_epoll_data->epfd, F_SETFD, FD_CLOEXEC);
  906. _st_epoll_data->pid = getpid();
  907. /* Allocate file descriptor data array */
  908. _st_epoll_data->fd_data_size = _st_epoll_data->fd_hint;
  909. _st_epoll_data->fd_data = (_epoll_fd_data_t *)calloc(_st_epoll_data->fd_data_size, sizeof(_epoll_fd_data_t));
  910. if (!_st_epoll_data->fd_data) {
  911. err = errno;
  912. rv = -1;
  913. goto cleanup_epoll;
  914. }
  915. /* Allocate event lists */
  916. _st_epoll_data->evtlist_size = _st_epoll_data->fd_hint;
  917. _st_epoll_data->evtlist = (struct epoll_event *)malloc(_st_epoll_data->evtlist_size * sizeof(struct epoll_event));
  918. if (!_st_epoll_data->evtlist) {
  919. err = errno;
  920. rv = -1;
  921. }
  922. cleanup_epoll:
  923. if (rv < 0) {
  924. if (_st_epoll_data->epfd >= 0)
  925. close(_st_epoll_data->epfd);
  926. free(_st_epoll_data->fd_data);
  927. free(_st_epoll_data->evtlist);
  928. free(_st_epoll_data);
  929. _st_epoll_data = NULL;
  930. errno = err;
  931. }
  932. return rv;
  933. }
  934. ST_HIDDEN int _st_epoll_fd_data_expand(int maxfd)
  935. {
  936. _epoll_fd_data_t *ptr;
  937. int n = _st_epoll_data->fd_data_size;
  938. while (maxfd >= n)
  939. n <<= 1;
  940. ptr = (_epoll_fd_data_t *)realloc(_st_epoll_data->fd_data, n * sizeof(_epoll_fd_data_t));
  941. if (!ptr)
  942. return -1;
  943. memset(ptr + _st_epoll_data->fd_data_size, 0, (n - _st_epoll_data->fd_data_size) * sizeof(_epoll_fd_data_t));
  944. _st_epoll_data->fd_data = ptr;
  945. _st_epoll_data->fd_data_size = n;
  946. return 0;
  947. }
  948. ST_HIDDEN void _st_epoll_evtlist_expand(void)
  949. {
  950. struct epoll_event *ptr;
  951. int n = _st_epoll_data->evtlist_size;
  952. while (_st_epoll_data->evtlist_cnt > n)
  953. n <<= 1;
  954. ptr = (struct epoll_event *)realloc(_st_epoll_data->evtlist, n * sizeof(struct epoll_event));
  955. if (ptr) {
  956. _st_epoll_data->evtlist = ptr;
  957. _st_epoll_data->evtlist_size = n;
  958. }
  959. }
  960. ST_HIDDEN void _st_epoll_pollset_del(struct pollfd *pds, int npds)
  961. {
  962. struct epoll_event ev;
  963. struct pollfd *pd;
  964. struct pollfd *epd = pds + npds;
  965. int old_events, events, op;
  966. /*
  967. * It's more or less OK if deleting fails because a descriptor
  968. * will either be closed or deleted in dispatch function after
  969. * it fires.
  970. */
  971. for (pd = pds; pd < epd; pd++) {
  972. old_events = _ST_EPOLL_EVENTS(pd->fd);
  973. if (pd->events & POLLIN)
  974. _ST_EPOLL_READ_CNT(pd->fd)--;
  975. if (pd->events & POLLOUT)
  976. _ST_EPOLL_WRITE_CNT(pd->fd)--;
  977. if (pd->events & POLLPRI)
  978. _ST_EPOLL_EXCEP_CNT(pd->fd)--;
  979. events = _ST_EPOLL_EVENTS(pd->fd);
  980. /*
  981. * The _ST_EPOLL_REVENTS check below is needed so we can use
  982. * this function inside dispatch(). Outside of dispatch()
  983. * _ST_EPOLL_REVENTS is always zero for all descriptors.
  984. */
  985. if (events != old_events && _ST_EPOLL_REVENTS(pd->fd) == 0) {
  986. op = events ? EPOLL_CTL_MOD : EPOLL_CTL_DEL;
  987. ev.events = events;
  988. ev.data.fd = pd->fd;
  989. if (epoll_ctl(_st_epoll_data->epfd, op, pd->fd, &ev) == 0 && op == EPOLL_CTL_DEL) {
  990. _st_epoll_data->evtlist_cnt--;
  991. }
  992. }
  993. }
  994. }
  995. ST_HIDDEN int _st_epoll_pollset_add(struct pollfd *pds, int npds)
  996. {
  997. struct epoll_event ev;
  998. int i, fd;
  999. int old_events, events, op;
  1000. /* Do as many checks as possible up front */
  1001. for (i = 0; i < npds; i++) {
  1002. fd = pds[i].fd;
  1003. if (fd < 0 || !pds[i].events ||
  1004. (pds[i].events & ~(POLLIN | POLLOUT | POLLPRI))) {
  1005. errno = EINVAL;
  1006. return -1;
  1007. }
  1008. if (fd >= _st_epoll_data->fd_data_size && _st_epoll_fd_data_expand(fd) < 0)
  1009. return -1;
  1010. }
  1011. for (i = 0; i < npds; i++) {
  1012. fd = pds[i].fd;
  1013. old_events = _ST_EPOLL_EVENTS(fd);
  1014. if (pds[i].events & POLLIN)
  1015. _ST_EPOLL_READ_CNT(fd)++;
  1016. if (pds[i].events & POLLOUT)
  1017. _ST_EPOLL_WRITE_CNT(fd)++;
  1018. if (pds[i].events & POLLPRI)
  1019. _ST_EPOLL_EXCEP_CNT(fd)++;
  1020. events = _ST_EPOLL_EVENTS(fd);
  1021. if (events != old_events) {
  1022. op = old_events ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
  1023. ev.events = events;
  1024. ev.data.fd = fd;
  1025. if (epoll_ctl(_st_epoll_data->epfd, op, fd, &ev) < 0 && (op != EPOLL_CTL_ADD || errno != EEXIST))
  1026. break;
  1027. if (op == EPOLL_CTL_ADD) {
  1028. _st_epoll_data->evtlist_cnt++;
  1029. if (_st_epoll_data->evtlist_cnt > _st_epoll_data->evtlist_size)
  1030. _st_epoll_evtlist_expand();
  1031. }
  1032. }
  1033. }
  1034. if (i < npds) {
  1035. /* Error */
  1036. int err = errno;
  1037. /* Unroll the state */
  1038. _st_epoll_pollset_del(pds, i + 1);
  1039. errno = err;
  1040. return -1;
  1041. }
  1042. return 0;
  1043. }
  1044. ST_HIDDEN void _st_epoll_dispatch(void)
  1045. {
  1046. st_utime_t min_timeout;
  1047. _st_clist_t *q;
  1048. _st_pollq_t *pq;
  1049. struct pollfd *pds, *epds;
  1050. struct epoll_event ev;
  1051. int timeout, nfd, i, osfd, notify;
  1052. int events, op;
  1053. short revents;
  1054. #if defined(DEBUG) && defined(DEBUG_STATS)
  1055. ++_st_stat_epoll;
  1056. #endif
  1057. if (_ST_SLEEPQ == NULL) {
  1058. timeout = -1;
  1059. } else {
  1060. min_timeout = (_ST_SLEEPQ->due <= _ST_LAST_CLOCK) ? 0 : (_ST_SLEEPQ->due - _ST_LAST_CLOCK);
  1061. timeout = (int) (min_timeout / 1000);
  1062. // At least wait 1ms when <1ms, to avoid epoll_wait spin loop.
  1063. if (timeout == 0) {
  1064. #if defined(DEBUG) && defined(DEBUG_STATS)
  1065. ++_st_stat_epoll_zero;
  1066. #endif
  1067. if (min_timeout > 0) {
  1068. #if defined(DEBUG) && defined(DEBUG_STATS)
  1069. ++_st_stat_epoll_shake;
  1070. #endif
  1071. timeout = 1;
  1072. }
  1073. }
  1074. }
  1075. if (_st_epoll_data->pid != getpid()) {
  1076. /* We probably forked, reinitialize epoll set */
  1077. close(_st_epoll_data->epfd);
  1078. _st_epoll_data->epfd = epoll_create(_st_epoll_data->fd_hint);
  1079. if (_st_epoll_data->epfd < 0) {
  1080. /* There is nothing we can do here, will retry later */
  1081. return;
  1082. }
  1083. fcntl(_st_epoll_data->epfd, F_SETFD, FD_CLOEXEC);
  1084. _st_epoll_data->pid = getpid();
  1085. /* Put all descriptors on ioq into new epoll set */
  1086. memset(_st_epoll_data->fd_data, 0, _st_epoll_data->fd_data_size * sizeof(_epoll_fd_data_t));
  1087. _st_epoll_data->evtlist_cnt = 0;
  1088. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  1089. pq = _ST_POLLQUEUE_PTR(q);
  1090. _st_epoll_pollset_add(pq->pds, pq->npds);
  1091. }
  1092. }
  1093. /* Check for I/O operations */
  1094. nfd = epoll_wait(_st_epoll_data->epfd, _st_epoll_data->evtlist, _st_epoll_data->evtlist_size, timeout);
  1095. #if defined(DEBUG) && defined(DEBUG_STATS)
  1096. if (nfd <= 0) {
  1097. ++_st_stat_epoll_spin;
  1098. }
  1099. #endif
  1100. if (nfd > 0) {
  1101. for (i = 0; i < nfd; i++) {
  1102. osfd = _st_epoll_data->evtlist[i].data.fd;
  1103. _ST_EPOLL_REVENTS(osfd) = _st_epoll_data->evtlist[i].events;
  1104. if (_ST_EPOLL_REVENTS(osfd) & (EPOLLERR | EPOLLHUP)) {
  1105. /* Also set I/O bits on error */
  1106. _ST_EPOLL_REVENTS(osfd) |= _ST_EPOLL_EVENTS(osfd);
  1107. }
  1108. }
  1109. for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {
  1110. pq = _ST_POLLQUEUE_PTR(q);
  1111. notify = 0;
  1112. epds = pq->pds + pq->npds;
  1113. for (pds = pq->pds; pds < epds; pds++) {
  1114. if (_ST_EPOLL_REVENTS(pds->fd) == 0) {
  1115. pds->revents = 0;
  1116. continue;
  1117. }
  1118. osfd = pds->fd;
  1119. events = pds->events;
  1120. revents = 0;
  1121. if ((events & POLLIN) && (_ST_EPOLL_REVENTS(osfd) & EPOLLIN))
  1122. revents |= POLLIN;
  1123. if ((events & POLLOUT) && (_ST_EPOLL_REVENTS(osfd) & EPOLLOUT))
  1124. revents |= POLLOUT;
  1125. if ((events & POLLPRI) && (_ST_EPOLL_REVENTS(osfd) & EPOLLPRI))
  1126. revents |= POLLPRI;
  1127. if (_ST_EPOLL_REVENTS(osfd) & EPOLLERR)
  1128. revents |= POLLERR;
  1129. if (_ST_EPOLL_REVENTS(osfd) & EPOLLHUP)
  1130. revents |= POLLHUP;
  1131. pds->revents = revents;
  1132. if (revents) {
  1133. notify = 1;
  1134. }
  1135. }
  1136. if (notify) {
  1137. ST_REMOVE_LINK(&pq->links);
  1138. pq->on_ioq = 0;
  1139. /*
  1140. * Here we will only delete/modify descriptors that
  1141. * didn't fire (see comments in _st_epoll_pollset_del()).
  1142. */
  1143. _st_epoll_pollset_del(pq->pds, pq->npds);
  1144. if (pq->thread->flags & _ST_FL_ON_SLEEPQ)
  1145. _ST_DEL_SLEEPQ(pq->thread);
  1146. pq->thread->state = _ST_ST_RUNNABLE;
  1147. _ST_ADD_RUNQ(pq->thread);
  1148. }
  1149. }
  1150. for (i = 0; i < nfd; i++) {
  1151. /* Delete/modify descriptors that fired */
  1152. osfd = _st_epoll_data->evtlist[i].data.fd;
  1153. _ST_EPOLL_REVENTS(osfd) = 0;
  1154. events = _ST_EPOLL_EVENTS(osfd);
  1155. op = events ? EPOLL_CTL_MOD : EPOLL_CTL_DEL;
  1156. ev.events = events;
  1157. ev.data.fd = osfd;
  1158. if (epoll_ctl(_st_epoll_data->epfd, op, osfd, &ev) == 0 && op == EPOLL_CTL_DEL) {
  1159. _st_epoll_data->evtlist_cnt--;
  1160. }
  1161. }
  1162. }
  1163. }
  1164. ST_HIDDEN int _st_epoll_fd_new(int osfd)
  1165. {
  1166. if (osfd >= _st_epoll_data->fd_data_size && _st_epoll_fd_data_expand(osfd) < 0)
  1167. return -1;
  1168. return 0;
  1169. }
  1170. ST_HIDDEN int _st_epoll_fd_close(int osfd)
  1171. {
  1172. if (_ST_EPOLL_READ_CNT(osfd) || _ST_EPOLL_WRITE_CNT(osfd) || _ST_EPOLL_EXCEP_CNT(osfd)) {
  1173. errno = EBUSY;
  1174. return -1;
  1175. }
  1176. return 0;
  1177. }
  1178. ST_HIDDEN int _st_epoll_fd_getlimit(void)
  1179. {
  1180. /* zero means no specific limit */
  1181. return 0;
  1182. }
  1183. /*
  1184. * Check if epoll functions are just stubs.
  1185. */
  1186. ST_HIDDEN int _st_epoll_is_supported(void)
  1187. {
  1188. struct epoll_event ev;
  1189. ev.events = EPOLLIN;
  1190. ev.data.ptr = NULL;
  1191. /* Guaranteed to fail */
  1192. epoll_ctl(-1, EPOLL_CTL_ADD, -1, &ev);
  1193. return (errno != ENOSYS);
  1194. }
  1195. static _st_eventsys_t _st_epoll_eventsys = {
  1196. "epoll",
  1197. ST_EVENTSYS_ALT,
  1198. _st_epoll_init,
  1199. _st_epoll_dispatch,
  1200. _st_epoll_pollset_add,
  1201. _st_epoll_pollset_del,
  1202. _st_epoll_fd_new,
  1203. _st_epoll_fd_close,
  1204. _st_epoll_fd_getlimit
  1205. };
  1206. #endif /* MD_HAVE_EPOLL */
  1207. /*****************************************
  1208. * Public functions
  1209. */
  1210. int st_set_eventsys(int eventsys)
  1211. {
  1212. if (_st_eventsys) {
  1213. errno = EBUSY;
  1214. return -1;
  1215. }
  1216. switch (eventsys) {
  1217. case ST_EVENTSYS_DEFAULT:
  1218. #ifdef USE_POLL
  1219. _st_eventsys = &_st_poll_eventsys;
  1220. #else
  1221. _st_eventsys = &_st_select_eventsys;
  1222. #endif
  1223. break;
  1224. case ST_EVENTSYS_SELECT:
  1225. _st_eventsys = &_st_select_eventsys;
  1226. break;
  1227. #ifdef MD_HAVE_POLL
  1228. case ST_EVENTSYS_POLL:
  1229. _st_eventsys = &_st_poll_eventsys;
  1230. break;
  1231. #endif
  1232. case ST_EVENTSYS_ALT:
  1233. #if defined (MD_HAVE_KQUEUE)
  1234. _st_eventsys = &_st_kq_eventsys;
  1235. #elif defined (MD_HAVE_EPOLL)
  1236. if (_st_epoll_is_supported())
  1237. _st_eventsys = &_st_epoll_eventsys;
  1238. #endif
  1239. break;
  1240. default:
  1241. errno = EINVAL;
  1242. return -1;
  1243. }
  1244. return 0;
  1245. }
  1246. int st_get_eventsys(void)
  1247. {
  1248. return _st_eventsys ? _st_eventsys->val : -1;
  1249. }
  1250. const char *st_get_eventsys_name(void)
  1251. {
  1252. return _st_eventsys ? _st_eventsys->name : "";
  1253. }