1 #define __MOS_CORE_
2
3 #ifndef NEWEV
4
5 /* NOTE TODO:
6 * We can improve performance and reduce memory usage by making MOS_NULL
7 * hook and MOS_HK_RCV share event structures since no event can be registered
8 * to both hooks. */
9
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <stdint.h>
13 #include <errno.h>
14 #include <assert.h>
15 #include <string.h>
16
17 #include "mtcp.h"
18 #include "event_callback.h"
19 #include "debug.h"
20 #include "mos_api.h"
21 #include "mtcp_api.h"
22 /*----------------------------------------------------------------------------*/
23 enum {
24 OP_REG,
25 OP_UNREG,
26 };
27
28 /* macros */
29 #define IS_PO2(x) (!((x - 1) & (x)))
30 #define NUM_UDE (MAX_EV - UDE_OFFSET)
31 #define EVENT_FOREACH_START(ev, idx, from, to, map) \
32 do {int idx; for (idx = (from); idx < (to); idx++) { \
33 const event_t ev = 1L << idx; if (!((map) & (ev))) continue;
34 #define EVENT_FOREACH_END(ev, map) \
35 if (!((map) &= ~ev)) break;}} while (0)
36
37 /* Global variables (per CPU core) */
38 static struct {
39 filter_t ft;
40 event_t ancestors;
41 int8_t parent;
42 } g_udes[NUM_UDE];
43 static event_t g_descendants[MAX_EV];
44 static event_t g_ude_map;
45 static const event_t g_bev_map = (~(((event_t)-1) << NUM_BEV)) << BEV_OFFSET;
46 static int8_t g_ude_id[MAX_EV][NUM_UDE];
47
48 /* FIXME: ft_map is not managed properly (especially in UnregCb()). */
49
50 /*----------------------------------------------------------------------------*/
51 void
GlobInitEvent(void)52 GlobInitEvent(void)
53 {
54 int i;
55 for (i = 0; i < NUM_UDE; i++) {
56 g_udes[i].ft = NULL;
57 g_udes[i].ancestors = 0;
58 g_udes[i].parent = -1;
59 }
60
61 for (i = 0; i < MAX_EV; i++)
62 g_descendants[i] = 0;
63
64 memset(g_ude_id, -1, MAX_EV * NUM_UDE);
65 g_ude_map = 0;
66 }
67 /*----------------------------------------------------------------------------*/
68 void
InitEvent(mtcp_manager_t mtcp,int num_evt)69 InitEvent(mtcp_manager_t mtcp, int num_evt)
70 {
71 if (!(mtcp->evt_pool = MPCreate(sizeof(struct ev_table),
72 sizeof(struct ev_table) * num_evt, 0))) {
73 TRACE_ERROR("Failed to allocate ev_table pool\n");
74 exit(0);
75 }
76 }
77 /*----------------------------------------------------------------------------*/
78 void
InitEvP(struct ev_pointer * evp,struct ev_base * evb)79 InitEvP(struct ev_pointer *evp, struct ev_base *evb)
80 {
81 *evp = evb->dflt_evp;
82 event_t map = evp->cb_map;
83
84 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, map) {
85 evp->evt->ent[i].ref++;
86 } EVENT_FOREACH_END(ev, map);
87
88 if (!map)
89 return;
90
91 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, map) {
92 evp->evt->ent[i].ref++;
93 } EVENT_FOREACH_END(ev, map);
94 }
95 /*----------------------------------------------------------------------------*/
96 void
CleanupEvP(struct ev_pointer * evp)97 CleanupEvP(struct ev_pointer *evp)
98 {
99 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, evp->cb_map) {
100 evp->evt->ent[i].ref--;
101 } EVENT_FOREACH_END(ev, evp->cb_map);
102
103 if (!evp->cb_map)
104 return;
105
106 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, evp->cb_map) {
107 evp->evt->ent[i].ref--;
108 } EVENT_FOREACH_END(ev, evp->cb_map);
109 }
110 /*----------------------------------------------------------------------------*/
111 void
InitEvB(mtcp_manager_t mtcp,struct ev_base * evb)112 InitEvB(mtcp_manager_t mtcp, struct ev_base *evb)
113 {
114 TAILQ_INIT(&evb->evth);
115 struct ev_table *dflt_evt = MPAllocateChunk(mtcp->evt_pool);
116 memset(dflt_evt, 0, sizeof(struct ev_table));
117
118 TAILQ_INSERT_HEAD(&evb->evth, dflt_evt, link);
119 evb->dflt_evp.cb_map = 0;
120 evb->dflt_evp.ft_map = 0;
121 evb->dflt_evp.evt = dflt_evt;
122 }
123 /*----------------------------------------------------------------------------*/
124 void
CleanupEvB(mtcp_manager_t mtcp,struct ev_base * evb)125 CleanupEvB(mtcp_manager_t mtcp, struct ev_base *evb)
126 {
127 struct ev_table *walk, *tmp;
128 for (walk = TAILQ_FIRST(&evb->evth); walk != NULL; walk = tmp) {
129 tmp = TAILQ_NEXT(walk, link);
130
131 MPFreeChunk(mtcp->evt_pool, walk);
132 }
133 }
134 /*----------------------------------------------------------------------------*/
135 static inline void
RegCbWCpy(struct ev_pointer * evp,struct ev_table * new_evt,event_t events,void * cb)136 RegCbWCpy(struct ev_pointer *evp, struct ev_table *new_evt,
137 event_t events, void *cb)
138 {
139 /* NOTE: We may apply binary search which is O(log(N)) later, while current
140 * linear search is O(N). */
141 event_t evcpy = 0, ev_total;
142 event_t ev_inc_ref = 0, ev_dec_ref = 0;
143 struct ev_table *cur_evt = evp->evt;
144
145 event_t overlap = events & new_evt->map;
146
147 assert(evp->evt != new_evt);
148 assert(!(evp->cb_map & events));
149 assert((evp->cb_map & cur_evt->map) == evp->cb_map);
150
151 /* event table will be changed to new_evt */
152 ev_total = events | evp->cb_map;
153 evcpy = evp->cb_map & ~new_evt->map;
154 evp->evt = new_evt;
155
156 ev_inc_ref = events | evp->cb_map;
157 ev_dec_ref = evp->cb_map;
158
159 new_evt->map |= ev_total;
160 evp->cb_map |= events;
161
162 /* For built-in events */
163 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, ev_total) {
164 if (events & ev) {
165 assert((ev & overlap) ? new_evt->ent[i].cb == cb
166 : new_evt->ent[i].ref == 0);
167 if (!(ev & overlap))
168 new_evt->ent[i].cb = cb;
169 } else if (evcpy & ev) {
170 assert(new_evt && new_evt != cur_evt);
171 new_evt->ent[i].cb = cur_evt->ent[i].cb;
172 }
173
174 /* ev_dec_ref is subset of ev_inc_ref */
175 if (ev_inc_ref & ev) {
176 new_evt->ent[i].ref++;
177 if (!(new_evt->map & ev))
178 new_evt->map |= ev;
179 if (ev_dec_ref & ev) {
180 if (--cur_evt->ent[i].ref)
181 cur_evt->map &= ~ev;
182 }
183 }
184 } EVENT_FOREACH_END(ev, ev_total);
185
186 if (!ev_total)
187 return;
188
189 /* For UDEs */
190 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, ev_total) {
191 if (events & ev) {
192 assert((ev & overlap) ? new_evt->ent[i].cb == cb
193 : new_evt->ent[i].ref == 0);
194 if (!(ev & overlap))
195 new_evt->ent[i].cb = cb;
196 /* update ft_map */
197 assert(g_udes[i - UDE_OFFSET].ft);
198 assert(g_udes[i - UDE_OFFSET].ancestors);
199 evp->ft_map |= g_udes[i - UDE_OFFSET].ancestors;
200 } else if (evcpy & ev) {
201 assert(new_evt && new_evt != cur_evt);
202 new_evt->ent[i].cb = cur_evt->ent[i].cb;
203 }
204
205 /* ev_dec_ref is subset of ev_inc_ref */
206 if (ev_inc_ref & ev) {
207 new_evt->ent[i].ref++;
208 if (!(new_evt->map & ev))
209 new_evt->map |= ev;
210 if (ev_dec_ref & ev) {
211 if (--cur_evt->ent[i].ref)
212 cur_evt->map &= ~ev;
213 }
214 }
215 } EVENT_FOREACH_END(ev, ev_total);
216 }
217 /*----------------------------------------------------------------------------*/
218 static inline void
RegCbWoCpy(struct ev_pointer * evp,event_t events,void * cb)219 RegCbWoCpy(struct ev_pointer *evp, event_t events, void *cb)
220 {
221 /* NOTE: We may apply binary search which is O(log(N)) later, while current
222 * linear search is O(N). */
223 event_t ev_inc_ref = 0;
224 struct ev_table *cur_evt = evp->evt;
225
226 event_t overlap = events & cur_evt->map;
227
228 assert(!(evp->cb_map & events));
229 assert((evp->cb_map & cur_evt->map) == evp->cb_map);
230
231 ev_inc_ref = events;
232
233 cur_evt->map |= events;
234 evp->cb_map |= events;
235
236 /* For built-in events */
237 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, events) {
238 if (events & ev) {
239 assert((ev & overlap) ? cur_evt->ent[i].cb == cb
240 : cur_evt->ent[i].ref == 0);
241 if (!(ev & overlap))
242 cur_evt->ent[i].cb = cb;
243 }
244
245 /* ev_dec_ref is subset of ev_inc_ref */
246 if (ev_inc_ref & ev) {
247 cur_evt->ent[i].ref++;
248 if (!(cur_evt->map & ev))
249 cur_evt->map |= ev;
250 }
251 } EVENT_FOREACH_END(ev, events);
252
253 if (!events)
254 return;
255
256 /* For UDEs */
257 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, events) {
258 if (events & ev) {
259 assert((ev & overlap) ? cur_evt->ent[i].cb == cb
260 : cur_evt->ent[i].ref == 0);
261 if (!(ev & overlap))
262 cur_evt->ent[i].cb = cb;
263 /* update ft_map */
264 assert(g_udes[i - UDE_OFFSET].ft);
265 assert(g_udes[i - UDE_OFFSET].ancestors);
266 evp->ft_map |= g_udes[i - UDE_OFFSET].ancestors;
267 }
268
269 /* ev_dec_ref is subset of ev_inc_ref */
270 if (ev_inc_ref & ev) {
271 cur_evt->ent[i].ref++;
272 if (!(cur_evt->map & ev))
273 cur_evt->map |= ev;
274 }
275 } EVENT_FOREACH_END(ev, events);
276 }
277 /*----------------------------------------------------------------------------*/
278 static inline void
UnregCb(struct ev_pointer * evp,event_t events)279 UnregCb(struct ev_pointer *evp, event_t events)
280 {
281 assert(evp);
282
283 struct ev_table *evt = evp->evt;
284 evp->cb_map &= ~events;
285
286 /* Unregister unnecessary UDEs */
287 if (events & g_ude_map) {
288 event_t evs = events & g_ude_map;
289 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, evs) {
290 int walk = i;
291 while (1) {
292 const event_t evid = 1L << walk;
293 if (/* no registered callback */
294 !(evid & evp->cb_map)
295 /* no child events */
296 && !(g_descendants[walk] & evp->cb_map)) {
297 /* this UDE filter is useless */
298 evp->ft_map &= ~(1L << g_udes[walk - UDE_OFFSET].parent);
299 /* No need to see this event in rest of EVENT_FOREACH */
300 evs &= ~evid;
301 if ((walk = g_udes[walk - UDE_OFFSET].parent) < UDE_OFFSET)
302 break;
303 } else
304 break;
305 }
306 } EVENT_FOREACH_END(ev, evs);
307 }
308
309 /* Placing reference counter for each event table entry, instead of each
310 * event table, and decrement them for every callback unregistration may
311 * look inefficient. However, actually, it does NOT. If reference counter
312 * is for each event table, then we need to call FindReusableEvT() for
313 * every callback unregistration to find reusable event table.
314 * FindReusableEvT() is heavier than per-event reference counter update.
315 * And that way also wastes memory. */
316
317 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, events) {
318 if (--evt->ent[i].ref == 0)
319 evt->map &= ~ev;
320 } EVENT_FOREACH_END(ev, events);
321
322 if (!events)
323 return;
324
325 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, events) {
326 if (--evt->ent[i].ref == 0)
327 evt->map &= ~ev;
328 } EVENT_FOREACH_END(ev, events);
329 }
330 /*----------------------------------------------------------------------------*/
331 static inline struct ev_table *
FindReusableEvT(struct ev_pointer * evp,struct ev_base * evb,event_t events,void * cb)332 FindReusableEvT(struct ev_pointer *evp, struct ev_base *evb,
333 event_t events, void *cb)
334 {
335 struct ev_table *cur_evt = evp->evt;
336 struct ev_table *walk;
337
338 assert((evp->cb_map & cur_evt->map) == evp->cb_map);
339
340 TAILQ_FOREACH(walk, &evb->evth, link) {
341 event_t overlap = evp->cb_map & walk->map;
342 assert((events & overlap) == 0);
343 event_t ev_total = events | overlap;
344
345 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, ev_total) {
346 if (ev & events) {
347 if (walk->ent[i].cb != cb)
348 goto __continue;
349 } else /* if (ev & overlap) */ {
350 if (walk->ent[i].cb != cur_evt->ent[i].cb)
351 goto __continue;
352 }
353 } EVENT_FOREACH_END(ev, ev_total);
354
355 if (!ev_total)
356 return walk;
357
358 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, ev_total) {
359 if (ev & events) {
360 if (walk->ent[i].cb != cb)
361 goto __continue;
362 } else /* if (ev & overlap) */ {
363 if (walk->ent[i].cb != cur_evt->ent[i].cb)
364 goto __continue;
365 }
366 } EVENT_FOREACH_END(ev, ev_total);
367
368 if (!ev_total)
369 return walk;
370 __continue:
371 continue;
372 }
373
374 return NULL;
375 }
376 /*----------------------------------------------------------------------------*/
377 inline int
ModCb(mtcp_manager_t mtcp,int op,struct ev_pointer * evp,struct ev_base * evb,event_t events,void * cb)378 ModCb(mtcp_manager_t mtcp, int op, struct ev_pointer *evp, struct ev_base *evb,
379 event_t events, void *cb)
380 {
381 struct ev_table *evt = evp->evt;
382
383 assert(evt);
384
385 if (op == OP_REG) {
386 /* NOTE: we do not register new callback if correponding 'map' is
387 * occupied */
388 if (events & evp->cb_map) {
389 /* callback overwrite error */
390 errno = EINVAL;
391 return -1;
392 } else if (events & evt->map) {
393 /* event registration conflict */
394 struct ev_table *nevt;
395 if (!(nevt = FindReusableEvT(evp, evb, events, cb))) {
396 nevt = MPAllocateChunk(mtcp->evt_pool);
397 assert(nevt);
398 TAILQ_INSERT_HEAD(&evb->evth, nevt, link);
399 }
400
401 /* register callback */
402 if (nevt != evt)
403 RegCbWCpy(evp, nevt, events, cb);
404 else
405 RegCbWoCpy(evp, events, cb);
406
407 } else {
408 /* reuse default event table */
409 RegCbWoCpy(evp, events, cb);
410 }
411 } else /* if (op == OP_UNREG) */ {
412 if ((events & evp->cb_map) != events) {
413 /* unregister unexisting callback error */
414 errno = EINVAL;
415 return -1;
416 } else {
417 /* unregister callback */
418 UnregCb(evp, events);
419 }
420 }
421
422 return 0;
423 }
424 /*----------------------------------------------------------------------------*/
425 static inline int
ModifyCallback(mctx_t mctx,int op,int sockid,event_t events,int hook_point,void * callback)426 ModifyCallback(mctx_t mctx, int op, int sockid, event_t events,
427 int hook_point, void* callback)
428 {
429 socket_map_t socket;
430 struct ev_pointer *evp;
431 struct ev_base *evb;
432
433 assert(op == OP_REG || op == OP_UNREG);
434
435 if ((events & (g_bev_map | g_ude_map)) != events) {
436 errno = EINVAL;
437 return -1;
438 }
439
440 if ((op == OP_REG) && !callback)
441 return -1;
442
443 mtcp_manager_t mtcp = GetMTCPManager(mctx);
444 if (!mtcp)
445 return -1;
446
447 socket = &mtcp->msmap[sockid];
448
449 if (socket->socktype == MOS_SOCK_MONITOR_STREAM_ACTIVE) {
450 if (hook_point == MOS_NULL) {
451 evp = &socket->monitor_stream->dontcare_evp;
452 evb = &socket->monitor_stream->monitor_listener->dontcare_evb;
453 } else if (hook_point == MOS_HK_RCV) {
454 evp = &socket->monitor_stream->pre_tcp_evp;
455 evb = &socket->monitor_stream->monitor_listener->pre_tcp_evb;
456 } else if (hook_point == MOS_HK_SND) {
457 evp = &socket->monitor_stream->post_tcp_evp;
458 evb = &socket->monitor_stream->monitor_listener->post_tcp_evb;
459 } else
460 return -1;
461
462 } else if (socket->socktype == MOS_SOCK_MONITOR_STREAM
463 || socket->socktype == MOS_SOCK_MONITOR_RAW) {
464 if (hook_point == MOS_NULL)
465 evb = &socket->monitor_listener->dontcare_evb;
466 else if (hook_point == MOS_HK_RCV)
467 evb = &socket->monitor_listener->pre_tcp_evb;
468 else if (hook_point == MOS_HK_SND)
469 evb = &socket->monitor_listener->post_tcp_evb;
470 else
471 return -1;
472
473 evp = &evb->dflt_evp;
474 } else {
475 errno = EINVAL;
476 return -1;
477 }
478
479 return ModCb(mtcp, op, evp, evb, events, callback);
480 }
481 /*----------------------------------------------------------------------------*/
482 int
mtcp_register_callback(mctx_t mctx,int sockid,event_t events,int hook_point,callback_t callback)483 mtcp_register_callback(mctx_t mctx, int sockid, event_t events,
484 int hook_point, callback_t callback)
485 {
486 if (!callback) {
487 errno = EINVAL;
488 return -1;
489 }
490
491 return ModifyCallback(mctx, OP_REG, sockid, events, hook_point, callback);
492 }
493 /*----------------------------------------------------------------------------*/
494 int
mtcp_unregister_callback(mctx_t mctx,int sockid,event_t events,int hook_point)495 mtcp_unregister_callback(mctx_t mctx, int sockid, event_t events,
496 int hook_point)
497 {
498 return ModifyCallback(mctx, OP_UNREG, sockid, events, hook_point, NULL);
499 }
500 /*----------------------------------------------------------------------------*/
501 event_t
mtcp_define_event(event_t event,filter_t filter)502 mtcp_define_event(event_t event, filter_t filter)
503 {
504 int i, j;
505 int evid;
506
507 if (!IS_PO2(event))
508 return 0;
509
510 if (!filter)
511 return 0;
512
513 for (i = 0; i < MAX_EV; i++)
514 if (event == 1L << i) {
515 evid = i;
516 break;
517 }
518 if (i == MAX_EV)
519 return 0;
520
521 for (i = 0; i < NUM_UDE; i++) {
522 const event_t ude = 1L << (i + UDE_OFFSET);
523 if (g_ude_map & ude)
524 continue;
525
526 for (j = 0; j < NUM_UDE; j++)
527 if (g_ude_id[evid][j] == -1) {
528 g_ude_id[evid][j] = i + UDE_OFFSET;
529 break;
530 }
531 if (j == NUM_UDE)
532 return 0;
533
534 /* Now we have valid UDE */
535
536 /* update ancestor's descendants map */
537 event_t ancestors = event |
538 ((evid >= UDE_OFFSET) ? g_udes[evid - UDE_OFFSET].ancestors : 0);
539 EVENT_FOREACH_START(ev, j, BEV_OFFSET, MAX_EV, ancestors) {
540 g_descendants[j] |= ude;
541 } EVENT_FOREACH_END(ev, ancestors);
542
543 /* update my ancestor map */
544 if (event & g_ude_map)
545 g_udes[i].ancestors = event | g_udes[evid - UDE_OFFSET].ancestors;
546 else
547 g_udes[i].ancestors = event;
548
549 g_udes[i].parent = evid;
550 g_udes[i].ft = filter;
551 g_ude_map |= ude;
552
553 return ude;
554 }
555
556 return 0;
557 }
558 /*----------------------------------------------------------------------------*/
559 int
mtcp_remove_ude(event_t event)560 mtcp_remove_ude(event_t event)
561 {
562 /* FIXME: this function is not implemented yet.
563 * What should we do if we remove UDE while running? */
564 if (!IS_PO2(event))
565 return -1;
566
567 if (!(g_ude_map & event))
568 return -1;
569
570 return -1;
571 }
572 /*----------------------------------------------------------------------------*/
573 #define PUSH(sp, ev_idx, ft_idx, data) \
574 do { \
575 sp->ev_idx = ev_idx; \
576 sp->ft_idx = ft_idx; \
577 sp->data.u64 = data.u64; \
578 sp++; \
579 } while (0)
580 #define POP(sp, ev_idx, ft_idx, data) \
581 do { \
582 sp--; \
583 ev_idx = sp->ev_idx; \
584 ft_idx = sp->ft_idx; \
585 data.u64 = sp->data.u64; \
586 } while (0)
587 /*----------------------------------------------------------------------------*/
588 /**
589 * TODO:
590 * - Donghwi, please change POST_TCP names to POST_SND &
591 * PRE_TCP names to POST_RCV.
592 *
593 * - Please make sure that the order of event invocations is:
594 * MOS_ON_CONN_START --> .. MOS_ON_* .. --> MOS_ON_CONN_END
595 */
596 inline void
HandleCallback(mtcp_manager_t mtcp,uint32_t hook,socket_map_t socket,int side,struct pkt_ctx * pctx,event_t events)597 HandleCallback(mtcp_manager_t mtcp, uint32_t hook,
598 socket_map_t socket, int side, struct pkt_ctx *pctx, event_t events)
599 {
600 struct sfbpf_program fcode;
601 int8_t ude_id;
602 uint64_t cb_map, ft_map;
603
604 int8_t ev_idx, ft_idx;
605 event_data_t data;
606
607 if (!socket)
608 return;
609
610 if (!events)
611 return;
612 assert(events);
613
614 /* if client side monitoring is disabled, then skip */
615 if (side == MOS_SIDE_CLI && socket->monitor_stream->client_mon == 0)
616 return;
617 /* if server side monitoring is disabled, then skip */
618 else if (side == MOS_SIDE_SVR && socket->monitor_stream->server_mon == 0)
619 return;
620
621 #define MSTRM(sock) (sock)->monitor_stream
622 #define MLSNR(sock) (sock)->monitor_listener
623 /* We use `?:` notation instead of `if/else` to make `evp` as const */
624 struct ev_pointer * const evp =
625 (socket->socktype == MOS_SOCK_MONITOR_STREAM_ACTIVE) ?
626 ((hook == MOS_HK_RCV) ? &MSTRM(socket)->pre_tcp_evp :
627 (hook == MOS_HK_SND) ? &MSTRM(socket)->post_tcp_evp :
628 &MSTRM(socket)->dontcare_evp)
629 : (socket->socktype == MOS_SOCK_MONITOR_STREAM)
630 || (socket->socktype == MOS_SOCK_MONITOR_RAW) ?
631 ((hook == MOS_HK_RCV) ? &MLSNR(socket)->pre_tcp_evb.dflt_evp :
632 (hook == MOS_HK_SND) ? &MLSNR(socket)->post_tcp_evb.dflt_evp :
633 &MLSNR(socket)->dontcare_evb.dflt_evp) :
634 NULL;
635
636 if (!evp || !((cb_map = events & evp->cb_map) || (g_ude_map & evp->cb_map)))
637 return;
638
639 /* mtcp_bind_monitor_filter()
640 * - BPF filter is evaluated only for RAW socket and PASSIVE socket (= orphan filter)
641 * - stream syn filter is moved to and evaluated on socket creation */
642 if (socket->socktype == MOS_SOCK_MONITOR_STREAM) {
643 fcode = MLSNR(socket)->stream_orphan_fcode;
644 /* if not match with filter, return */
645 if (ISSET_BPFFILTER(fcode) && pctx && EVAL_BPFFILTER(fcode,
646 (uint8_t *)pctx->p.iph - sizeof(struct ethhdr),
647 pctx->p.ip_len + sizeof(struct ethhdr)) == 0)
648 return;
649 }
650 if (socket->socktype == MOS_SOCK_MONITOR_RAW) {
651 fcode = MLSNR(socket)->raw_pkt_fcode;
652 /* if not match with filter, return */
653 if (ISSET_BPFFILTER(fcode) && pctx && EVAL_BPFFILTER(fcode,
654 (uint8_t *)pctx->p.iph - sizeof(struct ethhdr),
655 pctx->p.ip_len + sizeof(struct ethhdr)) == 0)
656 return;
657 }
658
659 ft_map = events & evp->ft_map;
660
661 event_t bev_map = cb_map | ft_map;
662 struct ev_table * const evt = evp->evt;
663
664 struct {
665 int8_t ev_idx;
666 int8_t ft_idx;
667 event_data_t data;
668 } stack[NUM_UDE + 1], *sp = stack;
669
670 mtcp->pctx = pctx; /* for mtcp_getlastpkt() */
671 mctx_t const mctx = g_ctx[mtcp->ctx->cpu];
672
673 EVENT_FOREACH_START(bev, bidx, BEV_OFFSET, BEV_OFFSET + NUM_BEV, bev_map) {
674 ev_idx = bidx;
675 ft_idx = 0;
676 data.u64 = 0;
677 const event_t descendants = g_descendants[ev_idx];
678 if (descendants) {
679 cb_map |= descendants & evp->cb_map;
680 ft_map |= descendants & evp->ft_map;
681 }
682
683 while (1) {
684 const uint64_t ev = (1L << ev_idx);
685
686 if (cb_map & ev) {
687 /* call callback */
688 evt->ent[ev_idx].cb(mctx, socket->id, side, ev, data);
689
690 if (!(cb_map &= ~ev))
691 return;
692 }
693
694 while (1) {
695 event_data_t tmpdata;
696 if (ft_idx >= NUM_UDE
697 || (ude_id = g_ude_id[ev_idx][ft_idx]) < 0) {
698 /* done with this event */
699 if (sp == stack /* stack is empty */) {
700 /* go to next built-in event */
701 goto __continue;
702 } else {
703 POP(sp, ev_idx, ft_idx, data);
704 ft_idx++;
705 }
706 break;
707 }
708
709 assert(ude_id >= UDE_OFFSET && ude_id < MAX_EV);
710
711 if (((1L << ude_id) & (cb_map | ft_map)) &&
712 (tmpdata.u64 = g_udes[ude_id - UDE_OFFSET].ft(mctx, socket->id, side, ev, data))) {
713 /* DFS jump */
714 PUSH(sp, ev_idx, ft_idx, data);
715 ev_idx = ude_id;
716 ft_idx = 0;
717 data.u64 = tmpdata.u64;
718 break;
719 }
720
721 ft_idx++;
722 }
723 }
724 }
725 __continue:
726 EVENT_FOREACH_END(bev, bev_map);
727 }
728 /*----------------------------------------------------------------------------*/
729
730 #endif
731