1 #define __MOS_CORE_ 2 3 #ifndef NEWEV 4 5 /* NOTE TODO: 6 * We can improve performance and reduce memory usage by making MOS_NULL 7 * hook and MOS_HK_RCV share event structures since no event can be registered 8 * to both hooks. */ 9 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <stdint.h> 13 #include <errno.h> 14 #include <assert.h> 15 #include <string.h> 16 17 #include "mtcp.h" 18 #include "event_callback.h" 19 #include "debug.h" 20 #include "mos_api.h" 21 #include "mtcp_api.h" 22 /*----------------------------------------------------------------------------*/ 23 enum { 24 OP_REG, 25 OP_UNREG, 26 }; 27 28 /* macros */ 29 #define IS_PO2(x) (!((x - 1) & (x))) 30 #define NUM_UDE (MAX_EV - UDE_OFFSET) 31 #define EVENT_FOREACH_START(ev, idx, from, to, map) \ 32 do {int idx; for (idx = (from); idx < (to); idx++) { \ 33 const event_t ev = 1L << idx; if (!((map) & (ev))) continue; 34 #define EVENT_FOREACH_END(ev, map) \ 35 if (!((map) &= ~ev)) break;}} while (0) 36 37 /* Global variables (per CPU core) */ 38 static struct { 39 filter_t ft; 40 event_t ancestors; 41 int8_t parent; 42 } g_udes[NUM_UDE]; 43 static event_t g_descendants[MAX_EV]; 44 static event_t g_ude_map; 45 static const event_t g_bev_map = (~(((event_t)-1) << NUM_BEV)) << BEV_OFFSET; 46 static int8_t g_ude_id[MAX_EV][NUM_UDE]; 47 48 /* FIXME: ft_map is not managed properly (especially in UnregCb()). */ 49 50 /*----------------------------------------------------------------------------*/ 51 void 52 GlobInitEvent(void) 53 { 54 int i; 55 for (i = 0; i < NUM_UDE; i++) { 56 g_udes[i].ft = NULL; 57 g_udes[i].ancestors = 0; 58 g_udes[i].parent = -1; 59 } 60 61 for (i = 0; i < MAX_EV; i++) 62 g_descendants[i] = 0; 63 64 memset(g_ude_id, -1, MAX_EV * NUM_UDE); 65 g_ude_map = 0; 66 } 67 /*----------------------------------------------------------------------------*/ 68 void 69 InitEvent(mtcp_manager_t mtcp, int num_evt) 70 { 71 if (!(mtcp->evt_pool = MPCreate(sizeof(struct ev_table), 72 sizeof(struct ev_table) * num_evt, 0))) { 73 TRACE_ERROR("Failed to allocate ev_table pool\n"); 74 exit(0); 75 } 76 } 77 /*----------------------------------------------------------------------------*/ 78 void 79 InitEvP(struct ev_pointer *evp, struct ev_base *evb) 80 { 81 *evp = evb->dflt_evp; 82 event_t map = evp->cb_map; 83 84 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, map) { 85 evp->evt->ent[i].ref++; 86 } EVENT_FOREACH_END(ev, map); 87 88 if (!map) 89 return; 90 91 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, map) { 92 evp->evt->ent[i].ref++; 93 } EVENT_FOREACH_END(ev, map); 94 } 95 /*----------------------------------------------------------------------------*/ 96 void 97 CleanupEvP(struct ev_pointer *evp) 98 { 99 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, evp->cb_map) { 100 evp->evt->ent[i].ref--; 101 } EVENT_FOREACH_END(ev, evp->cb_map); 102 103 if (!evp->cb_map) 104 return; 105 106 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, evp->cb_map) { 107 evp->evt->ent[i].ref--; 108 } EVENT_FOREACH_END(ev, evp->cb_map); 109 } 110 /*----------------------------------------------------------------------------*/ 111 void 112 InitEvB(mtcp_manager_t mtcp, struct ev_base *evb) 113 { 114 TAILQ_INIT(&evb->evth); 115 struct ev_table *dflt_evt = MPAllocateChunk(mtcp->evt_pool); 116 memset(dflt_evt, 0, sizeof(struct ev_table)); 117 118 TAILQ_INSERT_HEAD(&evb->evth, dflt_evt, link); 119 evb->dflt_evp.cb_map = 0; 120 evb->dflt_evp.ft_map = 0; 121 evb->dflt_evp.evt = dflt_evt; 122 } 123 /*----------------------------------------------------------------------------*/ 124 void 125 CleanupEvB(mtcp_manager_t mtcp, struct ev_base *evb) 126 { 127 struct ev_table *walk, *tmp; 128 for (walk = TAILQ_FIRST(&evb->evth); walk != NULL; walk = tmp) { 129 tmp = TAILQ_NEXT(walk, link); 130 131 MPFreeChunk(mtcp->evt_pool, walk); 132 } 133 } 134 /*----------------------------------------------------------------------------*/ 135 static inline void 136 RegCbWCpy(struct ev_pointer *evp, struct ev_table *new_evt, 137 event_t events, void *cb) 138 { 139 /* NOTE: We may apply binary search which is O(log(N)) later, while current 140 * linear search is O(N). */ 141 event_t evcpy = 0, ev_total; 142 event_t ev_inc_ref = 0, ev_dec_ref = 0; 143 struct ev_table *cur_evt = evp->evt; 144 145 event_t overlap = events & new_evt->map; 146 147 assert(evp->evt != new_evt); 148 assert(!(evp->cb_map & events)); 149 assert((evp->cb_map & cur_evt->map) == evp->cb_map); 150 151 /* event table will be changed to new_evt */ 152 ev_total = events | evp->cb_map; 153 evcpy = evp->cb_map & ~new_evt->map; 154 evp->evt = new_evt; 155 156 ev_inc_ref = events | evp->cb_map; 157 ev_dec_ref = evp->cb_map; 158 159 new_evt->map |= ev_total; 160 evp->cb_map |= events; 161 162 /* For built-in events */ 163 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, ev_total) { 164 if (events & ev) { 165 assert((ev & overlap) ? new_evt->ent[i].cb == cb 166 : new_evt->ent[i].ref == 0); 167 if (!(ev & overlap)) 168 new_evt->ent[i].cb = cb; 169 } else if (evcpy & ev) { 170 assert(new_evt && new_evt != cur_evt); 171 new_evt->ent[i].cb = cur_evt->ent[i].cb; 172 } 173 174 /* ev_dec_ref is subset of ev_inc_ref */ 175 if (ev_inc_ref & ev) { 176 new_evt->ent[i].ref++; 177 if (!(new_evt->map & ev)) 178 new_evt->map |= ev; 179 if (ev_dec_ref & ev) { 180 if (--cur_evt->ent[i].ref) 181 cur_evt->map &= ~ev; 182 } 183 } 184 } EVENT_FOREACH_END(ev, ev_total); 185 186 if (!ev_total) 187 return; 188 189 /* For UDEs */ 190 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, ev_total) { 191 if (events & ev) { 192 assert((ev & overlap) ? new_evt->ent[i].cb == cb 193 : new_evt->ent[i].ref == 0); 194 if (!(ev & overlap)) 195 new_evt->ent[i].cb = cb; 196 /* update ft_map */ 197 assert(g_udes[i - UDE_OFFSET].ft); 198 assert(g_udes[i - UDE_OFFSET].ancestors); 199 evp->ft_map |= g_udes[i - UDE_OFFSET].ancestors; 200 } else if (evcpy & ev) { 201 assert(new_evt && new_evt != cur_evt); 202 new_evt->ent[i].cb = cur_evt->ent[i].cb; 203 } 204 205 /* ev_dec_ref is subset of ev_inc_ref */ 206 if (ev_inc_ref & ev) { 207 new_evt->ent[i].ref++; 208 if (!(new_evt->map & ev)) 209 new_evt->map |= ev; 210 if (ev_dec_ref & ev) { 211 if (--cur_evt->ent[i].ref) 212 cur_evt->map &= ~ev; 213 } 214 } 215 } EVENT_FOREACH_END(ev, ev_total); 216 } 217 /*----------------------------------------------------------------------------*/ 218 static inline void 219 RegCbWoCpy(struct ev_pointer *evp, event_t events, void *cb) 220 { 221 /* NOTE: We may apply binary search which is O(log(N)) later, while current 222 * linear search is O(N). */ 223 event_t ev_inc_ref = 0; 224 struct ev_table *cur_evt = evp->evt; 225 226 event_t overlap = events & cur_evt->map; 227 228 assert(!(evp->cb_map & events)); 229 assert((evp->cb_map & cur_evt->map) == evp->cb_map); 230 231 ev_inc_ref = events; 232 233 cur_evt->map |= events; 234 evp->cb_map |= events; 235 236 /* For built-in events */ 237 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, events) { 238 if (events & ev) { 239 assert((ev & overlap) ? cur_evt->ent[i].cb == cb 240 : cur_evt->ent[i].ref == 0); 241 if (!(ev & overlap)) 242 cur_evt->ent[i].cb = cb; 243 } 244 245 /* ev_dec_ref is subset of ev_inc_ref */ 246 if (ev_inc_ref & ev) { 247 cur_evt->ent[i].ref++; 248 if (!(cur_evt->map & ev)) 249 cur_evt->map |= ev; 250 } 251 } EVENT_FOREACH_END(ev, events); 252 253 if (!events) 254 return; 255 256 /* For UDEs */ 257 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, events) { 258 if (events & ev) { 259 assert((ev & overlap) ? cur_evt->ent[i].cb == cb 260 : cur_evt->ent[i].ref == 0); 261 if (!(ev & overlap)) 262 cur_evt->ent[i].cb = cb; 263 /* update ft_map */ 264 assert(g_udes[i - UDE_OFFSET].ft); 265 assert(g_udes[i - UDE_OFFSET].ancestors); 266 evp->ft_map |= g_udes[i - UDE_OFFSET].ancestors; 267 } 268 269 /* ev_dec_ref is subset of ev_inc_ref */ 270 if (ev_inc_ref & ev) { 271 cur_evt->ent[i].ref++; 272 if (!(cur_evt->map & ev)) 273 cur_evt->map |= ev; 274 } 275 } EVENT_FOREACH_END(ev, events); 276 } 277 /*----------------------------------------------------------------------------*/ 278 static inline void 279 UnregCb(struct ev_pointer *evp, event_t events) 280 { 281 assert(evp); 282 283 struct ev_table *evt = evp->evt; 284 evp->cb_map &= ~events; 285 286 /* Unregister unnecessary UDEs */ 287 if (events & g_ude_map) { 288 event_t evs = events & g_ude_map; 289 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, evs) { 290 int walk = i; 291 while (1) { 292 const event_t evid = 1L << walk; 293 if (/* no registered callback */ 294 !(evid & evp->cb_map) 295 /* no child events */ 296 && !(g_descendants[walk] & evp->cb_map)) { 297 /* this UDE filter is useless */ 298 evp->ft_map &= ~(1L << g_udes[walk - UDE_OFFSET].parent); 299 /* No need to see this event in rest of EVENT_FOREACH */ 300 evs &= ~evid; 301 if ((walk = g_udes[walk - UDE_OFFSET].parent) < UDE_OFFSET) 302 break; 303 } else 304 break; 305 } 306 } EVENT_FOREACH_END(ev, evs); 307 } 308 309 /* Placing reference counter for each event table entry, instead of each 310 * event table, and decrement them for every callback unregistration may 311 * look inefficient. However, actually, it does NOT. If reference counter 312 * is for each event table, then we need to call FindReusableEvT() for 313 * every callback unregistration to find reusable event table. 314 * FindReusableEvT() is heavier than per-event reference counter update. 315 * And that way also wastes memory. */ 316 317 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, events) { 318 if (--evt->ent[i].ref == 0) 319 evt->map &= ~ev; 320 } EVENT_FOREACH_END(ev, events); 321 322 if (!events) 323 return; 324 325 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, events) { 326 if (--evt->ent[i].ref == 0) 327 evt->map &= ~ev; 328 } EVENT_FOREACH_END(ev, events); 329 } 330 /*----------------------------------------------------------------------------*/ 331 static inline struct ev_table * 332 FindReusableEvT(struct ev_pointer *evp, struct ev_base *evb, 333 event_t events, void *cb) 334 { 335 struct ev_table *cur_evt = evp->evt; 336 struct ev_table *walk; 337 338 assert((evp->cb_map & cur_evt->map) == evp->cb_map); 339 340 TAILQ_FOREACH(walk, &evb->evth, link) { 341 event_t overlap = evp->cb_map & walk->map; 342 assert((events & overlap) == 0); 343 event_t ev_total = events | overlap; 344 345 EVENT_FOREACH_START(ev, i, BEV_OFFSET, BEV_OFFSET + NUM_BEV, ev_total) { 346 if (ev & events) { 347 if (walk->ent[i].cb != cb) 348 goto __continue; 349 } else /* if (ev & overlap) */ { 350 if (walk->ent[i].cb != cur_evt->ent[i].cb) 351 goto __continue; 352 } 353 } EVENT_FOREACH_END(ev, ev_total); 354 355 if (!ev_total) 356 return walk; 357 358 EVENT_FOREACH_START(ev, i, UDE_OFFSET, MAX_EV, ev_total) { 359 if (ev & events) { 360 if (walk->ent[i].cb != cb) 361 goto __continue; 362 } else /* if (ev & overlap) */ { 363 if (walk->ent[i].cb != cur_evt->ent[i].cb) 364 goto __continue; 365 } 366 } EVENT_FOREACH_END(ev, ev_total); 367 368 if (!ev_total) 369 return walk; 370 __continue: 371 continue; 372 } 373 374 return NULL; 375 } 376 /*----------------------------------------------------------------------------*/ 377 inline int 378 ModCb(mtcp_manager_t mtcp, int op, struct ev_pointer *evp, struct ev_base *evb, 379 event_t events, void *cb) 380 { 381 struct ev_table *evt = evp->evt; 382 383 assert(evt); 384 385 if (op == OP_REG) { 386 /* NOTE: we do not register new callback if correponding 'map' is 387 * occupied */ 388 if (events & evp->cb_map) { 389 /* callback overwrite error */ 390 errno = EINVAL; 391 return -1; 392 } else if (events & evt->map) { 393 /* event registration conflict */ 394 struct ev_table *nevt; 395 if (!(nevt = FindReusableEvT(evp, evb, events, cb))) { 396 nevt = MPAllocateChunk(mtcp->evt_pool); 397 assert(nevt); 398 TAILQ_INSERT_HEAD(&evb->evth, nevt, link); 399 } 400 401 /* register callback */ 402 if (nevt != evt) 403 RegCbWCpy(evp, nevt, events, cb); 404 else 405 RegCbWoCpy(evp, events, cb); 406 407 } else { 408 /* reuse default event table */ 409 RegCbWoCpy(evp, events, cb); 410 } 411 } else /* if (op == OP_UNREG) */ { 412 if ((events & evp->cb_map) != events) { 413 /* unregister unexisting callback error */ 414 errno = EINVAL; 415 return -1; 416 } else { 417 /* unregister callback */ 418 UnregCb(evp, events); 419 } 420 } 421 422 return 0; 423 } 424 /*----------------------------------------------------------------------------*/ 425 static inline int 426 ModifyCallback(mctx_t mctx, int op, int sockid, event_t events, 427 int hook_point, void* callback) 428 { 429 socket_map_t socket; 430 struct ev_pointer *evp; 431 struct ev_base *evb; 432 433 assert(op == OP_REG || op == OP_UNREG); 434 435 if ((events & (g_bev_map | g_ude_map)) != events) { 436 errno = EINVAL; 437 return -1; 438 } 439 440 if ((op == OP_REG) && !callback) 441 return -1; 442 443 mtcp_manager_t mtcp = GetMTCPManager(mctx); 444 if (!mtcp) 445 return -1; 446 447 socket = &mtcp->msmap[sockid]; 448 449 if (socket->socktype == MOS_SOCK_MONITOR_STREAM_ACTIVE) { 450 if (hook_point == MOS_NULL) { 451 evp = &socket->monitor_stream->dontcare_evp; 452 evb = &socket->monitor_stream->monitor_listener->dontcare_evb; 453 } else if (hook_point == MOS_HK_RCV) { 454 evp = &socket->monitor_stream->pre_tcp_evp; 455 evb = &socket->monitor_stream->monitor_listener->pre_tcp_evb; 456 } else if (hook_point == MOS_HK_SND) { 457 evp = &socket->monitor_stream->post_tcp_evp; 458 evb = &socket->monitor_stream->monitor_listener->post_tcp_evb; 459 } else 460 return -1; 461 462 } else if (socket->socktype == MOS_SOCK_MONITOR_STREAM 463 || socket->socktype == MOS_SOCK_MONITOR_RAW) { 464 if (hook_point == MOS_NULL) 465 evb = &socket->monitor_listener->dontcare_evb; 466 else if (hook_point == MOS_HK_RCV) 467 evb = &socket->monitor_listener->pre_tcp_evb; 468 else if (hook_point == MOS_HK_SND) 469 evb = &socket->monitor_listener->post_tcp_evb; 470 else 471 return -1; 472 473 evp = &evb->dflt_evp; 474 } else { 475 errno = EINVAL; 476 return -1; 477 } 478 479 return ModCb(mtcp, op, evp, evb, events, callback); 480 } 481 /*----------------------------------------------------------------------------*/ 482 int 483 mtcp_register_callback(mctx_t mctx, int sockid, event_t events, 484 int hook_point, callback_t callback) 485 { 486 if (!callback) { 487 errno = EINVAL; 488 return -1; 489 } 490 491 return ModifyCallback(mctx, OP_REG, sockid, events, hook_point, callback); 492 } 493 /*----------------------------------------------------------------------------*/ 494 int 495 mtcp_unregister_callback(mctx_t mctx, int sockid, event_t events, 496 int hook_point) 497 { 498 return ModifyCallback(mctx, OP_UNREG, sockid, events, hook_point, NULL); 499 } 500 /*----------------------------------------------------------------------------*/ 501 event_t 502 mtcp_define_event(event_t event, filter_t filter) 503 { 504 int i, j; 505 int evid; 506 507 if (!IS_PO2(event)) 508 return 0; 509 510 if (!filter) 511 return 0; 512 513 for (i = 0; i < MAX_EV; i++) 514 if (event == 1L << i) { 515 evid = i; 516 break; 517 } 518 if (i == MAX_EV) 519 return 0; 520 521 for (i = 0; i < NUM_UDE; i++) { 522 const event_t ude = 1L << (i + UDE_OFFSET); 523 if (g_ude_map & ude) 524 continue; 525 526 for (j = 0; j < NUM_UDE; j++) 527 if (g_ude_id[evid][j] == -1) { 528 g_ude_id[evid][j] = i + UDE_OFFSET; 529 break; 530 } 531 if (j == NUM_UDE) 532 return 0; 533 534 /* Now we have valid UDE */ 535 536 /* update ancestor's descendants map */ 537 event_t ancestors = event | 538 ((evid >= UDE_OFFSET) ? g_udes[evid - UDE_OFFSET].ancestors : 0); 539 EVENT_FOREACH_START(ev, j, BEV_OFFSET, MAX_EV, ancestors) { 540 g_descendants[j] |= ude; 541 } EVENT_FOREACH_END(ev, ancestors); 542 543 /* update my ancestor map */ 544 if (event & g_ude_map) 545 g_udes[i].ancestors = event | g_udes[evid - UDE_OFFSET].ancestors; 546 else 547 g_udes[i].ancestors = event; 548 549 g_udes[i].parent = evid; 550 g_udes[i].ft = filter; 551 g_ude_map |= ude; 552 553 return ude; 554 } 555 556 return 0; 557 } 558 /*----------------------------------------------------------------------------*/ 559 int 560 mtcp_remove_ude(event_t event) 561 { 562 /* FIXME: this function is not implemented yet. 563 * What should we do if we remove UDE while running? */ 564 if (!IS_PO2(event)) 565 return -1; 566 567 if (!(g_ude_map & event)) 568 return -1; 569 570 return -1; 571 } 572 /*----------------------------------------------------------------------------*/ 573 #define PUSH(sp, ev_idx, ft_idx, data) \ 574 do { \ 575 sp->ev_idx = ev_idx; \ 576 sp->ft_idx = ft_idx; \ 577 sp->data.u64 = data.u64; \ 578 sp++; \ 579 } while (0) 580 #define POP(sp, ev_idx, ft_idx, data) \ 581 do { \ 582 sp--; \ 583 ev_idx = sp->ev_idx; \ 584 ft_idx = sp->ft_idx; \ 585 data.u64 = sp->data.u64; \ 586 } while (0) 587 /*----------------------------------------------------------------------------*/ 588 /** 589 * TODO: 590 * - Donghwi, please change POST_TCP names to POST_SND & 591 * PRE_TCP names to POST_RCV. 592 * 593 * - Please make sure that the order of event invocations is: 594 * MOS_ON_CONN_START --> .. MOS_ON_* .. --> MOS_ON_CONN_END 595 */ 596 inline void 597 HandleCallback(mtcp_manager_t mtcp, uint32_t hook, 598 socket_map_t socket, int side, struct pkt_ctx *pctx, event_t events) 599 { 600 struct sfbpf_program fcode; 601 int8_t ude_id; 602 uint64_t cb_map, ft_map; 603 604 int8_t ev_idx, ft_idx; 605 event_data_t data; 606 607 if (!socket) 608 return; 609 610 if (!events) 611 return; 612 assert(events); 613 614 /* if client side monitoring is disabled, then skip */ 615 if (side == MOS_SIDE_CLI && socket->monitor_stream->client_mon == 0) 616 return; 617 /* if server side monitoring is disabled, then skip */ 618 else if (side == MOS_SIDE_SVR && socket->monitor_stream->server_mon == 0) 619 return; 620 621 #define MSTRM(sock) (sock)->monitor_stream 622 #define MLSNR(sock) (sock)->monitor_listener 623 /* We use `?:` notation instead of `if/else` to make `evp` as const */ 624 struct ev_pointer * const evp = 625 (socket->socktype == MOS_SOCK_MONITOR_STREAM_ACTIVE) ? 626 ((hook == MOS_HK_RCV) ? &MSTRM(socket)->pre_tcp_evp : 627 (hook == MOS_HK_SND) ? &MSTRM(socket)->post_tcp_evp : 628 &MSTRM(socket)->dontcare_evp) 629 : (socket->socktype == MOS_SOCK_MONITOR_STREAM) 630 || (socket->socktype == MOS_SOCK_MONITOR_RAW) ? 631 ((hook == MOS_HK_RCV) ? &MLSNR(socket)->pre_tcp_evb.dflt_evp : 632 (hook == MOS_HK_SND) ? &MLSNR(socket)->post_tcp_evb.dflt_evp : 633 &MLSNR(socket)->dontcare_evb.dflt_evp) : 634 NULL; 635 636 if (!evp || !((cb_map = events & evp->cb_map) || (g_ude_map & evp->cb_map))) 637 return; 638 639 /* mtcp_bind_monitor_filter() 640 * - BPF filter is evaluated only for RAW socket and PASSIVE socket (= orphan filter) 641 * - stream syn filter is moved to and evaluated on socket creation */ 642 if (socket->socktype == MOS_SOCK_MONITOR_STREAM) { 643 fcode = MLSNR(socket)->stream_orphan_fcode; 644 /* if not match with filter, return */ 645 if (ISSET_BPFFILTER(fcode) && pctx && EVAL_BPFFILTER(fcode, 646 (uint8_t *)pctx->p.iph - sizeof(struct ethhdr), 647 pctx->p.ip_len + sizeof(struct ethhdr)) == 0) 648 return; 649 } 650 if (socket->socktype == MOS_SOCK_MONITOR_RAW) { 651 fcode = MLSNR(socket)->raw_pkt_fcode; 652 /* if not match with filter, return */ 653 if (ISSET_BPFFILTER(fcode) && pctx && EVAL_BPFFILTER(fcode, 654 (uint8_t *)pctx->p.iph - sizeof(struct ethhdr), 655 pctx->p.ip_len + sizeof(struct ethhdr)) == 0) 656 return; 657 } 658 659 ft_map = events & evp->ft_map; 660 661 event_t bev_map = cb_map | ft_map; 662 struct ev_table * const evt = evp->evt; 663 664 struct { 665 int8_t ev_idx; 666 int8_t ft_idx; 667 event_data_t data; 668 } stack[NUM_UDE + 1], *sp = stack; 669 670 mtcp->pctx = pctx; /* for mtcp_getlastpkt() */ 671 mctx_t const mctx = g_ctx[mtcp->ctx->cpu]; 672 673 EVENT_FOREACH_START(bev, bidx, BEV_OFFSET, BEV_OFFSET + NUM_BEV, bev_map) { 674 ev_idx = bidx; 675 ft_idx = 0; 676 data.u64 = 0; 677 const event_t descendants = g_descendants[ev_idx]; 678 if (descendants) { 679 cb_map |= descendants & evp->cb_map; 680 ft_map |= descendants & evp->ft_map; 681 } 682 683 while (1) { 684 const uint64_t ev = (1L << ev_idx); 685 686 if (cb_map & ev) { 687 /* call callback */ 688 evt->ent[ev_idx].cb(mctx, socket->id, side, ev, data); 689 690 if (!(cb_map &= ~ev)) 691 return; 692 } 693 694 while (1) { 695 event_data_t tmpdata; 696 if (ft_idx >= NUM_UDE 697 || (ude_id = g_ude_id[ev_idx][ft_idx]) < 0) { 698 /* done with this event */ 699 if (sp == stack /* stack is empty */) { 700 /* go to next built-in event */ 701 goto __continue; 702 } else { 703 POP(sp, ev_idx, ft_idx, data); 704 ft_idx++; 705 } 706 break; 707 } 708 709 assert(ude_id >= UDE_OFFSET && ude_id < MAX_EV); 710 711 if (((1L << ude_id) & (cb_map | ft_map)) && 712 (tmpdata.u64 = g_udes[ude_id - UDE_OFFSET].ft(mctx, socket->id, side, ev, data))) { 713 /* DFS jump */ 714 PUSH(sp, ev_idx, ft_idx, data); 715 ev_idx = ude_id; 716 ft_idx = 0; 717 data.u64 = tmpdata.u64; 718 break; 719 } 720 721 ft_idx++; 722 } 723 } 724 } 725 __continue: 726 EVENT_FOREACH_END(bev, bev_map); 727 } 728 /*----------------------------------------------------------------------------*/ 729 730 #endif 731