1 /*- 2 * Copyright (c) 1996-1999 Whistle Communications, Inc. 3 * All rights reserved. 4 * 5 * Subject to the following obligations and disclaimer of warranty, use and 6 * redistribution of this software, in source or object code forms, with or 7 * without modifications are expressly permitted by Whistle Communications; 8 * provided, however, that: 9 * 1. Any and all reproductions of the source or object code must include the 10 * copyright notice above and the following disclaimer of warranties; and 11 * 2. No rights are granted, in any manner or form, to use Whistle 12 * Communications, Inc. trademarks, including the mark "WHISTLE 13 * COMMUNICATIONS" on advertising, endorsements, or otherwise except as 14 * such appears in the above copyright notice or in the software. 15 * 16 * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND 17 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO 18 * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE, 19 * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. 21 * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY 22 * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS 23 * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE. 24 * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES 25 * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING 26 * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 27 * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR 28 * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY 32 * OF SUCH DAMAGE. 33 * 34 * Authors: Julian Elischer <[email protected]> 35 * Archie Cobbs <[email protected]> 36 * 37 * $FreeBSD$ 38 * $Whistle: ng_base.c,v 1.39 1999/01/28 23:54:53 julian Exp $ 39 */ 40 41 /* 42 * This file implements the base netgraph code. 43 */ 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/ctype.h> 48 #include <sys/hash.h> 49 #include <sys/kdb.h> 50 #include <sys/kernel.h> 51 #include <sys/kthread.h> 52 #include <sys/ktr.h> 53 #include <sys/limits.h> 54 #include <sys/lock.h> 55 #include <sys/malloc.h> 56 #include <sys/mbuf.h> 57 #include <sys/proc.h> 58 #include <sys/epoch.h> 59 #include <sys/queue.h> 60 #include <sys/refcount.h> 61 #include <sys/rwlock.h> 62 #include <sys/smp.h> 63 #include <sys/sysctl.h> 64 #include <sys/syslog.h> 65 #include <sys/unistd.h> 66 #include <machine/cpu.h> 67 #include <vm/uma.h> 68 69 #include <net/netisr.h> 70 #include <net/vnet.h> 71 72 #include <netgraph/ng_message.h> 73 #include <netgraph/netgraph.h> 74 #include <netgraph/ng_parse.h> 75 76 MODULE_VERSION(netgraph, NG_ABI_VERSION); 77 78 /* Mutex to protect topology events. */ 79 static struct rwlock ng_topo_lock; 80 #define TOPOLOGY_RLOCK() rw_rlock(&ng_topo_lock) 81 #define TOPOLOGY_RUNLOCK() rw_runlock(&ng_topo_lock) 82 #define TOPOLOGY_WLOCK() rw_wlock(&ng_topo_lock) 83 #define TOPOLOGY_WUNLOCK() rw_wunlock(&ng_topo_lock) 84 #define TOPOLOGY_NOTOWNED() rw_assert(&ng_topo_lock, RA_UNLOCKED) 85 86 #ifdef NETGRAPH_DEBUG 87 static struct mtx ng_nodelist_mtx; /* protects global node/hook lists */ 88 static struct mtx ngq_mtx; /* protects the queue item list */ 89 90 static SLIST_HEAD(, ng_node) ng_allnodes; 91 static LIST_HEAD(, ng_node) ng_freenodes; /* in debug, we never free() them */ 92 static SLIST_HEAD(, ng_hook) ng_allhooks; 93 static LIST_HEAD(, ng_hook) ng_freehooks; /* in debug, we never free() them */ 94 95 static void ng_dumpitems(void); 96 static void ng_dumpnodes(void); 97 static void ng_dumphooks(void); 98 99 #endif /* NETGRAPH_DEBUG */ 100 /* 101 * DEAD versions of the structures. 102 * In order to avoid races, it is sometimes necessary to point 103 * at SOMETHING even though theoretically, the current entity is 104 * INVALID. Use these to avoid these races. 105 */ 106 struct ng_type ng_deadtype = { 107 NG_ABI_VERSION, 108 "dead", 109 NULL, /* modevent */ 110 NULL, /* constructor */ 111 NULL, /* rcvmsg */ 112 NULL, /* shutdown */ 113 NULL, /* newhook */ 114 NULL, /* findhook */ 115 NULL, /* connect */ 116 NULL, /* rcvdata */ 117 NULL, /* disconnect */ 118 NULL, /* cmdlist */ 119 }; 120 121 struct ng_node ng_deadnode = { 122 "dead", 123 &ng_deadtype, 124 NGF_INVALID, 125 0, /* numhooks */ 126 NULL, /* private */ 127 0, /* ID */ 128 LIST_HEAD_INITIALIZER(ng_deadnode.nd_hooks), 129 {}, /* all_nodes list entry */ 130 {}, /* id hashtable list entry */ 131 { 0, 132 0, 133 {}, /* should never use! (should hang) */ 134 {}, /* workqueue entry */ 135 STAILQ_HEAD_INITIALIZER(ng_deadnode.nd_input_queue.queue), 136 }, 137 1, /* refs */ 138 NULL, /* vnet */ 139 #ifdef NETGRAPH_DEBUG 140 ND_MAGIC, 141 __FILE__, 142 __LINE__, 143 {NULL} 144 #endif /* NETGRAPH_DEBUG */ 145 }; 146 147 struct ng_hook ng_deadhook = { 148 "dead", 149 NULL, /* private */ 150 HK_INVALID | HK_DEAD, 151 0, /* undefined data link type */ 152 &ng_deadhook, /* Peer is self */ 153 &ng_deadnode, /* attached to deadnode */ 154 {}, /* hooks list */ 155 NULL, /* override rcvmsg() */ 156 NULL, /* override rcvdata() */ 157 1, /* refs always >= 1 */ 158 #ifdef NETGRAPH_DEBUG 159 HK_MAGIC, 160 __FILE__, 161 __LINE__, 162 {NULL} 163 #endif /* NETGRAPH_DEBUG */ 164 }; 165 166 /* 167 * END DEAD STRUCTURES 168 */ 169 /* List nodes with unallocated work */ 170 static STAILQ_HEAD(, ng_node) ng_worklist = STAILQ_HEAD_INITIALIZER(ng_worklist); 171 static struct mtx ng_worklist_mtx; /* MUST LOCK NODE FIRST */ 172 173 /* List of installed types */ 174 static LIST_HEAD(, ng_type) ng_typelist; 175 static struct rwlock ng_typelist_lock; 176 #define TYPELIST_RLOCK() rw_rlock(&ng_typelist_lock) 177 #define TYPELIST_RUNLOCK() rw_runlock(&ng_typelist_lock) 178 #define TYPELIST_WLOCK() rw_wlock(&ng_typelist_lock) 179 #define TYPELIST_WUNLOCK() rw_wunlock(&ng_typelist_lock) 180 181 /* Hash related definitions. */ 182 LIST_HEAD(nodehash, ng_node); 183 VNET_DEFINE_STATIC(struct nodehash *, ng_ID_hash); 184 VNET_DEFINE_STATIC(u_long, ng_ID_hmask); 185 VNET_DEFINE_STATIC(u_long, ng_nodes); 186 VNET_DEFINE_STATIC(struct nodehash *, ng_name_hash); 187 VNET_DEFINE_STATIC(u_long, ng_name_hmask); 188 VNET_DEFINE_STATIC(u_long, ng_named_nodes); 189 #define V_ng_ID_hash VNET(ng_ID_hash) 190 #define V_ng_ID_hmask VNET(ng_ID_hmask) 191 #define V_ng_nodes VNET(ng_nodes) 192 #define V_ng_name_hash VNET(ng_name_hash) 193 #define V_ng_name_hmask VNET(ng_name_hmask) 194 #define V_ng_named_nodes VNET(ng_named_nodes) 195 196 static struct rwlock ng_idhash_lock; 197 #define IDHASH_RLOCK() rw_rlock(&ng_idhash_lock) 198 #define IDHASH_RUNLOCK() rw_runlock(&ng_idhash_lock) 199 #define IDHASH_WLOCK() rw_wlock(&ng_idhash_lock) 200 #define IDHASH_WUNLOCK() rw_wunlock(&ng_idhash_lock) 201 202 /* Method to find a node.. used twice so do it here */ 203 #define NG_IDHASH_FN(ID) ((ID) % (V_ng_ID_hmask + 1)) 204 #define NG_IDHASH_FIND(ID, node) \ 205 do { \ 206 rw_assert(&ng_idhash_lock, RA_LOCKED); \ 207 LIST_FOREACH(node, &V_ng_ID_hash[NG_IDHASH_FN(ID)], \ 208 nd_idnodes) { \ 209 if (NG_NODE_IS_VALID(node) \ 210 && (NG_NODE_ID(node) == ID)) { \ 211 break; \ 212 } \ 213 } \ 214 } while (0) 215 216 static struct rwlock ng_namehash_lock; 217 #define NAMEHASH_RLOCK() rw_rlock(&ng_namehash_lock) 218 #define NAMEHASH_RUNLOCK() rw_runlock(&ng_namehash_lock) 219 #define NAMEHASH_WLOCK() rw_wlock(&ng_namehash_lock) 220 #define NAMEHASH_WUNLOCK() rw_wunlock(&ng_namehash_lock) 221 222 /* Internal functions */ 223 static int ng_add_hook(node_p node, const char *name, hook_p * hookp); 224 static int ng_generic_msg(node_p here, item_p item, hook_p lasthook); 225 static ng_ID_t ng_decodeidname(const char *name); 226 static int ngb_mod_event(module_t mod, int event, void *data); 227 #ifndef FSTACK 228 static void ng_worklist_add(node_p node); 229 static void ngthread(void *); 230 #endif 231 static int ng_apply_item(node_p node, item_p item, int rw); 232 #ifndef FSTACK 233 static void ng_flush_input_queue(node_p node); 234 #endif 235 static node_p ng_ID2noderef(ng_ID_t ID); 236 static int ng_con_nodes(item_p item, node_p node, const char *name, 237 node_p node2, const char *name2); 238 static int ng_con_part2(node_p node, item_p item, hook_p hook); 239 static int ng_con_part3(node_p node, item_p item, hook_p hook); 240 static int ng_mkpeer(node_p node, const char *name, const char *name2, 241 char *type); 242 static void ng_name_rehash(void); 243 static void ng_ID_rehash(void); 244 245 /* Imported, these used to be externally visible, some may go back. */ 246 void ng_destroy_hook(hook_p hook); 247 int ng_path2noderef(node_p here, const char *path, 248 node_p *dest, hook_p *lasthook); 249 int ng_make_node(const char *type, node_p *nodepp); 250 int ng_path_parse(char *addr, char **node, char **path, char **hook); 251 void ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3); 252 void ng_unname(node_p node); 253 254 /* Our own netgraph malloc type */ 255 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages"); 256 MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage"); 257 static MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", 258 "netgraph hook structures"); 259 static MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", 260 "netgraph node structures"); 261 static MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", 262 "netgraph item structures"); 263 264 /* Should not be visible outside this file */ 265 266 #define _NG_ALLOC_HOOK(hook) \ 267 hook = malloc(sizeof(*hook), M_NETGRAPH_HOOK, M_NOWAIT | M_ZERO) 268 #define _NG_ALLOC_NODE(node) \ 269 node = malloc(sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO) 270 271 #define NG_QUEUE_LOCK_INIT(n) \ 272 mtx_init(&(n)->q_mtx, "ng_node", NULL, MTX_DEF) 273 #define NG_QUEUE_LOCK(n) \ 274 mtx_lock(&(n)->q_mtx) 275 #define NG_QUEUE_UNLOCK(n) \ 276 mtx_unlock(&(n)->q_mtx) 277 #define NG_WORKLIST_LOCK_INIT() \ 278 mtx_init(&ng_worklist_mtx, "ng_worklist", NULL, MTX_DEF) 279 #define NG_WORKLIST_LOCK() \ 280 mtx_lock(&ng_worklist_mtx) 281 #define NG_WORKLIST_UNLOCK() \ 282 mtx_unlock(&ng_worklist_mtx) 283 #define NG_WORKLIST_SLEEP() \ 284 mtx_sleep(&ng_worklist, &ng_worklist_mtx, PI_NET, "sleep", 0) 285 #define NG_WORKLIST_WAKEUP() \ 286 wakeup_one(&ng_worklist) 287 288 #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/ 289 /* 290 * In debug mode: 291 * In an attempt to help track reference count screwups 292 * we do not free objects back to the malloc system, but keep them 293 * in a local cache where we can examine them and keep information safely 294 * after they have been freed. 295 * We use this scheme for nodes and hooks, and to some extent for items. 296 */ 297 static __inline hook_p 298 ng_alloc_hook(void) 299 { 300 hook_p hook; 301 SLIST_ENTRY(ng_hook) temp; 302 mtx_lock(&ng_nodelist_mtx); 303 hook = LIST_FIRST(&ng_freehooks); 304 if (hook) { 305 LIST_REMOVE(hook, hk_hooks); 306 bcopy(&hook->hk_all, &temp, sizeof(temp)); 307 bzero(hook, sizeof(struct ng_hook)); 308 bcopy(&temp, &hook->hk_all, sizeof(temp)); 309 mtx_unlock(&ng_nodelist_mtx); 310 hook->hk_magic = HK_MAGIC; 311 } else { 312 mtx_unlock(&ng_nodelist_mtx); 313 _NG_ALLOC_HOOK(hook); 314 if (hook) { 315 hook->hk_magic = HK_MAGIC; 316 mtx_lock(&ng_nodelist_mtx); 317 SLIST_INSERT_HEAD(&ng_allhooks, hook, hk_all); 318 mtx_unlock(&ng_nodelist_mtx); 319 } 320 } 321 return (hook); 322 } 323 324 static __inline node_p 325 ng_alloc_node(void) 326 { 327 node_p node; 328 SLIST_ENTRY(ng_node) temp; 329 mtx_lock(&ng_nodelist_mtx); 330 node = LIST_FIRST(&ng_freenodes); 331 if (node) { 332 LIST_REMOVE(node, nd_nodes); 333 bcopy(&node->nd_all, &temp, sizeof(temp)); 334 bzero(node, sizeof(struct ng_node)); 335 bcopy(&temp, &node->nd_all, sizeof(temp)); 336 mtx_unlock(&ng_nodelist_mtx); 337 node->nd_magic = ND_MAGIC; 338 } else { 339 mtx_unlock(&ng_nodelist_mtx); 340 _NG_ALLOC_NODE(node); 341 if (node) { 342 node->nd_magic = ND_MAGIC; 343 mtx_lock(&ng_nodelist_mtx); 344 SLIST_INSERT_HEAD(&ng_allnodes, node, nd_all); 345 mtx_unlock(&ng_nodelist_mtx); 346 } 347 } 348 return (node); 349 } 350 351 #define NG_ALLOC_HOOK(hook) do { (hook) = ng_alloc_hook(); } while (0) 352 #define NG_ALLOC_NODE(node) do { (node) = ng_alloc_node(); } while (0) 353 354 #define NG_FREE_HOOK(hook) \ 355 do { \ 356 mtx_lock(&ng_nodelist_mtx); \ 357 LIST_INSERT_HEAD(&ng_freehooks, hook, hk_hooks); \ 358 hook->hk_magic = 0; \ 359 mtx_unlock(&ng_nodelist_mtx); \ 360 } while (0) 361 362 #define NG_FREE_NODE(node) \ 363 do { \ 364 mtx_lock(&ng_nodelist_mtx); \ 365 LIST_INSERT_HEAD(&ng_freenodes, node, nd_nodes); \ 366 node->nd_magic = 0; \ 367 mtx_unlock(&ng_nodelist_mtx); \ 368 } while (0) 369 370 #else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 371 372 #define NG_ALLOC_HOOK(hook) _NG_ALLOC_HOOK(hook) 373 #define NG_ALLOC_NODE(node) _NG_ALLOC_NODE(node) 374 375 #define NG_FREE_HOOK(hook) do { free((hook), M_NETGRAPH_HOOK); } while (0) 376 #define NG_FREE_NODE(node) do { free((node), M_NETGRAPH_NODE); } while (0) 377 378 #endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 379 380 /* Set this to kdb_enter("X") to catch all errors as they occur */ 381 #ifndef TRAP_ERROR 382 #define TRAP_ERROR() 383 #endif 384 385 VNET_DEFINE_STATIC(ng_ID_t, nextID) = 1; 386 #define V_nextID VNET(nextID) 387 388 #ifdef INVARIANTS 389 #define CHECK_DATA_MBUF(m) do { \ 390 struct mbuf *n; \ 391 int total; \ 392 \ 393 M_ASSERTPKTHDR(m); \ 394 for (total = 0, n = (m); n != NULL; n = n->m_next) { \ 395 total += n->m_len; \ 396 if (n->m_nextpkt != NULL) \ 397 panic("%s: m_nextpkt", __func__); \ 398 } \ 399 \ 400 if ((m)->m_pkthdr.len != total) { \ 401 panic("%s: %d != %d", \ 402 __func__, (m)->m_pkthdr.len, total); \ 403 } \ 404 } while (0) 405 #else 406 #define CHECK_DATA_MBUF(m) 407 #endif 408 409 #define ERROUT(x) do { error = (x); goto done; } while (0) 410 411 /************************************************************************ 412 Parse type definitions for generic messages 413 ************************************************************************/ 414 415 /* Handy structure parse type defining macro */ 416 #define DEFINE_PARSE_STRUCT_TYPE(lo, up, args) \ 417 static const struct ng_parse_struct_field \ 418 ng_ ## lo ## _type_fields[] = NG_GENERIC_ ## up ## _INFO args; \ 419 static const struct ng_parse_type ng_generic_ ## lo ## _type = { \ 420 &ng_parse_struct_type, \ 421 &ng_ ## lo ## _type_fields \ 422 } 423 424 DEFINE_PARSE_STRUCT_TYPE(mkpeer, MKPEER, ()); 425 DEFINE_PARSE_STRUCT_TYPE(connect, CONNECT, ()); 426 DEFINE_PARSE_STRUCT_TYPE(name, NAME, ()); 427 DEFINE_PARSE_STRUCT_TYPE(rmhook, RMHOOK, ()); 428 DEFINE_PARSE_STRUCT_TYPE(nodeinfo, NODEINFO, ()); 429 DEFINE_PARSE_STRUCT_TYPE(typeinfo, TYPEINFO, ()); 430 DEFINE_PARSE_STRUCT_TYPE(linkinfo, LINKINFO, (&ng_generic_nodeinfo_type)); 431 432 /* Get length of an array when the length is stored as a 32 bit 433 value immediately preceding the array -- as with struct namelist 434 and struct typelist. */ 435 static int 436 ng_generic_list_getLength(const struct ng_parse_type *type, 437 const u_char *start, const u_char *buf) 438 { 439 return *((const u_int32_t *)(buf - 4)); 440 } 441 442 /* Get length of the array of struct linkinfo inside a struct hooklist */ 443 static int 444 ng_generic_linkinfo_getLength(const struct ng_parse_type *type, 445 const u_char *start, const u_char *buf) 446 { 447 const struct hooklist *hl = (const struct hooklist *)start; 448 449 return hl->nodeinfo.hooks; 450 } 451 452 /* Array type for a variable length array of struct namelist */ 453 static const struct ng_parse_array_info ng_nodeinfoarray_type_info = { 454 &ng_generic_nodeinfo_type, 455 &ng_generic_list_getLength 456 }; 457 static const struct ng_parse_type ng_generic_nodeinfoarray_type = { 458 &ng_parse_array_type, 459 &ng_nodeinfoarray_type_info 460 }; 461 462 /* Array type for a variable length array of struct typelist */ 463 static const struct ng_parse_array_info ng_typeinfoarray_type_info = { 464 &ng_generic_typeinfo_type, 465 &ng_generic_list_getLength 466 }; 467 static const struct ng_parse_type ng_generic_typeinfoarray_type = { 468 &ng_parse_array_type, 469 &ng_typeinfoarray_type_info 470 }; 471 472 /* Array type for array of struct linkinfo in struct hooklist */ 473 static const struct ng_parse_array_info ng_generic_linkinfo_array_type_info = { 474 &ng_generic_linkinfo_type, 475 &ng_generic_linkinfo_getLength 476 }; 477 static const struct ng_parse_type ng_generic_linkinfo_array_type = { 478 &ng_parse_array_type, 479 &ng_generic_linkinfo_array_type_info 480 }; 481 482 DEFINE_PARSE_STRUCT_TYPE(typelist, TYPELIST, (&ng_generic_typeinfoarray_type)); 483 DEFINE_PARSE_STRUCT_TYPE(hooklist, HOOKLIST, 484 (&ng_generic_nodeinfo_type, &ng_generic_linkinfo_array_type)); 485 DEFINE_PARSE_STRUCT_TYPE(listnodes, LISTNODES, 486 (&ng_generic_nodeinfoarray_type)); 487 488 /* List of commands and how to convert arguments to/from ASCII */ 489 static const struct ng_cmdlist ng_generic_cmds[] = { 490 { 491 NGM_GENERIC_COOKIE, 492 NGM_SHUTDOWN, 493 "shutdown", 494 NULL, 495 NULL 496 }, 497 { 498 NGM_GENERIC_COOKIE, 499 NGM_MKPEER, 500 "mkpeer", 501 &ng_generic_mkpeer_type, 502 NULL 503 }, 504 { 505 NGM_GENERIC_COOKIE, 506 NGM_CONNECT, 507 "connect", 508 &ng_generic_connect_type, 509 NULL 510 }, 511 { 512 NGM_GENERIC_COOKIE, 513 NGM_NAME, 514 "name", 515 &ng_generic_name_type, 516 NULL 517 }, 518 { 519 NGM_GENERIC_COOKIE, 520 NGM_RMHOOK, 521 "rmhook", 522 &ng_generic_rmhook_type, 523 NULL 524 }, 525 { 526 NGM_GENERIC_COOKIE, 527 NGM_NODEINFO, 528 "nodeinfo", 529 NULL, 530 &ng_generic_nodeinfo_type 531 }, 532 { 533 NGM_GENERIC_COOKIE, 534 NGM_LISTHOOKS, 535 "listhooks", 536 NULL, 537 &ng_generic_hooklist_type 538 }, 539 { 540 NGM_GENERIC_COOKIE, 541 NGM_LISTNAMES, 542 "listnames", 543 NULL, 544 &ng_generic_listnodes_type /* same as NGM_LISTNODES */ 545 }, 546 { 547 NGM_GENERIC_COOKIE, 548 NGM_LISTNODES, 549 "listnodes", 550 NULL, 551 &ng_generic_listnodes_type 552 }, 553 { 554 NGM_GENERIC_COOKIE, 555 NGM_LISTTYPES, 556 "listtypes", 557 NULL, 558 &ng_generic_typelist_type 559 }, 560 { 561 NGM_GENERIC_COOKIE, 562 NGM_TEXT_CONFIG, 563 "textconfig", 564 NULL, 565 &ng_parse_string_type 566 }, 567 { 568 NGM_GENERIC_COOKIE, 569 NGM_TEXT_STATUS, 570 "textstatus", 571 NULL, 572 &ng_parse_string_type 573 }, 574 { 575 NGM_GENERIC_COOKIE, 576 NGM_ASCII2BINARY, 577 "ascii2binary", 578 &ng_parse_ng_mesg_type, 579 &ng_parse_ng_mesg_type 580 }, 581 { 582 NGM_GENERIC_COOKIE, 583 NGM_BINARY2ASCII, 584 "binary2ascii", 585 &ng_parse_ng_mesg_type, 586 &ng_parse_ng_mesg_type 587 }, 588 { 0 } 589 }; 590 591 /************************************************************************ 592 Node routines 593 ************************************************************************/ 594 595 /* 596 * Instantiate a node of the requested type 597 */ 598 int 599 ng_make_node(const char *typename, node_p *nodepp) 600 { 601 struct ng_type *type; 602 int error; 603 604 /* Check that the type makes sense */ 605 if (typename == NULL) { 606 TRAP_ERROR(); 607 return (EINVAL); 608 } 609 610 /* Locate the node type. If we fail we return. Do not try to load 611 * module. 612 */ 613 if ((type = ng_findtype(typename)) == NULL) 614 return (ENXIO); 615 616 /* 617 * If we have a constructor, then make the node and 618 * call the constructor to do type specific initialisation. 619 */ 620 if (type->constructor != NULL) { 621 if ((error = ng_make_node_common(type, nodepp)) == 0) { 622 if ((error = ((*type->constructor)(*nodepp))) != 0) { 623 NG_NODE_UNREF(*nodepp); 624 } 625 } 626 } else { 627 /* 628 * Node has no constructor. We cannot ask for one 629 * to be made. It must be brought into existence by 630 * some external agency. The external agency should 631 * call ng_make_node_common() directly to get the 632 * netgraph part initialised. 633 */ 634 TRAP_ERROR(); 635 error = EINVAL; 636 } 637 return (error); 638 } 639 640 /* 641 * Generic node creation. Called by node initialisation for externally 642 * instantiated nodes (e.g. hardware, sockets, etc ). 643 * The returned node has a reference count of 1. 644 */ 645 int 646 ng_make_node_common(struct ng_type *type, node_p *nodepp) 647 { 648 node_p node; 649 650 /* Require the node type to have been already installed */ 651 if (ng_findtype(type->name) == NULL) { 652 TRAP_ERROR(); 653 return (EINVAL); 654 } 655 656 /* Make a node and try attach it to the type */ 657 NG_ALLOC_NODE(node); 658 if (node == NULL) { 659 TRAP_ERROR(); 660 return (ENOMEM); 661 } 662 node->nd_type = type; 663 #ifdef VIMAGE 664 node->nd_vnet = curvnet; 665 #endif 666 NG_NODE_REF(node); /* note reference */ 667 type->refs++; 668 669 NG_QUEUE_LOCK_INIT(&node->nd_input_queue); 670 STAILQ_INIT(&node->nd_input_queue.queue); 671 node->nd_input_queue.q_flags = 0; 672 673 /* Initialize hook list for new node */ 674 LIST_INIT(&node->nd_hooks); 675 676 /* Get an ID and put us in the hash chain. */ 677 IDHASH_WLOCK(); 678 for (;;) { /* wrap protection, even if silly */ 679 node_p node2 = NULL; 680 node->nd_ID = V_nextID++; /* 137/sec for 1 year before wrap */ 681 682 /* Is there a problem with the new number? */ 683 NG_IDHASH_FIND(node->nd_ID, node2); /* already taken? */ 684 if ((node->nd_ID != 0) && (node2 == NULL)) { 685 break; 686 } 687 } 688 V_ng_nodes++; 689 if (V_ng_nodes * 2 > V_ng_ID_hmask) 690 ng_ID_rehash(); 691 LIST_INSERT_HEAD(&V_ng_ID_hash[NG_IDHASH_FN(node->nd_ID)], node, 692 nd_idnodes); 693 IDHASH_WUNLOCK(); 694 695 /* Done */ 696 *nodepp = node; 697 return (0); 698 } 699 700 /* 701 * Forceably start the shutdown process on a node. Either call 702 * its shutdown method, or do the default shutdown if there is 703 * no type-specific method. 704 * 705 * We can only be called from a shutdown message, so we know we have 706 * a writer lock, and therefore exclusive access. It also means 707 * that we should not be on the work queue, but we check anyhow. 708 * 709 * Persistent node types must have a type-specific method which 710 * allocates a new node in which case, this one is irretrievably going away, 711 * or cleans up anything it needs, and just makes the node valid again, 712 * in which case we allow the node to survive. 713 * 714 * XXX We need to think of how to tell a persistent node that we 715 * REALLY need to go away because the hardware has gone or we 716 * are rebooting.... etc. 717 */ 718 void 719 ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3) 720 { 721 hook_p hook; 722 723 /* Check if it's already shutting down */ 724 if ((node->nd_flags & NGF_CLOSING) != 0) 725 return; 726 727 if (node == &ng_deadnode) { 728 printf ("shutdown called on deadnode\n"); 729 return; 730 } 731 732 /* Add an extra reference so it doesn't go away during this */ 733 NG_NODE_REF(node); 734 735 /* 736 * Mark it invalid so any newcomers know not to try use it 737 * Also add our own mark so we can't recurse 738 * note that NGF_INVALID does not do this as it's also set during 739 * creation 740 */ 741 node->nd_flags |= NGF_INVALID|NGF_CLOSING; 742 743 /* If node has its pre-shutdown method, then call it first*/ 744 if (node->nd_type && node->nd_type->close) 745 (*node->nd_type->close)(node); 746 747 /* Notify all remaining connected nodes to disconnect */ 748 while ((hook = LIST_FIRST(&node->nd_hooks)) != NULL) 749 ng_destroy_hook(hook); 750 751 #ifndef FSTACK 752 /* 753 * Drain the input queue forceably. 754 * it has no hooks so what's it going to do, bleed on someone? 755 * Theoretically we came here from a queue entry that was added 756 * Just before the queue was closed, so it should be empty anyway. 757 * Also removes us from worklist if needed. 758 */ 759 ng_flush_input_queue(node); 760 #endif 761 762 /* Ask the type if it has anything to do in this case */ 763 if (node->nd_type && node->nd_type->shutdown) { 764 (*node->nd_type->shutdown)(node); 765 if (NG_NODE_IS_VALID(node)) { 766 /* 767 * Well, blow me down if the node code hasn't declared 768 * that it doesn't want to die. 769 * Presumably it is a persistent node. 770 * If we REALLY want it to go away, 771 * e.g. hardware going away, 772 * Our caller should set NGF_REALLY_DIE in nd_flags. 773 */ 774 node->nd_flags &= ~(NGF_INVALID|NGF_CLOSING); 775 NG_NODE_UNREF(node); /* Assume they still have theirs */ 776 return; 777 } 778 } else { /* do the default thing */ 779 NG_NODE_UNREF(node); 780 } 781 782 ng_unname(node); /* basically a NOP these days */ 783 784 /* 785 * Remove extra reference, possibly the last 786 * Possible other holders of references may include 787 * timeout callouts, but theoretically the node's supposed to 788 * have cancelled them. Possibly hardware dependencies may 789 * force a driver to 'linger' with a reference. 790 */ 791 NG_NODE_UNREF(node); 792 } 793 794 /* 795 * Remove a reference to the node, possibly the last. 796 * deadnode always acts as it it were the last. 797 */ 798 void 799 ng_unref_node(node_p node) 800 { 801 802 if (node == &ng_deadnode) 803 return; 804 805 CURVNET_SET(node->nd_vnet); 806 807 if (refcount_release(&node->nd_refs)) { /* we were the last */ 808 809 node->nd_type->refs--; /* XXX maybe should get types lock? */ 810 NAMEHASH_WLOCK(); 811 if (NG_NODE_HAS_NAME(node)) { 812 V_ng_named_nodes--; 813 LIST_REMOVE(node, nd_nodes); 814 } 815 NAMEHASH_WUNLOCK(); 816 817 IDHASH_WLOCK(); 818 V_ng_nodes--; 819 LIST_REMOVE(node, nd_idnodes); 820 IDHASH_WUNLOCK(); 821 822 mtx_destroy(&node->nd_input_queue.q_mtx); 823 NG_FREE_NODE(node); 824 } 825 CURVNET_RESTORE(); 826 } 827 828 /************************************************************************ 829 Node ID handling 830 ************************************************************************/ 831 static node_p 832 ng_ID2noderef(ng_ID_t ID) 833 { 834 node_p node; 835 836 IDHASH_RLOCK(); 837 NG_IDHASH_FIND(ID, node); 838 if (node) 839 NG_NODE_REF(node); 840 IDHASH_RUNLOCK(); 841 return(node); 842 } 843 844 ng_ID_t 845 ng_node2ID(node_p node) 846 { 847 return (node ? NG_NODE_ID(node) : 0); 848 } 849 850 /************************************************************************ 851 Node name handling 852 ************************************************************************/ 853 854 /* 855 * Assign a node a name. 856 */ 857 int 858 ng_name_node(node_p node, const char *name) 859 { 860 uint32_t hash; 861 node_p node2; 862 int i; 863 864 /* Check the name is valid */ 865 for (i = 0; i < NG_NODESIZ; i++) { 866 if (name[i] == '\0' || name[i] == '.' || name[i] == ':') 867 break; 868 } 869 if (i == 0 || name[i] != '\0') { 870 TRAP_ERROR(); 871 return (EINVAL); 872 } 873 if (ng_decodeidname(name) != 0) { /* valid IDs not allowed here */ 874 TRAP_ERROR(); 875 return (EINVAL); 876 } 877 878 NAMEHASH_WLOCK(); 879 if (V_ng_named_nodes * 2 > V_ng_name_hmask) 880 ng_name_rehash(); 881 882 hash = hash32_str(name, HASHINIT) & V_ng_name_hmask; 883 /* Check the name isn't already being used. */ 884 LIST_FOREACH(node2, &V_ng_name_hash[hash], nd_nodes) 885 if (NG_NODE_IS_VALID(node2) && 886 (strcmp(NG_NODE_NAME(node2), name) == 0)) { 887 NAMEHASH_WUNLOCK(); 888 return (EADDRINUSE); 889 } 890 891 if (NG_NODE_HAS_NAME(node)) 892 LIST_REMOVE(node, nd_nodes); 893 else 894 V_ng_named_nodes++; 895 /* Copy it. */ 896 strlcpy(NG_NODE_NAME(node), name, NG_NODESIZ); 897 /* Update name hash. */ 898 LIST_INSERT_HEAD(&V_ng_name_hash[hash], node, nd_nodes); 899 NAMEHASH_WUNLOCK(); 900 901 return (0); 902 } 903 904 /* 905 * Find a node by absolute name. The name should NOT end with ':' 906 * The name "." means "this node" and "[xxx]" means "the node 907 * with ID (ie, at address) xxx". 908 * 909 * Returns the node if found, else NULL. 910 * Eventually should add something faster than a sequential search. 911 * Note it acquires a reference on the node so you can be sure it's still 912 * there. 913 */ 914 node_p 915 ng_name2noderef(node_p here, const char *name) 916 { 917 node_p node; 918 ng_ID_t temp; 919 int hash; 920 921 /* "." means "this node" */ 922 if (strcmp(name, ".") == 0) { 923 NG_NODE_REF(here); 924 return(here); 925 } 926 927 /* Check for name-by-ID */ 928 if ((temp = ng_decodeidname(name)) != 0) { 929 return (ng_ID2noderef(temp)); 930 } 931 932 /* Find node by name. */ 933 hash = hash32_str(name, HASHINIT) & V_ng_name_hmask; 934 NAMEHASH_RLOCK(); 935 LIST_FOREACH(node, &V_ng_name_hash[hash], nd_nodes) 936 if (NG_NODE_IS_VALID(node) && 937 (strcmp(NG_NODE_NAME(node), name) == 0)) { 938 NG_NODE_REF(node); 939 break; 940 } 941 NAMEHASH_RUNLOCK(); 942 943 return (node); 944 } 945 946 /* 947 * Decode an ID name, eg. "[f03034de]". Returns 0 if the 948 * string is not valid, otherwise returns the value. 949 */ 950 static ng_ID_t 951 ng_decodeidname(const char *name) 952 { 953 const int len = strlen(name); 954 char *eptr; 955 u_long val; 956 957 /* Check for proper length, brackets, no leading junk */ 958 if ((len < 3) || (name[0] != '[') || (name[len - 1] != ']') || 959 (!isxdigit(name[1]))) 960 return ((ng_ID_t)0); 961 962 /* Decode number */ 963 val = strtoul(name + 1, &eptr, 16); 964 if ((eptr - name != len - 1) || (val == ULONG_MAX) || (val == 0)) 965 return ((ng_ID_t)0); 966 967 return ((ng_ID_t)val); 968 } 969 970 /* 971 * Remove a name from a node. This should only be called 972 * when shutting down and removing the node. 973 */ 974 void 975 ng_unname(node_p node) 976 { 977 } 978 979 /* 980 * Allocate a bigger name hash. 981 */ 982 static void 983 ng_name_rehash() 984 { 985 struct nodehash *new; 986 uint32_t hash; 987 u_long hmask; 988 node_p node, node2; 989 int i; 990 991 new = hashinit_flags((V_ng_name_hmask + 1) * 2, M_NETGRAPH_NODE, &hmask, 992 HASH_NOWAIT); 993 if (new == NULL) 994 return; 995 996 for (i = 0; i <= V_ng_name_hmask; i++) 997 LIST_FOREACH_SAFE(node, &V_ng_name_hash[i], nd_nodes, node2) { 998 #ifdef INVARIANTS 999 LIST_REMOVE(node, nd_nodes); 1000 #endif 1001 hash = hash32_str(NG_NODE_NAME(node), HASHINIT) & hmask; 1002 LIST_INSERT_HEAD(&new[hash], node, nd_nodes); 1003 } 1004 1005 hashdestroy(V_ng_name_hash, M_NETGRAPH_NODE, V_ng_name_hmask); 1006 V_ng_name_hash = new; 1007 V_ng_name_hmask = hmask; 1008 } 1009 1010 /* 1011 * Allocate a bigger ID hash. 1012 */ 1013 static void 1014 ng_ID_rehash() 1015 { 1016 struct nodehash *new; 1017 uint32_t hash; 1018 u_long hmask; 1019 node_p node, node2; 1020 int i; 1021 1022 new = hashinit_flags((V_ng_ID_hmask + 1) * 2, M_NETGRAPH_NODE, &hmask, 1023 HASH_NOWAIT); 1024 if (new == NULL) 1025 return; 1026 1027 for (i = 0; i <= V_ng_ID_hmask; i++) 1028 LIST_FOREACH_SAFE(node, &V_ng_ID_hash[i], nd_idnodes, node2) { 1029 #ifdef INVARIANTS 1030 LIST_REMOVE(node, nd_idnodes); 1031 #endif 1032 hash = (node->nd_ID % (hmask + 1)); 1033 LIST_INSERT_HEAD(&new[hash], node, nd_idnodes); 1034 } 1035 1036 hashdestroy(V_ng_ID_hash, M_NETGRAPH_NODE, V_ng_name_hmask); 1037 V_ng_ID_hash = new; 1038 V_ng_ID_hmask = hmask; 1039 } 1040 1041 /************************************************************************ 1042 Hook routines 1043 Names are not optional. Hooks are always connected, except for a 1044 brief moment within these routines. On invalidation or during creation 1045 they are connected to the 'dead' hook. 1046 ************************************************************************/ 1047 1048 /* 1049 * Remove a hook reference 1050 */ 1051 void 1052 ng_unref_hook(hook_p hook) 1053 { 1054 1055 if (hook == &ng_deadhook) 1056 return; 1057 1058 if (refcount_release(&hook->hk_refs)) { /* we were the last */ 1059 if (_NG_HOOK_NODE(hook)) /* it'll probably be ng_deadnode */ 1060 _NG_NODE_UNREF((_NG_HOOK_NODE(hook))); 1061 NG_FREE_HOOK(hook); 1062 } 1063 } 1064 1065 /* 1066 * Add an unconnected hook to a node. Only used internally. 1067 * Assumes node is locked. (XXX not yet true ) 1068 */ 1069 static int 1070 ng_add_hook(node_p node, const char *name, hook_p *hookp) 1071 { 1072 hook_p hook; 1073 int error = 0; 1074 1075 /* Check that the given name is good */ 1076 if (name == NULL) { 1077 TRAP_ERROR(); 1078 return (EINVAL); 1079 } 1080 if (ng_findhook(node, name) != NULL) { 1081 TRAP_ERROR(); 1082 return (EEXIST); 1083 } 1084 1085 /* Allocate the hook and link it up */ 1086 NG_ALLOC_HOOK(hook); 1087 if (hook == NULL) { 1088 TRAP_ERROR(); 1089 return (ENOMEM); 1090 } 1091 hook->hk_refs = 1; /* add a reference for us to return */ 1092 hook->hk_flags = HK_INVALID; 1093 hook->hk_peer = &ng_deadhook; /* start off this way */ 1094 hook->hk_node = node; 1095 NG_NODE_REF(node); /* each hook counts as a reference */ 1096 1097 /* Set hook name */ 1098 strlcpy(NG_HOOK_NAME(hook), name, NG_HOOKSIZ); 1099 1100 /* 1101 * Check if the node type code has something to say about it 1102 * If it fails, the unref of the hook will also unref the node. 1103 */ 1104 if (node->nd_type->newhook != NULL) { 1105 if ((error = (*node->nd_type->newhook)(node, hook, name))) { 1106 NG_HOOK_UNREF(hook); /* this frees the hook */ 1107 return (error); 1108 } 1109 } 1110 /* 1111 * The 'type' agrees so far, so go ahead and link it in. 1112 * We'll ask again later when we actually connect the hooks. 1113 */ 1114 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1115 node->nd_numhooks++; 1116 NG_HOOK_REF(hook); /* one for the node */ 1117 1118 if (hookp) 1119 *hookp = hook; 1120 return (0); 1121 } 1122 1123 /* 1124 * Find a hook 1125 * 1126 * Node types may supply their own optimized routines for finding 1127 * hooks. If none is supplied, we just do a linear search. 1128 * XXX Possibly we should add a reference to the hook? 1129 */ 1130 hook_p 1131 ng_findhook(node_p node, const char *name) 1132 { 1133 hook_p hook; 1134 1135 if (node->nd_type->findhook != NULL) 1136 return (*node->nd_type->findhook)(node, name); 1137 LIST_FOREACH(hook, &node->nd_hooks, hk_hooks) { 1138 if (NG_HOOK_IS_VALID(hook) && 1139 (strcmp(NG_HOOK_NAME(hook), name) == 0)) 1140 return (hook); 1141 } 1142 return (NULL); 1143 } 1144 1145 /* 1146 * Destroy a hook 1147 * 1148 * As hooks are always attached, this really destroys two hooks. 1149 * The one given, and the one attached to it. Disconnect the hooks 1150 * from each other first. We reconnect the peer hook to the 'dead' 1151 * hook so that it can still exist after we depart. We then 1152 * send the peer its own destroy message. This ensures that we only 1153 * interact with the peer's structures when it is locked processing that 1154 * message. We hold a reference to the peer hook so we are guaranteed that 1155 * the peer hook and node are still going to exist until 1156 * we are finished there as the hook holds a ref on the node. 1157 * We run this same code again on the peer hook, but that time it is already 1158 * attached to the 'dead' hook. 1159 * 1160 * This routine is called at all stages of hook creation 1161 * on error detection and must be able to handle any such stage. 1162 */ 1163 void 1164 ng_destroy_hook(hook_p hook) 1165 { 1166 hook_p peer; 1167 node_p node; 1168 1169 if (hook == &ng_deadhook) { /* better safe than sorry */ 1170 printf("ng_destroy_hook called on deadhook\n"); 1171 return; 1172 } 1173 1174 /* 1175 * Protect divorce process with mutex, to avoid races on 1176 * simultaneous disconnect. 1177 */ 1178 TOPOLOGY_WLOCK(); 1179 1180 hook->hk_flags |= HK_INVALID; 1181 1182 peer = NG_HOOK_PEER(hook); 1183 node = NG_HOOK_NODE(hook); 1184 1185 if (peer && (peer != &ng_deadhook)) { 1186 /* 1187 * Set the peer to point to ng_deadhook 1188 * from this moment on we are effectively independent it. 1189 * send it an rmhook message of its own. 1190 */ 1191 peer->hk_peer = &ng_deadhook; /* They no longer know us */ 1192 hook->hk_peer = &ng_deadhook; /* Nor us, them */ 1193 if (NG_HOOK_NODE(peer) == &ng_deadnode) { 1194 /* 1195 * If it's already divorced from a node, 1196 * just free it. 1197 */ 1198 TOPOLOGY_WUNLOCK(); 1199 } else { 1200 TOPOLOGY_WUNLOCK(); 1201 ng_rmhook_self(peer); /* Send it a surprise */ 1202 } 1203 NG_HOOK_UNREF(peer); /* account for peer link */ 1204 NG_HOOK_UNREF(hook); /* account for peer link */ 1205 } else 1206 TOPOLOGY_WUNLOCK(); 1207 1208 TOPOLOGY_NOTOWNED(); 1209 1210 /* 1211 * Remove the hook from the node's list to avoid possible recursion 1212 * in case the disconnection results in node shutdown. 1213 */ 1214 if (node == &ng_deadnode) { /* happens if called from ng_con_nodes() */ 1215 return; 1216 } 1217 LIST_REMOVE(hook, hk_hooks); 1218 node->nd_numhooks--; 1219 if (node->nd_type->disconnect) { 1220 /* 1221 * The type handler may elect to destroy the node so don't 1222 * trust its existence after this point. (except 1223 * that we still hold a reference on it. (which we 1224 * inherrited from the hook we are destroying) 1225 */ 1226 (*node->nd_type->disconnect) (hook); 1227 } 1228 1229 /* 1230 * Note that because we will point to ng_deadnode, the original node 1231 * is not decremented automatically so we do that manually. 1232 */ 1233 _NG_HOOK_NODE(hook) = &ng_deadnode; 1234 NG_NODE_UNREF(node); /* We no longer point to it so adjust count */ 1235 NG_HOOK_UNREF(hook); /* Account for linkage (in list) to node */ 1236 } 1237 1238 /* 1239 * Take two hooks on a node and merge the connection so that the given node 1240 * is effectively bypassed. 1241 */ 1242 int 1243 ng_bypass(hook_p hook1, hook_p hook2) 1244 { 1245 if (hook1->hk_node != hook2->hk_node) { 1246 TRAP_ERROR(); 1247 return (EINVAL); 1248 } 1249 TOPOLOGY_WLOCK(); 1250 if (NG_HOOK_NOT_VALID(hook1) || NG_HOOK_NOT_VALID(hook2)) { 1251 TOPOLOGY_WUNLOCK(); 1252 return (EINVAL); 1253 } 1254 hook1->hk_peer->hk_peer = hook2->hk_peer; 1255 hook2->hk_peer->hk_peer = hook1->hk_peer; 1256 1257 hook1->hk_peer = &ng_deadhook; 1258 hook2->hk_peer = &ng_deadhook; 1259 TOPOLOGY_WUNLOCK(); 1260 1261 NG_HOOK_UNREF(hook1); 1262 NG_HOOK_UNREF(hook2); 1263 1264 /* XXX If we ever cache methods on hooks update them as well */ 1265 ng_destroy_hook(hook1); 1266 ng_destroy_hook(hook2); 1267 return (0); 1268 } 1269 1270 /* 1271 * Install a new netgraph type 1272 */ 1273 int 1274 ng_newtype(struct ng_type *tp) 1275 { 1276 const size_t namelen = strlen(tp->name); 1277 1278 /* Check version and type name fields */ 1279 if ((tp->version != NG_ABI_VERSION) || (namelen == 0) || 1280 (namelen >= NG_TYPESIZ)) { 1281 TRAP_ERROR(); 1282 if (tp->version != NG_ABI_VERSION) { 1283 printf("Netgraph: Node type rejected. ABI mismatch. " 1284 "Suggest recompile\n"); 1285 } 1286 return (EINVAL); 1287 } 1288 1289 /* Check for name collision */ 1290 if (ng_findtype(tp->name) != NULL) { 1291 TRAP_ERROR(); 1292 return (EEXIST); 1293 } 1294 1295 /* Link in new type */ 1296 TYPELIST_WLOCK(); 1297 LIST_INSERT_HEAD(&ng_typelist, tp, types); 1298 tp->refs = 1; /* first ref is linked list */ 1299 TYPELIST_WUNLOCK(); 1300 return (0); 1301 } 1302 1303 /* 1304 * unlink a netgraph type 1305 * If no examples exist 1306 */ 1307 int 1308 ng_rmtype(struct ng_type *tp) 1309 { 1310 /* Check for name collision */ 1311 if (tp->refs != 1) { 1312 TRAP_ERROR(); 1313 return (EBUSY); 1314 } 1315 1316 /* Unlink type */ 1317 TYPELIST_WLOCK(); 1318 LIST_REMOVE(tp, types); 1319 TYPELIST_WUNLOCK(); 1320 return (0); 1321 } 1322 1323 /* 1324 * Look for a type of the name given 1325 */ 1326 struct ng_type * 1327 ng_findtype(const char *typename) 1328 { 1329 struct ng_type *type; 1330 1331 TYPELIST_RLOCK(); 1332 LIST_FOREACH(type, &ng_typelist, types) { 1333 if (strcmp(type->name, typename) == 0) 1334 break; 1335 } 1336 TYPELIST_RUNLOCK(); 1337 return (type); 1338 } 1339 1340 /************************************************************************ 1341 Composite routines 1342 ************************************************************************/ 1343 /* 1344 * Connect two nodes using the specified hooks, using queued functions. 1345 */ 1346 static int 1347 ng_con_part3(node_p node, item_p item, hook_p hook) 1348 { 1349 int error = 0; 1350 1351 /* 1352 * When we run, we know that the node 'node' is locked for us. 1353 * Our caller has a reference on the hook. 1354 * Our caller has a reference on the node. 1355 * (In this case our caller is ng_apply_item() ). 1356 * The peer hook has a reference on the hook. 1357 * We are all set up except for the final call to the node, and 1358 * the clearing of the INVALID flag. 1359 */ 1360 if (NG_HOOK_NODE(hook) == &ng_deadnode) { 1361 /* 1362 * The node must have been freed again since we last visited 1363 * here. ng_destry_hook() has this effect but nothing else does. 1364 * We should just release our references and 1365 * free anything we can think of. 1366 * Since we know it's been destroyed, and it's our caller 1367 * that holds the references, just return. 1368 */ 1369 ERROUT(ENOENT); 1370 } 1371 if (hook->hk_node->nd_type->connect) { 1372 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1373 ng_destroy_hook(hook); /* also zaps peer */ 1374 printf("failed in ng_con_part3()\n"); 1375 ERROUT(error); 1376 } 1377 } 1378 /* 1379 * XXX this is wrong for SMP. Possibly we need 1380 * to separate out 'create' and 'invalid' flags. 1381 * should only set flags on hooks we have locked under our node. 1382 */ 1383 hook->hk_flags &= ~HK_INVALID; 1384 done: 1385 NG_FREE_ITEM(item); 1386 return (error); 1387 } 1388 1389 static int 1390 ng_con_part2(node_p node, item_p item, hook_p hook) 1391 { 1392 hook_p peer; 1393 int error = 0; 1394 1395 /* 1396 * When we run, we know that the node 'node' is locked for us. 1397 * Our caller has a reference on the hook. 1398 * Our caller has a reference on the node. 1399 * (In this case our caller is ng_apply_item() ). 1400 * The peer hook has a reference on the hook. 1401 * our node pointer points to the 'dead' node. 1402 * First check the hook name is unique. 1403 * Should not happen because we checked before queueing this. 1404 */ 1405 if (ng_findhook(node, NG_HOOK_NAME(hook)) != NULL) { 1406 TRAP_ERROR(); 1407 ng_destroy_hook(hook); /* should destroy peer too */ 1408 printf("failed in ng_con_part2()\n"); 1409 ERROUT(EEXIST); 1410 } 1411 /* 1412 * Check if the node type code has something to say about it 1413 * If it fails, the unref of the hook will also unref the attached node, 1414 * however since that node is 'ng_deadnode' this will do nothing. 1415 * The peer hook will also be destroyed. 1416 */ 1417 if (node->nd_type->newhook != NULL) { 1418 if ((error = (*node->nd_type->newhook)(node, hook, 1419 hook->hk_name))) { 1420 ng_destroy_hook(hook); /* should destroy peer too */ 1421 printf("failed in ng_con_part2()\n"); 1422 ERROUT(error); 1423 } 1424 } 1425 1426 /* 1427 * The 'type' agrees so far, so go ahead and link it in. 1428 * We'll ask again later when we actually connect the hooks. 1429 */ 1430 hook->hk_node = node; /* just overwrite ng_deadnode */ 1431 NG_NODE_REF(node); /* each hook counts as a reference */ 1432 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1433 node->nd_numhooks++; 1434 NG_HOOK_REF(hook); /* one for the node */ 1435 1436 /* 1437 * We now have a symmetrical situation, where both hooks have been 1438 * linked to their nodes, the newhook methods have been called 1439 * And the references are all correct. The hooks are still marked 1440 * as invalid, as we have not called the 'connect' methods 1441 * yet. 1442 * We can call the local one immediately as we have the 1443 * node locked, but we need to queue the remote one. 1444 */ 1445 if (hook->hk_node->nd_type->connect) { 1446 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1447 ng_destroy_hook(hook); /* also zaps peer */ 1448 printf("failed in ng_con_part2(A)\n"); 1449 ERROUT(error); 1450 } 1451 } 1452 1453 /* 1454 * Acquire topo mutex to avoid race with ng_destroy_hook(). 1455 */ 1456 TOPOLOGY_RLOCK(); 1457 peer = hook->hk_peer; 1458 if (peer == &ng_deadhook) { 1459 TOPOLOGY_RUNLOCK(); 1460 printf("failed in ng_con_part2(B)\n"); 1461 ng_destroy_hook(hook); 1462 ERROUT(ENOENT); 1463 } 1464 TOPOLOGY_RUNLOCK(); 1465 1466 if ((error = ng_send_fn2(peer->hk_node, peer, item, &ng_con_part3, 1467 NULL, 0, NG_REUSE_ITEM))) { 1468 printf("failed in ng_con_part2(C)\n"); 1469 ng_destroy_hook(hook); /* also zaps peer */ 1470 return (error); /* item was consumed. */ 1471 } 1472 hook->hk_flags &= ~HK_INVALID; /* need both to be able to work */ 1473 return (0); /* item was consumed. */ 1474 done: 1475 NG_FREE_ITEM(item); 1476 return (error); 1477 } 1478 1479 /* 1480 * Connect this node with another node. We assume that this node is 1481 * currently locked, as we are only called from an NGM_CONNECT message. 1482 */ 1483 static int 1484 ng_con_nodes(item_p item, node_p node, const char *name, 1485 node_p node2, const char *name2) 1486 { 1487 int error; 1488 hook_p hook; 1489 hook_p hook2; 1490 1491 if (ng_findhook(node2, name2) != NULL) { 1492 return(EEXIST); 1493 } 1494 if ((error = ng_add_hook(node, name, &hook))) /* gives us a ref */ 1495 return (error); 1496 /* Allocate the other hook and link it up */ 1497 NG_ALLOC_HOOK(hook2); 1498 if (hook2 == NULL) { 1499 TRAP_ERROR(); 1500 ng_destroy_hook(hook); /* XXX check ref counts so far */ 1501 NG_HOOK_UNREF(hook); /* including our ref */ 1502 return (ENOMEM); 1503 } 1504 hook2->hk_refs = 1; /* start with a reference for us. */ 1505 hook2->hk_flags = HK_INVALID; 1506 hook2->hk_peer = hook; /* Link the two together */ 1507 hook->hk_peer = hook2; 1508 NG_HOOK_REF(hook); /* Add a ref for the peer to each*/ 1509 NG_HOOK_REF(hook2); 1510 hook2->hk_node = &ng_deadnode; 1511 strlcpy(NG_HOOK_NAME(hook2), name2, NG_HOOKSIZ); 1512 1513 /* 1514 * Queue the function above. 1515 * Procesing continues in that function in the lock context of 1516 * the other node. 1517 */ 1518 if ((error = ng_send_fn2(node2, hook2, item, &ng_con_part2, NULL, 0, 1519 NG_NOFLAGS))) { 1520 printf("failed in ng_con_nodes(): %d\n", error); 1521 ng_destroy_hook(hook); /* also zaps peer */ 1522 } 1523 1524 NG_HOOK_UNREF(hook); /* Let each hook go if it wants to */ 1525 NG_HOOK_UNREF(hook2); 1526 return (error); 1527 } 1528 1529 /* 1530 * Make a peer and connect. 1531 * We assume that the local node is locked. 1532 * The new node probably doesn't need a lock until 1533 * it has a hook, because it cannot really have any work until then, 1534 * but we should think about it a bit more. 1535 * 1536 * The problem may come if the other node also fires up 1537 * some hardware or a timer or some other source of activation, 1538 * also it may already get a command msg via it's ID. 1539 * 1540 * We could use the same method as ng_con_nodes() but we'd have 1541 * to add ability to remove the node when failing. (Not hard, just 1542 * make arg1 point to the node to remove). 1543 * Unless of course we just ignore failure to connect and leave 1544 * an unconnected node? 1545 */ 1546 static int 1547 ng_mkpeer(node_p node, const char *name, const char *name2, char *type) 1548 { 1549 node_p node2; 1550 hook_p hook1, hook2; 1551 int error; 1552 1553 if ((error = ng_make_node(type, &node2))) { 1554 return (error); 1555 } 1556 1557 if ((error = ng_add_hook(node, name, &hook1))) { /* gives us a ref */ 1558 ng_rmnode(node2, NULL, NULL, 0); 1559 return (error); 1560 } 1561 1562 if ((error = ng_add_hook(node2, name2, &hook2))) { 1563 ng_rmnode(node2, NULL, NULL, 0); 1564 ng_destroy_hook(hook1); 1565 NG_HOOK_UNREF(hook1); 1566 return (error); 1567 } 1568 1569 /* 1570 * Actually link the two hooks together. 1571 */ 1572 hook1->hk_peer = hook2; 1573 hook2->hk_peer = hook1; 1574 1575 /* Each hook is referenced by the other */ 1576 NG_HOOK_REF(hook1); 1577 NG_HOOK_REF(hook2); 1578 1579 /* Give each node the opportunity to veto the pending connection */ 1580 if (hook1->hk_node->nd_type->connect) { 1581 error = (*hook1->hk_node->nd_type->connect) (hook1); 1582 } 1583 1584 if ((error == 0) && hook2->hk_node->nd_type->connect) { 1585 error = (*hook2->hk_node->nd_type->connect) (hook2); 1586 } 1587 1588 /* 1589 * drop the references we were holding on the two hooks. 1590 */ 1591 if (error) { 1592 ng_destroy_hook(hook2); /* also zaps hook1 */ 1593 ng_rmnode(node2, NULL, NULL, 0); 1594 } else { 1595 /* As a last act, allow the hooks to be used */ 1596 hook1->hk_flags &= ~HK_INVALID; 1597 hook2->hk_flags &= ~HK_INVALID; 1598 } 1599 NG_HOOK_UNREF(hook1); 1600 NG_HOOK_UNREF(hook2); 1601 return (error); 1602 } 1603 1604 /************************************************************************ 1605 Utility routines to send self messages 1606 ************************************************************************/ 1607 1608 /* Shut this node down as soon as everyone is clear of it */ 1609 /* Should add arg "immediately" to jump the queue */ 1610 int 1611 ng_rmnode_self(node_p node) 1612 { 1613 int error; 1614 1615 if (node == &ng_deadnode) 1616 return (0); 1617 node->nd_flags |= NGF_INVALID; 1618 if (node->nd_flags & NGF_CLOSING) 1619 return (0); 1620 1621 error = ng_send_fn(node, NULL, &ng_rmnode, NULL, 0); 1622 return (error); 1623 } 1624 1625 static void 1626 ng_rmhook_part2(node_p node, hook_p hook, void *arg1, int arg2) 1627 { 1628 ng_destroy_hook(hook); 1629 return ; 1630 } 1631 1632 int 1633 ng_rmhook_self(hook_p hook) 1634 { 1635 int error; 1636 node_p node = NG_HOOK_NODE(hook); 1637 1638 if (node == &ng_deadnode) 1639 return (0); 1640 1641 error = ng_send_fn(node, hook, &ng_rmhook_part2, NULL, 0); 1642 return (error); 1643 } 1644 1645 /*********************************************************************** 1646 * Parse and verify a string of the form: <NODE:><PATH> 1647 * 1648 * Such a string can refer to a specific node or a specific hook 1649 * on a specific node, depending on how you look at it. In the 1650 * latter case, the PATH component must not end in a dot. 1651 * 1652 * Both <NODE:> and <PATH> are optional. The <PATH> is a string 1653 * of hook names separated by dots. This breaks out the original 1654 * string, setting *nodep to "NODE" (or NULL if none) and *pathp 1655 * to "PATH" (or NULL if degenerate). Also, *hookp will point to 1656 * the final hook component of <PATH>, if any, otherwise NULL. 1657 * 1658 * This returns -1 if the path is malformed. The char ** are optional. 1659 ***********************************************************************/ 1660 int 1661 ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp) 1662 { 1663 char *node, *path, *hook; 1664 int k; 1665 1666 /* 1667 * Extract absolute NODE, if any 1668 */ 1669 for (path = addr; *path && *path != ':'; path++); 1670 if (*path) { 1671 node = addr; /* Here's the NODE */ 1672 *path++ = '\0'; /* Here's the PATH */ 1673 1674 /* Node name must not be empty */ 1675 if (!*node) 1676 return -1; 1677 1678 /* A name of "." is OK; otherwise '.' not allowed */ 1679 if (strcmp(node, ".") != 0) { 1680 for (k = 0; node[k]; k++) 1681 if (node[k] == '.') 1682 return -1; 1683 } 1684 } else { 1685 node = NULL; /* No absolute NODE */ 1686 path = addr; /* Here's the PATH */ 1687 } 1688 1689 /* Snoop for illegal characters in PATH */ 1690 for (k = 0; path[k]; k++) 1691 if (path[k] == ':') 1692 return -1; 1693 1694 /* Check for no repeated dots in PATH */ 1695 for (k = 0; path[k]; k++) 1696 if (path[k] == '.' && path[k + 1] == '.') 1697 return -1; 1698 1699 /* Remove extra (degenerate) dots from beginning or end of PATH */ 1700 if (path[0] == '.') 1701 path++; 1702 if (*path && path[strlen(path) - 1] == '.') 1703 path[strlen(path) - 1] = 0; 1704 1705 /* If PATH has a dot, then we're not talking about a hook */ 1706 if (*path) { 1707 for (hook = path, k = 0; path[k]; k++) 1708 if (path[k] == '.') { 1709 hook = NULL; 1710 break; 1711 } 1712 } else 1713 path = hook = NULL; 1714 1715 /* Done */ 1716 if (nodep) 1717 *nodep = node; 1718 if (pathp) 1719 *pathp = path; 1720 if (hookp) 1721 *hookp = hook; 1722 return (0); 1723 } 1724 1725 /* 1726 * Given a path, which may be absolute or relative, and a starting node, 1727 * return the destination node. 1728 */ 1729 int 1730 ng_path2noderef(node_p here, const char *address, node_p *destp, 1731 hook_p *lasthook) 1732 { 1733 char fullpath[NG_PATHSIZ]; 1734 char *nodename, *path; 1735 node_p node, oldnode; 1736 1737 /* Initialize */ 1738 if (destp == NULL) { 1739 TRAP_ERROR(); 1740 return EINVAL; 1741 } 1742 *destp = NULL; 1743 1744 /* Make a writable copy of address for ng_path_parse() */ 1745 strncpy(fullpath, address, sizeof(fullpath) - 1); 1746 fullpath[sizeof(fullpath) - 1] = '\0'; 1747 1748 /* Parse out node and sequence of hooks */ 1749 if (ng_path_parse(fullpath, &nodename, &path, NULL) < 0) { 1750 TRAP_ERROR(); 1751 return EINVAL; 1752 } 1753 1754 /* 1755 * For an absolute address, jump to the starting node. 1756 * Note that this holds a reference on the node for us. 1757 * Don't forget to drop the reference if we don't need it. 1758 */ 1759 if (nodename) { 1760 node = ng_name2noderef(here, nodename); 1761 if (node == NULL) { 1762 TRAP_ERROR(); 1763 return (ENOENT); 1764 } 1765 } else { 1766 if (here == NULL) { 1767 TRAP_ERROR(); 1768 return (EINVAL); 1769 } 1770 node = here; 1771 NG_NODE_REF(node); 1772 } 1773 1774 if (path == NULL) { 1775 if (lasthook != NULL) 1776 *lasthook = NULL; 1777 *destp = node; 1778 return (0); 1779 } 1780 1781 /* 1782 * Now follow the sequence of hooks 1783 * 1784 * XXXGL: The path may demolish as we go the sequence, but if 1785 * we hold the topology mutex at critical places, then, I hope, 1786 * we would always have valid pointers in hand, although the 1787 * path behind us may no longer exist. 1788 */ 1789 for (;;) { 1790 hook_p hook; 1791 char *segment; 1792 1793 /* 1794 * Break out the next path segment. Replace the dot we just 1795 * found with a NUL; "path" points to the next segment (or the 1796 * NUL at the end). 1797 */ 1798 for (segment = path; *path != '\0'; path++) { 1799 if (*path == '.') { 1800 *path++ = '\0'; 1801 break; 1802 } 1803 } 1804 1805 /* We have a segment, so look for a hook by that name */ 1806 hook = ng_findhook(node, segment); 1807 1808 TOPOLOGY_WLOCK(); 1809 /* Can't get there from here... */ 1810 if (hook == NULL || NG_HOOK_PEER(hook) == NULL || 1811 NG_HOOK_NOT_VALID(hook) || 1812 NG_HOOK_NOT_VALID(NG_HOOK_PEER(hook))) { 1813 TRAP_ERROR(); 1814 NG_NODE_UNREF(node); 1815 TOPOLOGY_WUNLOCK(); 1816 return (ENOENT); 1817 } 1818 1819 /* 1820 * Hop on over to the next node 1821 * XXX 1822 * Big race conditions here as hooks and nodes go away 1823 * *** Idea.. store an ng_ID_t in each hook and use that 1824 * instead of the direct hook in this crawl? 1825 */ 1826 oldnode = node; 1827 if ((node = NG_PEER_NODE(hook))) 1828 NG_NODE_REF(node); /* XXX RACE */ 1829 NG_NODE_UNREF(oldnode); /* XXX another race */ 1830 if (NG_NODE_NOT_VALID(node)) { 1831 NG_NODE_UNREF(node); /* XXX more races */ 1832 TOPOLOGY_WUNLOCK(); 1833 TRAP_ERROR(); 1834 return (ENXIO); 1835 } 1836 1837 if (*path == '\0') { 1838 if (lasthook != NULL) { 1839 if (hook != NULL) { 1840 *lasthook = NG_HOOK_PEER(hook); 1841 NG_HOOK_REF(*lasthook); 1842 } else 1843 *lasthook = NULL; 1844 } 1845 TOPOLOGY_WUNLOCK(); 1846 *destp = node; 1847 return (0); 1848 } 1849 TOPOLOGY_WUNLOCK(); 1850 } 1851 } 1852 1853 #ifndef FSTACK 1854 /***************************************************************\ 1855 * Input queue handling. 1856 * All activities are submitted to the node via the input queue 1857 * which implements a multiple-reader/single-writer gate. 1858 * Items which cannot be handled immediately are queued. 1859 * 1860 * read-write queue locking inline functions * 1861 \***************************************************************/ 1862 1863 static __inline void ng_queue_rw(node_p node, item_p item, int rw); 1864 static __inline item_p ng_dequeue(node_p node, int *rw); 1865 static __inline item_p ng_acquire_read(node_p node, item_p item); 1866 static __inline item_p ng_acquire_write(node_p node, item_p item); 1867 static __inline void ng_leave_read(node_p node); 1868 static __inline void ng_leave_write(node_p node); 1869 1870 /* 1871 * Definition of the bits fields in the ng_queue flag word. 1872 * Defined here rather than in netgraph.h because no-one should fiddle 1873 * with them. 1874 * 1875 * The ordering here may be important! don't shuffle these. 1876 */ 1877 /*- 1878 Safety Barrier--------+ (adjustable to suit taste) (not used yet) 1879 | 1880 V 1881 +-------+-------+-------+-------+-------+-------+-------+-------+ 1882 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | 1883 | |A|c|t|i|v|e| |R|e|a|d|e|r| |C|o|u|n|t| | | | | | | | | |P|A| 1884 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |O|W| 1885 +-------+-------+-------+-------+-------+-------+-------+-------+ 1886 \___________________________ ____________________________/ | | 1887 V | | 1888 [active reader count] | | 1889 | | 1890 Operation Pending -------------------------------+ | 1891 | 1892 Active Writer ---------------------------------------+ 1893 1894 Node queue has such semantics: 1895 - All flags modifications are atomic. 1896 - Reader count can be incremented only if there is no writer or pending flags. 1897 As soon as this can't be done with single operation, it is implemented with 1898 spin loop and atomic_cmpset(). 1899 - Writer flag can be set only if there is no any bits set. 1900 It is implemented with atomic_cmpset(). 1901 - Pending flag can be set any time, but to avoid collision on queue processing 1902 all queue fields are protected by the mutex. 1903 - Queue processing thread reads queue holding the mutex, but releases it while 1904 processing. When queue is empty pending flag is removed. 1905 */ 1906 1907 #define WRITER_ACTIVE 0x00000001 1908 #define OP_PENDING 0x00000002 1909 #define READER_INCREMENT 0x00000004 1910 #define READER_MASK 0xfffffffc /* Not valid if WRITER_ACTIVE is set */ 1911 #define SAFETY_BARRIER 0x00100000 /* 128K items queued should be enough */ 1912 1913 /* Defines of more elaborate states on the queue */ 1914 /* Mask of bits a new read cares about */ 1915 #define NGQ_RMASK (WRITER_ACTIVE|OP_PENDING) 1916 1917 /* Mask of bits a new write cares about */ 1918 #define NGQ_WMASK (NGQ_RMASK|READER_MASK) 1919 1920 /* Test to decide if there is something on the queue. */ 1921 #define QUEUE_ACTIVE(QP) ((QP)->q_flags & OP_PENDING) 1922 1923 /* How to decide what the next queued item is. */ 1924 #define HEAD_IS_READER(QP) NGI_QUEUED_READER(STAILQ_FIRST(&(QP)->queue)) 1925 #define HEAD_IS_WRITER(QP) NGI_QUEUED_WRITER(STAILQ_FIRST(&(QP)->queue)) /* notused */ 1926 1927 /* Read the status to decide if the next item on the queue can now run. */ 1928 #define QUEUED_READER_CAN_PROCEED(QP) \ 1929 (((QP)->q_flags & (NGQ_RMASK & ~OP_PENDING)) == 0) 1930 #define QUEUED_WRITER_CAN_PROCEED(QP) \ 1931 (((QP)->q_flags & (NGQ_WMASK & ~OP_PENDING)) == 0) 1932 1933 /* Is there a chance of getting ANY work off the queue? */ 1934 #define NEXT_QUEUED_ITEM_CAN_PROCEED(QP) \ 1935 ((HEAD_IS_READER(QP)) ? QUEUED_READER_CAN_PROCEED(QP) : \ 1936 QUEUED_WRITER_CAN_PROCEED(QP)) 1937 1938 #define NGQRW_R 0 1939 #define NGQRW_W 1 1940 1941 #define NGQ2_WORKQ 0x00000001 1942 1943 /* 1944 * Taking into account the current state of the queue and node, possibly take 1945 * the next entry off the queue and return it. Return NULL if there was 1946 * nothing we could return, either because there really was nothing there, or 1947 * because the node was in a state where it cannot yet process the next item 1948 * on the queue. 1949 */ 1950 static __inline item_p 1951 ng_dequeue(node_p node, int *rw) 1952 { 1953 item_p item; 1954 struct ng_queue *ngq = &node->nd_input_queue; 1955 1956 /* This MUST be called with the mutex held. */ 1957 mtx_assert(&ngq->q_mtx, MA_OWNED); 1958 1959 /* If there is nothing queued, then just return. */ 1960 if (!QUEUE_ACTIVE(ngq)) { 1961 CTR4(KTR_NET, "%20s: node [%x] (%p) queue empty; " 1962 "queue flags 0x%lx", __func__, 1963 node->nd_ID, node, ngq->q_flags); 1964 return (NULL); 1965 } 1966 1967 /* 1968 * From here, we can assume there is a head item. 1969 * We need to find out what it is and if it can be dequeued, given 1970 * the current state of the node. 1971 */ 1972 if (HEAD_IS_READER(ngq)) { 1973 while (1) { 1974 long t = ngq->q_flags; 1975 if (t & WRITER_ACTIVE) { 1976 /* There is writer, reader can't proceed. */ 1977 CTR4(KTR_NET, "%20s: node [%x] (%p) queued " 1978 "reader can't proceed; queue flags 0x%lx", 1979 __func__, node->nd_ID, node, t); 1980 return (NULL); 1981 } 1982 if (atomic_cmpset_acq_int(&ngq->q_flags, t, 1983 t + READER_INCREMENT)) 1984 break; 1985 cpu_spinwait(); 1986 } 1987 /* We have got reader lock for the node. */ 1988 *rw = NGQRW_R; 1989 } else if (atomic_cmpset_acq_int(&ngq->q_flags, OP_PENDING, 1990 OP_PENDING + WRITER_ACTIVE)) { 1991 /* We have got writer lock for the node. */ 1992 *rw = NGQRW_W; 1993 } else { 1994 /* There is somebody other, writer can't proceed. */ 1995 CTR4(KTR_NET, "%20s: node [%x] (%p) queued writer can't " 1996 "proceed; queue flags 0x%lx", __func__, node->nd_ID, node, 1997 ngq->q_flags); 1998 return (NULL); 1999 } 2000 2001 /* 2002 * Now we dequeue the request (whatever it may be) and correct the 2003 * pending flags and the next and last pointers. 2004 */ 2005 item = STAILQ_FIRST(&ngq->queue); 2006 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 2007 if (STAILQ_EMPTY(&ngq->queue)) 2008 atomic_clear_int(&ngq->q_flags, OP_PENDING); 2009 CTR6(KTR_NET, "%20s: node [%x] (%p) returning item %p as %s; queue " 2010 "flags 0x%lx", __func__, node->nd_ID, node, item, *rw ? "WRITER" : 2011 "READER", ngq->q_flags); 2012 return (item); 2013 } 2014 2015 /* 2016 * Queue a packet to be picked up later by someone else. 2017 * If the queue could be run now, add node to the queue handler's worklist. 2018 */ 2019 static __inline void 2020 ng_queue_rw(node_p node, item_p item, int rw) 2021 { 2022 struct ng_queue *ngq = &node->nd_input_queue; 2023 if (rw == NGQRW_W) 2024 NGI_SET_WRITER(item); 2025 else 2026 NGI_SET_READER(item); 2027 item->depth = 1; 2028 2029 NG_QUEUE_LOCK(ngq); 2030 /* Set OP_PENDING flag and enqueue the item. */ 2031 atomic_set_int(&ngq->q_flags, OP_PENDING); 2032 STAILQ_INSERT_TAIL(&ngq->queue, item, el_next); 2033 2034 CTR5(KTR_NET, "%20s: node [%x] (%p) queued item %p as %s", __func__, 2035 node->nd_ID, node, item, rw ? "WRITER" : "READER" ); 2036 2037 /* 2038 * We can take the worklist lock with the node locked 2039 * BUT NOT THE REVERSE! 2040 */ 2041 if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2042 ng_worklist_add(node); 2043 NG_QUEUE_UNLOCK(ngq); 2044 } 2045 2046 /* Acquire reader lock on node. If node is busy, queue the packet. */ 2047 static __inline item_p 2048 ng_acquire_read(node_p node, item_p item) 2049 { 2050 KASSERT(node != &ng_deadnode, 2051 ("%s: working on deadnode", __func__)); 2052 2053 /* Reader needs node without writer and pending items. */ 2054 for (;;) { 2055 long t = node->nd_input_queue.q_flags; 2056 if (t & NGQ_RMASK) 2057 break; /* Node is not ready for reader. */ 2058 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, t, 2059 t + READER_INCREMENT)) { 2060 /* Successfully grabbed node */ 2061 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 2062 __func__, node->nd_ID, node, item); 2063 return (item); 2064 } 2065 cpu_spinwait(); 2066 } 2067 2068 /* Queue the request for later. */ 2069 ng_queue_rw(node, item, NGQRW_R); 2070 2071 return (NULL); 2072 } 2073 2074 /* Acquire writer lock on node. If node is busy, queue the packet. */ 2075 static __inline item_p 2076 ng_acquire_write(node_p node, item_p item) 2077 { 2078 KASSERT(node != &ng_deadnode, 2079 ("%s: working on deadnode", __func__)); 2080 2081 /* Writer needs completely idle node. */ 2082 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, 0, 2083 WRITER_ACTIVE)) { 2084 /* Successfully grabbed node */ 2085 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 2086 __func__, node->nd_ID, node, item); 2087 return (item); 2088 } 2089 2090 /* Queue the request for later. */ 2091 ng_queue_rw(node, item, NGQRW_W); 2092 2093 return (NULL); 2094 } 2095 2096 #if 0 2097 static __inline item_p 2098 ng_upgrade_write(node_p node, item_p item) 2099 { 2100 struct ng_queue *ngq = &node->nd_input_queue; 2101 KASSERT(node != &ng_deadnode, 2102 ("%s: working on deadnode", __func__)); 2103 2104 NGI_SET_WRITER(item); 2105 2106 NG_QUEUE_LOCK(ngq); 2107 2108 /* 2109 * There will never be no readers as we are there ourselves. 2110 * Set the WRITER_ACTIVE flags ASAP to block out fast track readers. 2111 * The caller we are running from will call ng_leave_read() 2112 * soon, so we must account for that. We must leave again with the 2113 * READER lock. If we find other readers, then 2114 * queue the request for later. However "later" may be rignt now 2115 * if there are no readers. We don't really care if there are queued 2116 * items as we will bypass them anyhow. 2117 */ 2118 atomic_add_int(&ngq->q_flags, WRITER_ACTIVE - READER_INCREMENT); 2119 if ((ngq->q_flags & (NGQ_WMASK & ~OP_PENDING)) == WRITER_ACTIVE) { 2120 NG_QUEUE_UNLOCK(ngq); 2121 2122 /* It's just us, act on the item. */ 2123 /* will NOT drop writer lock when done */ 2124 ng_apply_item(node, item, 0); 2125 2126 /* 2127 * Having acted on the item, atomically 2128 * downgrade back to READER and finish up. 2129 */ 2130 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2131 2132 /* Our caller will call ng_leave_read() */ 2133 return; 2134 } 2135 /* 2136 * It's not just us active, so queue us AT THE HEAD. 2137 * "Why?" I hear you ask. 2138 * Put us at the head of the queue as we've already been 2139 * through it once. If there is nothing else waiting, 2140 * set the correct flags. 2141 */ 2142 if (STAILQ_EMPTY(&ngq->queue)) { 2143 /* We've gone from, 0 to 1 item in the queue */ 2144 atomic_set_int(&ngq->q_flags, OP_PENDING); 2145 2146 CTR3(KTR_NET, "%20s: node [%x] (%p) set OP_PENDING", __func__, 2147 node->nd_ID, node); 2148 }; 2149 STAILQ_INSERT_HEAD(&ngq->queue, item, el_next); 2150 CTR4(KTR_NET, "%20s: node [%x] (%p) requeued item %p as WRITER", 2151 __func__, node->nd_ID, node, item ); 2152 2153 /* Reverse what we did above. That downgrades us back to reader */ 2154 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2155 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2156 ng_worklist_add(node); 2157 NG_QUEUE_UNLOCK(ngq); 2158 2159 return; 2160 } 2161 #endif 2162 2163 /* Release reader lock. */ 2164 static __inline void 2165 ng_leave_read(node_p node) 2166 { 2167 atomic_subtract_rel_int(&node->nd_input_queue.q_flags, READER_INCREMENT); 2168 } 2169 2170 /* Release writer lock. */ 2171 static __inline void 2172 ng_leave_write(node_p node) 2173 { 2174 atomic_clear_rel_int(&node->nd_input_queue.q_flags, WRITER_ACTIVE); 2175 } 2176 2177 /* Purge node queue. Called on node shutdown. */ 2178 static void 2179 ng_flush_input_queue(node_p node) 2180 { 2181 struct ng_queue *ngq = &node->nd_input_queue; 2182 item_p item; 2183 2184 NG_QUEUE_LOCK(ngq); 2185 while ((item = STAILQ_FIRST(&ngq->queue)) != NULL) { 2186 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 2187 if (STAILQ_EMPTY(&ngq->queue)) 2188 atomic_clear_int(&ngq->q_flags, OP_PENDING); 2189 NG_QUEUE_UNLOCK(ngq); 2190 2191 /* If the item is supplying a callback, call it with an error */ 2192 if (item->apply != NULL) { 2193 if (item->depth == 1) 2194 item->apply->error = ENOENT; 2195 if (refcount_release(&item->apply->refs)) { 2196 (*item->apply->apply)(item->apply->context, 2197 item->apply->error); 2198 } 2199 } 2200 NG_FREE_ITEM(item); 2201 NG_QUEUE_LOCK(ngq); 2202 } 2203 NG_QUEUE_UNLOCK(ngq); 2204 } 2205 #endif 2206 2207 /*********************************************************************** 2208 * Externally visible method for sending or queueing messages or data. 2209 ***********************************************************************/ 2210 2211 /* 2212 * The module code should have filled out the item correctly by this stage: 2213 * Common: 2214 * reference to destination node. 2215 * Reference to destination rcv hook if relevant. 2216 * apply pointer must be or NULL or reference valid struct ng_apply_info. 2217 * Data: 2218 * pointer to mbuf 2219 * Control_Message: 2220 * pointer to msg. 2221 * ID of original sender node. (return address) 2222 * Function: 2223 * Function pointer 2224 * void * argument 2225 * integer argument 2226 * 2227 * The nodes have several routines and macros to help with this task: 2228 */ 2229 2230 int 2231 ng_snd_item(item_p item, int flags) 2232 { 2233 hook_p hook; 2234 node_p node; 2235 #ifndef FSTACK 2236 int queue, rw; 2237 struct ng_queue *ngq; 2238 #endif 2239 int error = 0; 2240 2241 /* We are sending item, so it must be present! */ 2242 KASSERT(item != NULL, ("ng_snd_item: item is NULL")); 2243 2244 #ifdef NETGRAPH_DEBUG 2245 _ngi_check(item, __FILE__, __LINE__); 2246 #endif 2247 2248 /* Item was sent once more, postpone apply() call. */ 2249 if (item->apply) 2250 refcount_acquire(&item->apply->refs); 2251 2252 node = NGI_NODE(item); 2253 /* Node is never optional. */ 2254 KASSERT(node != NULL, ("ng_snd_item: node is NULL")); 2255 2256 hook = NGI_HOOK(item); 2257 /* Valid hook and mbuf are mandatory for data. */ 2258 if ((item->el_flags & NGQF_TYPE) == NGQF_DATA) { 2259 KASSERT(hook != NULL, ("ng_snd_item: hook for data is NULL")); 2260 if (NGI_M(item) == NULL) 2261 ERROUT(EINVAL); 2262 CHECK_DATA_MBUF(NGI_M(item)); 2263 } 2264 2265 #ifndef FSTACK 2266 /* 2267 * If the item or the node specifies single threading, force 2268 * writer semantics. Similarly, the node may say one hook always 2269 * produces writers. These are overrides. 2270 */ 2271 if (((item->el_flags & NGQF_RW) == NGQF_WRITER) || 2272 (node->nd_flags & NGF_FORCE_WRITER) || 2273 (hook && (hook->hk_flags & HK_FORCE_WRITER))) { 2274 rw = NGQRW_W; 2275 } else { 2276 rw = NGQRW_R; 2277 } 2278 2279 /* 2280 * If sender or receiver requests queued delivery, or call graph 2281 * loops back from outbound to inbound path, or stack usage 2282 * level is dangerous - enqueue message. 2283 */ 2284 if ((flags & NG_QUEUE) || (hook && (hook->hk_flags & HK_QUEUE))) { 2285 queue = 1; 2286 } else if (hook && (hook->hk_flags & HK_TO_INBOUND) && 2287 curthread->td_ng_outbound) { 2288 queue = 1; 2289 } else { 2290 queue = 0; 2291 #ifdef GET_STACK_USAGE 2292 /* 2293 * Most of netgraph nodes have small stack consumption and 2294 * for them 25% of free stack space is more than enough. 2295 * Nodes/hooks with higher stack usage should be marked as 2296 * HI_STACK. For them 50% of stack will be guaranteed then. 2297 * XXX: Values 25% and 50% are completely empirical. 2298 */ 2299 size_t st, su, sl; 2300 GET_STACK_USAGE(st, su); 2301 sl = st - su; 2302 if ((sl * 4 < st) || ((sl * 2 < st) && 2303 ((node->nd_flags & NGF_HI_STACK) || (hook && 2304 (hook->hk_flags & HK_HI_STACK))))) 2305 queue = 1; 2306 #endif 2307 } 2308 2309 if (queue) { 2310 /* Put it on the queue for that node*/ 2311 ng_queue_rw(node, item, rw); 2312 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2313 } 2314 2315 /* 2316 * We already decided how we will be queueud or treated. 2317 * Try get the appropriate operating permission. 2318 */ 2319 if (rw == NGQRW_R) 2320 item = ng_acquire_read(node, item); 2321 else 2322 item = ng_acquire_write(node, item); 2323 2324 /* Item was queued while trying to get permission. */ 2325 if (item == NULL) 2326 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2327 #endif 2328 2329 NGI_GET_NODE(item, node); /* zaps stored node */ 2330 2331 item->depth++; 2332 #ifndef FSTACK 2333 error = ng_apply_item(node, item, rw); /* drops r/w lock when done */ 2334 #else 2335 error = ng_apply_item(node, item, 0); 2336 #endif 2337 2338 #ifndef FSTACK 2339 /* If something is waiting on queue and ready, schedule it. */ 2340 ngq = &node->nd_input_queue; 2341 if (QUEUE_ACTIVE(ngq)) { 2342 NG_QUEUE_LOCK(ngq); 2343 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2344 ng_worklist_add(node); 2345 NG_QUEUE_UNLOCK(ngq); 2346 } 2347 #endif 2348 2349 /* 2350 * Node may go away as soon as we remove the reference. 2351 * Whatever we do, DO NOT access the node again! 2352 */ 2353 NG_NODE_UNREF(node); 2354 2355 return (error); 2356 2357 done: 2358 /* If was not sent, apply callback here. */ 2359 if (item->apply != NULL) { 2360 if (item->depth == 0 && error != 0) 2361 item->apply->error = error; 2362 if (refcount_release(&item->apply->refs)) { 2363 (*item->apply->apply)(item->apply->context, 2364 item->apply->error); 2365 } 2366 } 2367 2368 NG_FREE_ITEM(item); 2369 return (error); 2370 } 2371 2372 /* 2373 * We have an item that was possibly queued somewhere. 2374 * It should contain all the information needed 2375 * to run it on the appropriate node/hook. 2376 * If there is apply pointer and we own the last reference, call apply(). 2377 */ 2378 static int 2379 ng_apply_item(node_p node, item_p item, int rw) 2380 { 2381 hook_p hook; 2382 ng_rcvdata_t *rcvdata; 2383 ng_rcvmsg_t *rcvmsg; 2384 struct ng_apply_info *apply; 2385 int error = 0, depth; 2386 2387 /* Node and item are never optional. */ 2388 KASSERT(node != NULL, ("ng_apply_item: node is NULL")); 2389 KASSERT(item != NULL, ("ng_apply_item: item is NULL")); 2390 2391 NGI_GET_HOOK(item, hook); /* clears stored hook */ 2392 #ifdef NETGRAPH_DEBUG 2393 _ngi_check(item, __FILE__, __LINE__); 2394 #endif 2395 2396 apply = item->apply; 2397 depth = item->depth; 2398 2399 switch (item->el_flags & NGQF_TYPE) { 2400 case NGQF_DATA: 2401 /* 2402 * Check things are still ok as when we were queued. 2403 */ 2404 KASSERT(hook != NULL, ("ng_apply_item: hook for data is NULL")); 2405 if (NG_HOOK_NOT_VALID(hook) || 2406 NG_NODE_NOT_VALID(node)) { 2407 error = EIO; 2408 NG_FREE_ITEM(item); 2409 break; 2410 } 2411 /* 2412 * If no receive method, just silently drop it. 2413 * Give preference to the hook over-ride method. 2414 */ 2415 if ((!(rcvdata = hook->hk_rcvdata)) && 2416 (!(rcvdata = NG_HOOK_NODE(hook)->nd_type->rcvdata))) { 2417 error = 0; 2418 NG_FREE_ITEM(item); 2419 break; 2420 } 2421 error = (*rcvdata)(hook, item); 2422 break; 2423 case NGQF_MESG: 2424 if (hook && NG_HOOK_NOT_VALID(hook)) { 2425 /* 2426 * The hook has been zapped then we can't use it. 2427 * Immediately drop its reference. 2428 * The message may not need it. 2429 */ 2430 NG_HOOK_UNREF(hook); 2431 hook = NULL; 2432 } 2433 /* 2434 * Similarly, if the node is a zombie there is 2435 * nothing we can do with it, drop everything. 2436 */ 2437 if (NG_NODE_NOT_VALID(node)) { 2438 TRAP_ERROR(); 2439 error = EINVAL; 2440 NG_FREE_ITEM(item); 2441 break; 2442 } 2443 /* 2444 * Call the appropriate message handler for the object. 2445 * It is up to the message handler to free the message. 2446 * If it's a generic message, handle it generically, 2447 * otherwise call the type's message handler (if it exists). 2448 * XXX (race). Remember that a queued message may 2449 * reference a node or hook that has just been 2450 * invalidated. It will exist as the queue code 2451 * is holding a reference, but.. 2452 */ 2453 if ((NGI_MSG(item)->header.typecookie == NGM_GENERIC_COOKIE) && 2454 ((NGI_MSG(item)->header.flags & NGF_RESP) == 0)) { 2455 error = ng_generic_msg(node, item, hook); 2456 break; 2457 } 2458 if (((!hook) || (!(rcvmsg = hook->hk_rcvmsg))) && 2459 (!(rcvmsg = node->nd_type->rcvmsg))) { 2460 TRAP_ERROR(); 2461 error = 0; 2462 NG_FREE_ITEM(item); 2463 break; 2464 } 2465 error = (*rcvmsg)(node, item, hook); 2466 break; 2467 case NGQF_FN: 2468 case NGQF_FN2: 2469 /* 2470 * In the case of the shutdown message we allow it to hit 2471 * even if the node is invalid. 2472 */ 2473 if (NG_NODE_NOT_VALID(node) && 2474 NGI_FN(item) != &ng_rmnode) { 2475 TRAP_ERROR(); 2476 error = EINVAL; 2477 NG_FREE_ITEM(item); 2478 break; 2479 } 2480 /* Same is about some internal functions and invalid hook. */ 2481 if (hook && NG_HOOK_NOT_VALID(hook) && 2482 NGI_FN2(item) != &ng_con_part2 && 2483 NGI_FN2(item) != &ng_con_part3 && 2484 NGI_FN(item) != &ng_rmhook_part2) { 2485 TRAP_ERROR(); 2486 error = EINVAL; 2487 NG_FREE_ITEM(item); 2488 break; 2489 } 2490 2491 if ((item->el_flags & NGQF_TYPE) == NGQF_FN) { 2492 (*NGI_FN(item))(node, hook, NGI_ARG1(item), 2493 NGI_ARG2(item)); 2494 NG_FREE_ITEM(item); 2495 } else /* it is NGQF_FN2 */ 2496 error = (*NGI_FN2(item))(node, item, hook); 2497 break; 2498 } 2499 /* 2500 * We held references on some of the resources 2501 * that we took from the item. Now that we have 2502 * finished doing everything, drop those references. 2503 */ 2504 if (hook) 2505 NG_HOOK_UNREF(hook); 2506 2507 #ifndef FSTACK 2508 if (rw == NGQRW_R) 2509 ng_leave_read(node); 2510 else 2511 ng_leave_write(node); 2512 #endif 2513 2514 /* Apply callback. */ 2515 if (apply != NULL) { 2516 if (depth == 1 && error != 0) 2517 apply->error = error; 2518 if (refcount_release(&apply->refs)) 2519 (*apply->apply)(apply->context, apply->error); 2520 } 2521 2522 return (error); 2523 } 2524 2525 /*********************************************************************** 2526 * Implement the 'generic' control messages 2527 ***********************************************************************/ 2528 static int 2529 ng_generic_msg(node_p here, item_p item, hook_p lasthook) 2530 { 2531 int error = 0; 2532 struct ng_mesg *msg; 2533 struct ng_mesg *resp = NULL; 2534 2535 NGI_GET_MSG(item, msg); 2536 if (msg->header.typecookie != NGM_GENERIC_COOKIE) { 2537 TRAP_ERROR(); 2538 error = EINVAL; 2539 goto out; 2540 } 2541 switch (msg->header.cmd) { 2542 case NGM_SHUTDOWN: 2543 ng_rmnode(here, NULL, NULL, 0); 2544 break; 2545 case NGM_MKPEER: 2546 { 2547 struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data; 2548 2549 if (msg->header.arglen != sizeof(*mkp)) { 2550 TRAP_ERROR(); 2551 error = EINVAL; 2552 break; 2553 } 2554 mkp->type[sizeof(mkp->type) - 1] = '\0'; 2555 mkp->ourhook[sizeof(mkp->ourhook) - 1] = '\0'; 2556 mkp->peerhook[sizeof(mkp->peerhook) - 1] = '\0'; 2557 error = ng_mkpeer(here, mkp->ourhook, mkp->peerhook, mkp->type); 2558 break; 2559 } 2560 case NGM_CONNECT: 2561 { 2562 struct ngm_connect *const con = 2563 (struct ngm_connect *) msg->data; 2564 node_p node2; 2565 2566 if (msg->header.arglen != sizeof(*con)) { 2567 TRAP_ERROR(); 2568 error = EINVAL; 2569 break; 2570 } 2571 con->path[sizeof(con->path) - 1] = '\0'; 2572 con->ourhook[sizeof(con->ourhook) - 1] = '\0'; 2573 con->peerhook[sizeof(con->peerhook) - 1] = '\0'; 2574 /* Don't forget we get a reference.. */ 2575 error = ng_path2noderef(here, con->path, &node2, NULL); 2576 if (error) 2577 break; 2578 error = ng_con_nodes(item, here, con->ourhook, 2579 node2, con->peerhook); 2580 NG_NODE_UNREF(node2); 2581 break; 2582 } 2583 case NGM_NAME: 2584 { 2585 struct ngm_name *const nam = (struct ngm_name *) msg->data; 2586 2587 if (msg->header.arglen != sizeof(*nam)) { 2588 TRAP_ERROR(); 2589 error = EINVAL; 2590 break; 2591 } 2592 nam->name[sizeof(nam->name) - 1] = '\0'; 2593 error = ng_name_node(here, nam->name); 2594 break; 2595 } 2596 case NGM_RMHOOK: 2597 { 2598 struct ngm_rmhook *const rmh = (struct ngm_rmhook *) msg->data; 2599 hook_p hook; 2600 2601 if (msg->header.arglen != sizeof(*rmh)) { 2602 TRAP_ERROR(); 2603 error = EINVAL; 2604 break; 2605 } 2606 rmh->ourhook[sizeof(rmh->ourhook) - 1] = '\0'; 2607 if ((hook = ng_findhook(here, rmh->ourhook)) != NULL) 2608 ng_destroy_hook(hook); 2609 break; 2610 } 2611 case NGM_NODEINFO: 2612 { 2613 struct nodeinfo *ni; 2614 2615 NG_MKRESPONSE(resp, msg, sizeof(*ni), M_NOWAIT); 2616 if (resp == NULL) { 2617 error = ENOMEM; 2618 break; 2619 } 2620 2621 /* Fill in node info */ 2622 ni = (struct nodeinfo *) resp->data; 2623 if (NG_NODE_HAS_NAME(here)) 2624 strcpy(ni->name, NG_NODE_NAME(here)); 2625 strcpy(ni->type, here->nd_type->name); 2626 ni->id = ng_node2ID(here); 2627 ni->hooks = here->nd_numhooks; 2628 break; 2629 } 2630 case NGM_LISTHOOKS: 2631 { 2632 const int nhooks = here->nd_numhooks; 2633 struct hooklist *hl; 2634 struct nodeinfo *ni; 2635 hook_p hook; 2636 2637 /* Get response struct */ 2638 NG_MKRESPONSE(resp, msg, sizeof(*hl) + 2639 (nhooks * sizeof(struct linkinfo)), M_NOWAIT); 2640 if (resp == NULL) { 2641 error = ENOMEM; 2642 break; 2643 } 2644 hl = (struct hooklist *) resp->data; 2645 ni = &hl->nodeinfo; 2646 2647 /* Fill in node info */ 2648 if (NG_NODE_HAS_NAME(here)) 2649 strcpy(ni->name, NG_NODE_NAME(here)); 2650 strcpy(ni->type, here->nd_type->name); 2651 ni->id = ng_node2ID(here); 2652 2653 /* Cycle through the linked list of hooks */ 2654 ni->hooks = 0; 2655 LIST_FOREACH(hook, &here->nd_hooks, hk_hooks) { 2656 struct linkinfo *const link = &hl->link[ni->hooks]; 2657 2658 if (ni->hooks >= nhooks) { 2659 log(LOG_ERR, "%s: number of %s changed\n", 2660 __func__, "hooks"); 2661 break; 2662 } 2663 if (NG_HOOK_NOT_VALID(hook)) 2664 continue; 2665 strcpy(link->ourhook, NG_HOOK_NAME(hook)); 2666 strcpy(link->peerhook, NG_PEER_HOOK_NAME(hook)); 2667 if (NG_PEER_NODE_NAME(hook)[0] != '\0') 2668 strcpy(link->nodeinfo.name, 2669 NG_PEER_NODE_NAME(hook)); 2670 strcpy(link->nodeinfo.type, 2671 NG_PEER_NODE(hook)->nd_type->name); 2672 link->nodeinfo.id = ng_node2ID(NG_PEER_NODE(hook)); 2673 link->nodeinfo.hooks = NG_PEER_NODE(hook)->nd_numhooks; 2674 ni->hooks++; 2675 } 2676 break; 2677 } 2678 2679 case NGM_LISTNODES: 2680 { 2681 struct namelist *nl; 2682 node_p node; 2683 int i; 2684 2685 IDHASH_RLOCK(); 2686 /* Get response struct. */ 2687 NG_MKRESPONSE(resp, msg, sizeof(*nl) + 2688 (V_ng_nodes * sizeof(struct nodeinfo)), M_NOWAIT); 2689 if (resp == NULL) { 2690 IDHASH_RUNLOCK(); 2691 error = ENOMEM; 2692 break; 2693 } 2694 nl = (struct namelist *) resp->data; 2695 2696 /* Cycle through the lists of nodes. */ 2697 nl->numnames = 0; 2698 for (i = 0; i <= V_ng_ID_hmask; i++) { 2699 LIST_FOREACH(node, &V_ng_ID_hash[i], nd_idnodes) { 2700 struct nodeinfo *const np = 2701 &nl->nodeinfo[nl->numnames]; 2702 2703 if (NG_NODE_NOT_VALID(node)) 2704 continue; 2705 if (NG_NODE_HAS_NAME(node)) 2706 strcpy(np->name, NG_NODE_NAME(node)); 2707 strcpy(np->type, node->nd_type->name); 2708 np->id = ng_node2ID(node); 2709 np->hooks = node->nd_numhooks; 2710 KASSERT(nl->numnames < V_ng_nodes, 2711 ("%s: no space", __func__)); 2712 nl->numnames++; 2713 } 2714 } 2715 IDHASH_RUNLOCK(); 2716 break; 2717 } 2718 case NGM_LISTNAMES: 2719 { 2720 struct namelist *nl; 2721 node_p node; 2722 int i; 2723 2724 NAMEHASH_RLOCK(); 2725 /* Get response struct. */ 2726 NG_MKRESPONSE(resp, msg, sizeof(*nl) + 2727 (V_ng_named_nodes * sizeof(struct nodeinfo)), M_NOWAIT); 2728 if (resp == NULL) { 2729 NAMEHASH_RUNLOCK(); 2730 error = ENOMEM; 2731 break; 2732 } 2733 nl = (struct namelist *) resp->data; 2734 2735 /* Cycle through the lists of nodes. */ 2736 nl->numnames = 0; 2737 for (i = 0; i <= V_ng_name_hmask; i++) { 2738 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 2739 struct nodeinfo *const np = 2740 &nl->nodeinfo[nl->numnames]; 2741 2742 if (NG_NODE_NOT_VALID(node)) 2743 continue; 2744 strcpy(np->name, NG_NODE_NAME(node)); 2745 strcpy(np->type, node->nd_type->name); 2746 np->id = ng_node2ID(node); 2747 np->hooks = node->nd_numhooks; 2748 KASSERT(nl->numnames < V_ng_named_nodes, 2749 ("%s: no space", __func__)); 2750 nl->numnames++; 2751 } 2752 } 2753 NAMEHASH_RUNLOCK(); 2754 break; 2755 } 2756 2757 case NGM_LISTTYPES: 2758 { 2759 struct typelist *tl; 2760 struct ng_type *type; 2761 int num = 0; 2762 2763 TYPELIST_RLOCK(); 2764 /* Count number of types */ 2765 LIST_FOREACH(type, &ng_typelist, types) 2766 num++; 2767 2768 /* Get response struct */ 2769 NG_MKRESPONSE(resp, msg, sizeof(*tl) + 2770 (num * sizeof(struct typeinfo)), M_NOWAIT); 2771 if (resp == NULL) { 2772 TYPELIST_RUNLOCK(); 2773 error = ENOMEM; 2774 break; 2775 } 2776 tl = (struct typelist *) resp->data; 2777 2778 /* Cycle through the linked list of types */ 2779 tl->numtypes = 0; 2780 LIST_FOREACH(type, &ng_typelist, types) { 2781 struct typeinfo *const tp = &tl->typeinfo[tl->numtypes]; 2782 2783 strcpy(tp->type_name, type->name); 2784 tp->numnodes = type->refs - 1; /* don't count list */ 2785 KASSERT(tl->numtypes < num, ("%s: no space", __func__)); 2786 tl->numtypes++; 2787 } 2788 TYPELIST_RUNLOCK(); 2789 break; 2790 } 2791 2792 case NGM_BINARY2ASCII: 2793 { 2794 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2795 const struct ng_parse_type *argstype; 2796 const struct ng_cmdlist *c; 2797 struct ng_mesg *binary, *ascii; 2798 2799 /* Data area must contain a valid netgraph message */ 2800 binary = (struct ng_mesg *)msg->data; 2801 if (msg->header.arglen < sizeof(struct ng_mesg) || 2802 (msg->header.arglen - sizeof(struct ng_mesg) < 2803 binary->header.arglen)) { 2804 TRAP_ERROR(); 2805 error = EINVAL; 2806 break; 2807 } 2808 2809 /* Get a response message with lots of room */ 2810 NG_MKRESPONSE(resp, msg, sizeof(*ascii) + bufSize, M_NOWAIT); 2811 if (resp == NULL) { 2812 error = ENOMEM; 2813 break; 2814 } 2815 ascii = (struct ng_mesg *)resp->data; 2816 2817 /* Copy binary message header to response message payload */ 2818 bcopy(binary, ascii, sizeof(*binary)); 2819 2820 /* Find command by matching typecookie and command number */ 2821 for (c = here->nd_type->cmdlist; c != NULL && c->name != NULL; 2822 c++) { 2823 if (binary->header.typecookie == c->cookie && 2824 binary->header.cmd == c->cmd) 2825 break; 2826 } 2827 if (c == NULL || c->name == NULL) { 2828 for (c = ng_generic_cmds; c->name != NULL; c++) { 2829 if (binary->header.typecookie == c->cookie && 2830 binary->header.cmd == c->cmd) 2831 break; 2832 } 2833 if (c->name == NULL) { 2834 NG_FREE_MSG(resp); 2835 error = ENOSYS; 2836 break; 2837 } 2838 } 2839 2840 /* Convert command name to ASCII */ 2841 snprintf(ascii->header.cmdstr, sizeof(ascii->header.cmdstr), 2842 "%s", c->name); 2843 2844 /* Convert command arguments to ASCII */ 2845 argstype = (binary->header.flags & NGF_RESP) ? 2846 c->respType : c->mesgType; 2847 if (argstype == NULL) { 2848 *ascii->data = '\0'; 2849 } else { 2850 if ((error = ng_unparse(argstype, 2851 (u_char *)binary->data, 2852 ascii->data, bufSize)) != 0) { 2853 NG_FREE_MSG(resp); 2854 break; 2855 } 2856 } 2857 2858 /* Return the result as struct ng_mesg plus ASCII string */ 2859 bufSize = strlen(ascii->data) + 1; 2860 ascii->header.arglen = bufSize; 2861 resp->header.arglen = sizeof(*ascii) + bufSize; 2862 break; 2863 } 2864 2865 case NGM_ASCII2BINARY: 2866 { 2867 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2868 const struct ng_cmdlist *c; 2869 const struct ng_parse_type *argstype; 2870 struct ng_mesg *ascii, *binary; 2871 int off = 0; 2872 2873 /* Data area must contain at least a struct ng_mesg + '\0' */ 2874 ascii = (struct ng_mesg *)msg->data; 2875 if ((msg->header.arglen < sizeof(*ascii) + 1) || 2876 (ascii->header.arglen < 1) || 2877 (msg->header.arglen < sizeof(*ascii) + 2878 ascii->header.arglen)) { 2879 TRAP_ERROR(); 2880 error = EINVAL; 2881 break; 2882 } 2883 ascii->data[ascii->header.arglen - 1] = '\0'; 2884 2885 /* Get a response message with lots of room */ 2886 NG_MKRESPONSE(resp, msg, sizeof(*binary) + bufSize, M_NOWAIT); 2887 if (resp == NULL) { 2888 error = ENOMEM; 2889 break; 2890 } 2891 binary = (struct ng_mesg *)resp->data; 2892 2893 /* Copy ASCII message header to response message payload */ 2894 bcopy(ascii, binary, sizeof(*ascii)); 2895 2896 /* Find command by matching ASCII command string */ 2897 for (c = here->nd_type->cmdlist; 2898 c != NULL && c->name != NULL; c++) { 2899 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2900 break; 2901 } 2902 if (c == NULL || c->name == NULL) { 2903 for (c = ng_generic_cmds; c->name != NULL; c++) { 2904 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2905 break; 2906 } 2907 if (c->name == NULL) { 2908 NG_FREE_MSG(resp); 2909 error = ENOSYS; 2910 break; 2911 } 2912 } 2913 2914 /* Convert command name to binary */ 2915 binary->header.cmd = c->cmd; 2916 binary->header.typecookie = c->cookie; 2917 2918 /* Convert command arguments to binary */ 2919 argstype = (binary->header.flags & NGF_RESP) ? 2920 c->respType : c->mesgType; 2921 if (argstype == NULL) { 2922 bufSize = 0; 2923 } else { 2924 if ((error = ng_parse(argstype, ascii->data, &off, 2925 (u_char *)binary->data, &bufSize)) != 0) { 2926 NG_FREE_MSG(resp); 2927 break; 2928 } 2929 } 2930 2931 /* Return the result */ 2932 binary->header.arglen = bufSize; 2933 resp->header.arglen = sizeof(*binary) + bufSize; 2934 break; 2935 } 2936 2937 case NGM_TEXT_CONFIG: 2938 case NGM_TEXT_STATUS: 2939 /* 2940 * This one is tricky as it passes the command down to the 2941 * actual node, even though it is a generic type command. 2942 * This means we must assume that the item/msg is already freed 2943 * when control passes back to us. 2944 */ 2945 if (here->nd_type->rcvmsg != NULL) { 2946 NGI_MSG(item) = msg; /* put it back as we found it */ 2947 return((*here->nd_type->rcvmsg)(here, item, lasthook)); 2948 } 2949 /* Fall through if rcvmsg not supported */ 2950 default: 2951 TRAP_ERROR(); 2952 error = EINVAL; 2953 } 2954 /* 2955 * Sometimes a generic message may be statically allocated 2956 * to avoid problems with allocating when in tight memory situations. 2957 * Don't free it if it is so. 2958 * I break them apart here, because erros may cause a free if the item 2959 * in which case we'd be doing it twice. 2960 * they are kept together above, to simplify freeing. 2961 */ 2962 out: 2963 NG_RESPOND_MSG(error, here, item, resp); 2964 NG_FREE_MSG(msg); 2965 return (error); 2966 } 2967 2968 /************************************************************************ 2969 Queue element get/free routines 2970 ************************************************************************/ 2971 2972 uma_zone_t ng_qzone; 2973 uma_zone_t ng_qdzone; 2974 #ifndef FSTACK 2975 static int numthreads = 0; /* number of queue threads */ 2976 #endif 2977 static int maxalloc = 4096;/* limit the damage of a leak */ 2978 static int maxdata = 4096; /* limit the damage of a DoS */ 2979 2980 #ifndef FSTACK 2981 SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads, 2982 0, "Number of queue processing threads"); 2983 #endif 2984 SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc, 2985 0, "Maximum number of non-data queue items to allocate"); 2986 SYSCTL_INT(_net_graph, OID_AUTO, maxdata, CTLFLAG_RDTUN, &maxdata, 2987 0, "Maximum number of data queue items to allocate"); 2988 2989 #ifdef NETGRAPH_DEBUG 2990 static TAILQ_HEAD(, ng_item) ng_itemlist = TAILQ_HEAD_INITIALIZER(ng_itemlist); 2991 static int allocated; /* number of items malloc'd */ 2992 #endif 2993 2994 /* 2995 * Get a queue entry. 2996 * This is usually called when a packet first enters netgraph. 2997 * By definition, this is usually from an interrupt, or from a user. 2998 * Users are not so important, but try be quick for the times that it's 2999 * an interrupt. 3000 */ 3001 static __inline item_p 3002 ng_alloc_item(int type, int flags) 3003 { 3004 item_p item; 3005 3006 KASSERT(((type & ~NGQF_TYPE) == 0), 3007 ("%s: incorrect item type: %d", __func__, type)); 3008 3009 item = uma_zalloc((type == NGQF_DATA) ? ng_qdzone : ng_qzone, 3010 ((flags & NG_WAITOK) ? M_WAITOK : M_NOWAIT) | M_ZERO); 3011 3012 if (item) { 3013 item->el_flags = type; 3014 #ifdef NETGRAPH_DEBUG 3015 mtx_lock(&ngq_mtx); 3016 TAILQ_INSERT_TAIL(&ng_itemlist, item, all); 3017 allocated++; 3018 mtx_unlock(&ngq_mtx); 3019 #endif 3020 } 3021 3022 return (item); 3023 } 3024 3025 /* 3026 * Release a queue entry 3027 */ 3028 void 3029 ng_free_item(item_p item) 3030 { 3031 /* 3032 * The item may hold resources on its own. We need to free 3033 * these before we can free the item. What they are depends upon 3034 * what kind of item it is. it is important that nodes zero 3035 * out pointers to resources that they remove from the item 3036 * or we release them again here. 3037 */ 3038 switch (item->el_flags & NGQF_TYPE) { 3039 case NGQF_DATA: 3040 /* If we have an mbuf still attached.. */ 3041 NG_FREE_M(_NGI_M(item)); 3042 break; 3043 case NGQF_MESG: 3044 _NGI_RETADDR(item) = 0; 3045 NG_FREE_MSG(_NGI_MSG(item)); 3046 break; 3047 case NGQF_FN: 3048 case NGQF_FN2: 3049 /* nothing to free really, */ 3050 _NGI_FN(item) = NULL; 3051 _NGI_ARG1(item) = NULL; 3052 _NGI_ARG2(item) = 0; 3053 break; 3054 } 3055 /* If we still have a node or hook referenced... */ 3056 _NGI_CLR_NODE(item); 3057 _NGI_CLR_HOOK(item); 3058 3059 #ifdef NETGRAPH_DEBUG 3060 mtx_lock(&ngq_mtx); 3061 TAILQ_REMOVE(&ng_itemlist, item, all); 3062 allocated--; 3063 mtx_unlock(&ngq_mtx); 3064 #endif 3065 uma_zfree(((item->el_flags & NGQF_TYPE) == NGQF_DATA) ? 3066 ng_qdzone : ng_qzone, item); 3067 } 3068 3069 /* 3070 * Change type of the queue entry. 3071 * Possibly reallocates it from another UMA zone. 3072 */ 3073 static __inline item_p 3074 ng_realloc_item(item_p pitem, int type, int flags) 3075 { 3076 item_p item; 3077 int from, to; 3078 3079 KASSERT((pitem != NULL), ("%s: can't reallocate NULL", __func__)); 3080 KASSERT(((type & ~NGQF_TYPE) == 0), 3081 ("%s: incorrect item type: %d", __func__, type)); 3082 3083 from = ((pitem->el_flags & NGQF_TYPE) == NGQF_DATA); 3084 to = (type == NGQF_DATA); 3085 if (from != to) { 3086 /* If reallocation is required do it and copy item. */ 3087 if ((item = ng_alloc_item(type, flags)) == NULL) { 3088 ng_free_item(pitem); 3089 return (NULL); 3090 } 3091 *item = *pitem; 3092 ng_free_item(pitem); 3093 } else 3094 item = pitem; 3095 item->el_flags = (item->el_flags & ~NGQF_TYPE) | type; 3096 3097 return (item); 3098 } 3099 3100 /************************************************************************ 3101 Module routines 3102 ************************************************************************/ 3103 3104 /* 3105 * Handle the loading/unloading of a netgraph node type module 3106 */ 3107 int 3108 ng_mod_event(module_t mod, int event, void *data) 3109 { 3110 struct ng_type *const type = data; 3111 int error = 0; 3112 3113 switch (event) { 3114 case MOD_LOAD: 3115 3116 /* Register new netgraph node type */ 3117 if ((error = ng_newtype(type)) != 0) 3118 break; 3119 3120 /* Call type specific code */ 3121 if (type->mod_event != NULL) 3122 if ((error = (*type->mod_event)(mod, event, data))) { 3123 TYPELIST_WLOCK(); 3124 type->refs--; /* undo it */ 3125 LIST_REMOVE(type, types); 3126 TYPELIST_WUNLOCK(); 3127 } 3128 break; 3129 3130 case MOD_UNLOAD: 3131 if (type->refs > 1) { /* make sure no nodes exist! */ 3132 error = EBUSY; 3133 } else { 3134 if (type->refs == 0) /* failed load, nothing to undo */ 3135 break; 3136 if (type->mod_event != NULL) { /* check with type */ 3137 error = (*type->mod_event)(mod, event, data); 3138 if (error != 0) /* type refuses.. */ 3139 break; 3140 } 3141 TYPELIST_WLOCK(); 3142 LIST_REMOVE(type, types); 3143 TYPELIST_WUNLOCK(); 3144 } 3145 break; 3146 3147 default: 3148 if (type->mod_event != NULL) 3149 error = (*type->mod_event)(mod, event, data); 3150 else 3151 error = EOPNOTSUPP; /* XXX ? */ 3152 break; 3153 } 3154 return (error); 3155 } 3156 3157 static void 3158 vnet_netgraph_init(const void *unused __unused) 3159 { 3160 3161 /* We start with small hashes, but they can grow. */ 3162 V_ng_ID_hash = hashinit(16, M_NETGRAPH_NODE, &V_ng_ID_hmask); 3163 V_ng_name_hash = hashinit(16, M_NETGRAPH_NODE, &V_ng_name_hmask); 3164 } 3165 VNET_SYSINIT(vnet_netgraph_init, SI_SUB_NETGRAPH, SI_ORDER_FIRST, 3166 vnet_netgraph_init, NULL); 3167 3168 #ifdef VIMAGE 3169 static void 3170 vnet_netgraph_uninit(const void *unused __unused) 3171 { 3172 node_p node = NULL, last_killed = NULL; 3173 int i; 3174 3175 do { 3176 /* Find a node to kill */ 3177 IDHASH_RLOCK(); 3178 for (i = 0; i <= V_ng_ID_hmask; i++) { 3179 LIST_FOREACH(node, &V_ng_ID_hash[i], nd_idnodes) { 3180 if (node != &ng_deadnode) { 3181 NG_NODE_REF(node); 3182 break; 3183 } 3184 } 3185 if (node != NULL) 3186 break; 3187 } 3188 IDHASH_RUNLOCK(); 3189 3190 /* Attempt to kill it only if it is a regular node */ 3191 if (node != NULL) { 3192 if (node == last_killed) { 3193 if (node->nd_flags & NGF_REALLY_DIE) 3194 panic("ng node %s won't die", 3195 node->nd_name); 3196 /* The node persisted itself. Try again. */ 3197 node->nd_flags |= NGF_REALLY_DIE; 3198 } 3199 ng_rmnode(node, NULL, NULL, 0); 3200 NG_NODE_UNREF(node); 3201 last_killed = node; 3202 } 3203 } while (node != NULL); 3204 3205 hashdestroy(V_ng_name_hash, M_NETGRAPH_NODE, V_ng_name_hmask); 3206 hashdestroy(V_ng_ID_hash, M_NETGRAPH_NODE, V_ng_ID_hmask); 3207 } 3208 VNET_SYSUNINIT(vnet_netgraph_uninit, SI_SUB_NETGRAPH, SI_ORDER_FIRST, 3209 vnet_netgraph_uninit, NULL); 3210 #endif /* VIMAGE */ 3211 3212 /* 3213 * Handle loading and unloading for this code. 3214 * The only thing we need to link into is the NETISR strucure. 3215 */ 3216 static int 3217 ngb_mod_event(module_t mod, int event, void *data) 3218 { 3219 #ifndef FSTACK 3220 struct proc *p; 3221 struct thread *td; 3222 #endif 3223 int i, error = 0; 3224 3225 switch (event) { 3226 case MOD_LOAD: 3227 /* Initialize everything. */ 3228 NG_WORKLIST_LOCK_INIT(); 3229 rw_init(&ng_typelist_lock, "netgraph types"); 3230 rw_init(&ng_idhash_lock, "netgraph idhash"); 3231 rw_init(&ng_namehash_lock, "netgraph namehash"); 3232 rw_init(&ng_topo_lock, "netgraph topology mutex"); 3233 #ifdef NETGRAPH_DEBUG 3234 mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", NULL, 3235 MTX_DEF); 3236 mtx_init(&ngq_mtx, "netgraph item list mutex", NULL, 3237 MTX_DEF); 3238 #endif 3239 ng_qzone = uma_zcreate("NetGraph items", sizeof(struct ng_item), 3240 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 3241 uma_zone_set_max(ng_qzone, maxalloc); 3242 ng_qdzone = uma_zcreate("NetGraph data items", 3243 sizeof(struct ng_item), NULL, NULL, NULL, NULL, 3244 UMA_ALIGN_CACHE, 0); 3245 uma_zone_set_max(ng_qdzone, maxdata); 3246 #ifndef FSTACK 3247 /* Autoconfigure number of threads. */ 3248 if (numthreads <= 0) 3249 numthreads = mp_ncpus; 3250 /* Create threads. */ 3251 p = NULL; /* start with no process */ 3252 for (i = 0; i < numthreads; i++) { 3253 if (kproc_kthread_add(ngthread, NULL, &p, &td, 3254 RFHIGHPID, 0, "ng_queue", "ng_queue%d", i)) { 3255 numthreads = i; 3256 break; 3257 } 3258 } 3259 #endif 3260 break; 3261 case MOD_UNLOAD: 3262 /* You can't unload it because an interface may be using it. */ 3263 error = EBUSY; 3264 break; 3265 default: 3266 error = EOPNOTSUPP; 3267 break; 3268 } 3269 return (error); 3270 } 3271 3272 static moduledata_t netgraph_mod = { 3273 "netgraph", 3274 ngb_mod_event, 3275 (NULL) 3276 }; 3277 DECLARE_MODULE(netgraph, netgraph_mod, SI_SUB_NETGRAPH, SI_ORDER_FIRST); 3278 SYSCTL_NODE(_net, OID_AUTO, graph, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 3279 "netgraph Family"); 3280 SYSCTL_INT(_net_graph, OID_AUTO, abi_version, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, NG_ABI_VERSION,""); 3281 SYSCTL_INT(_net_graph, OID_AUTO, msg_version, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, NG_VERSION, ""); 3282 3283 #ifdef NETGRAPH_DEBUG 3284 void 3285 dumphook (hook_p hook, char *file, int line) 3286 { 3287 printf("hook: name %s, %d refs, Last touched:\n", 3288 _NG_HOOK_NAME(hook), hook->hk_refs); 3289 printf(" Last active @ %s, line %d\n", 3290 hook->lastfile, hook->lastline); 3291 if (line) { 3292 printf(" problem discovered at file %s, line %d\n", file, line); 3293 #ifdef KDB 3294 kdb_backtrace(); 3295 #endif 3296 } 3297 } 3298 3299 void 3300 dumpnode(node_p node, char *file, int line) 3301 { 3302 printf("node: ID [%x]: type '%s', %d hooks, flags 0x%x, %d refs, %s:\n", 3303 _NG_NODE_ID(node), node->nd_type->name, 3304 node->nd_numhooks, node->nd_flags, 3305 node->nd_refs, node->nd_name); 3306 printf(" Last active @ %s, line %d\n", 3307 node->lastfile, node->lastline); 3308 if (line) { 3309 printf(" problem discovered at file %s, line %d\n", file, line); 3310 #ifdef KDB 3311 kdb_backtrace(); 3312 #endif 3313 } 3314 } 3315 3316 void 3317 dumpitem(item_p item, char *file, int line) 3318 { 3319 printf(" ACTIVE item, last used at %s, line %d", 3320 item->lastfile, item->lastline); 3321 switch(item->el_flags & NGQF_TYPE) { 3322 case NGQF_DATA: 3323 printf(" - [data]\n"); 3324 break; 3325 case NGQF_MESG: 3326 printf(" - retaddr[%d]:\n", _NGI_RETADDR(item)); 3327 break; 3328 case NGQF_FN: 3329 printf(" - fn@%p (%p, %p, %p, %d (%x))\n", 3330 _NGI_FN(item), 3331 _NGI_NODE(item), 3332 _NGI_HOOK(item), 3333 item->body.fn.fn_arg1, 3334 item->body.fn.fn_arg2, 3335 item->body.fn.fn_arg2); 3336 break; 3337 case NGQF_FN2: 3338 printf(" - fn2@%p (%p, %p, %p, %d (%x))\n", 3339 _NGI_FN2(item), 3340 _NGI_NODE(item), 3341 _NGI_HOOK(item), 3342 item->body.fn.fn_arg1, 3343 item->body.fn.fn_arg2, 3344 item->body.fn.fn_arg2); 3345 break; 3346 } 3347 if (line) { 3348 printf(" problem discovered at file %s, line %d\n", file, line); 3349 if (_NGI_NODE(item)) { 3350 printf("node %p ([%x])\n", 3351 _NGI_NODE(item), ng_node2ID(_NGI_NODE(item))); 3352 } 3353 } 3354 } 3355 3356 static void 3357 ng_dumpitems(void) 3358 { 3359 item_p item; 3360 int i = 1; 3361 TAILQ_FOREACH(item, &ng_itemlist, all) { 3362 printf("[%d] ", i++); 3363 dumpitem(item, NULL, 0); 3364 } 3365 } 3366 3367 static void 3368 ng_dumpnodes(void) 3369 { 3370 node_p node; 3371 int i = 1; 3372 mtx_lock(&ng_nodelist_mtx); 3373 SLIST_FOREACH(node, &ng_allnodes, nd_all) { 3374 printf("[%d] ", i++); 3375 dumpnode(node, NULL, 0); 3376 } 3377 mtx_unlock(&ng_nodelist_mtx); 3378 } 3379 3380 static void 3381 ng_dumphooks(void) 3382 { 3383 hook_p hook; 3384 int i = 1; 3385 mtx_lock(&ng_nodelist_mtx); 3386 SLIST_FOREACH(hook, &ng_allhooks, hk_all) { 3387 printf("[%d] ", i++); 3388 dumphook(hook, NULL, 0); 3389 } 3390 mtx_unlock(&ng_nodelist_mtx); 3391 } 3392 3393 static int 3394 sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS) 3395 { 3396 int error; 3397 int val; 3398 int i; 3399 3400 val = allocated; 3401 i = 1; 3402 error = sysctl_handle_int(oidp, &val, 0, req); 3403 if (error != 0 || req->newptr == NULL) 3404 return (error); 3405 if (val == 42) { 3406 ng_dumpitems(); 3407 ng_dumpnodes(); 3408 ng_dumphooks(); 3409 } 3410 return (0); 3411 } 3412 3413 SYSCTL_PROC(_debug, OID_AUTO, ng_dump_items, 3414 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, sizeof(int), 3415 sysctl_debug_ng_dump_items, "I", 3416 "Number of allocated items"); 3417 #endif /* NETGRAPH_DEBUG */ 3418 3419 #ifndef FSTACK 3420 /*********************************************************************** 3421 * Worklist routines 3422 **********************************************************************/ 3423 /* 3424 * Pick a node off the list of nodes with work, 3425 * try get an item to process off it. Remove the node from the list. 3426 */ 3427 static void 3428 ngthread(void *arg) 3429 { 3430 for (;;) { 3431 struct epoch_tracker et; 3432 node_p node; 3433 3434 /* Get node from the worklist. */ 3435 NG_WORKLIST_LOCK(); 3436 while ((node = STAILQ_FIRST(&ng_worklist)) == NULL) 3437 NG_WORKLIST_SLEEP(); 3438 STAILQ_REMOVE_HEAD(&ng_worklist, nd_input_queue.q_work); 3439 NG_WORKLIST_UNLOCK(); 3440 CURVNET_SET(node->nd_vnet); 3441 CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist", 3442 __func__, node->nd_ID, node); 3443 /* 3444 * We have the node. We also take over the reference 3445 * that the list had on it. 3446 * Now process as much as you can, until it won't 3447 * let you have another item off the queue. 3448 * All this time, keep the reference 3449 * that lets us be sure that the node still exists. 3450 * Let the reference go at the last minute. 3451 */ 3452 NET_EPOCH_ENTER(et); 3453 for (;;) { 3454 item_p item; 3455 int rw; 3456 3457 NG_QUEUE_LOCK(&node->nd_input_queue); 3458 item = ng_dequeue(node, &rw); 3459 if (item == NULL) { 3460 node->nd_input_queue.q_flags2 &= ~NGQ2_WORKQ; 3461 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3462 break; /* go look for another node */ 3463 } else { 3464 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3465 NGI_GET_NODE(item, node); /* zaps stored node */ 3466 ng_apply_item(node, item, rw); 3467 NG_NODE_UNREF(node); 3468 } 3469 } 3470 NET_EPOCH_EXIT(et); 3471 NG_NODE_UNREF(node); 3472 CURVNET_RESTORE(); 3473 } 3474 } 3475 3476 /* 3477 * XXX 3478 * It's posible that a debugging NG_NODE_REF may need 3479 * to be outside the mutex zone 3480 */ 3481 static void 3482 ng_worklist_add(node_p node) 3483 { 3484 3485 mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED); 3486 3487 if ((node->nd_input_queue.q_flags2 & NGQ2_WORKQ) == 0) { 3488 /* 3489 * If we are not already on the work queue, 3490 * then put us on. 3491 */ 3492 node->nd_input_queue.q_flags2 |= NGQ2_WORKQ; 3493 NG_NODE_REF(node); /* XXX safe in mutex? */ 3494 NG_WORKLIST_LOCK(); 3495 STAILQ_INSERT_TAIL(&ng_worklist, node, nd_input_queue.q_work); 3496 NG_WORKLIST_UNLOCK(); 3497 CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__, 3498 node->nd_ID, node); 3499 NG_WORKLIST_WAKEUP(); 3500 } else { 3501 CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist", 3502 __func__, node->nd_ID, node); 3503 } 3504 } 3505 #endif 3506 3507 /*********************************************************************** 3508 * Externally useable functions to set up a queue item ready for sending 3509 ***********************************************************************/ 3510 3511 #ifdef NETGRAPH_DEBUG 3512 #define ITEM_DEBUG_CHECKS \ 3513 do { \ 3514 if (NGI_NODE(item) ) { \ 3515 printf("item already has node"); \ 3516 kdb_enter(KDB_WHY_NETGRAPH, "has node"); \ 3517 NGI_CLR_NODE(item); \ 3518 } \ 3519 if (NGI_HOOK(item) ) { \ 3520 printf("item already has hook"); \ 3521 kdb_enter(KDB_WHY_NETGRAPH, "has hook"); \ 3522 NGI_CLR_HOOK(item); \ 3523 } \ 3524 } while (0) 3525 #else 3526 #define ITEM_DEBUG_CHECKS 3527 #endif 3528 3529 /* 3530 * Put mbuf into the item. 3531 * Hook and node references will be removed when the item is dequeued. 3532 * (or equivalent) 3533 * (XXX) Unsafe because no reference held by peer on remote node. 3534 * remote node might go away in this timescale. 3535 * We know the hooks can't go away because that would require getting 3536 * a writer item on both nodes and we must have at least a reader 3537 * here to be able to do this. 3538 * Note that the hook loaded is the REMOTE hook. 3539 * 3540 * This is possibly in the critical path for new data. 3541 */ 3542 item_p 3543 ng_package_data(struct mbuf *m, int flags) 3544 { 3545 item_p item; 3546 3547 if ((item = ng_alloc_item(NGQF_DATA, flags)) == NULL) { 3548 NG_FREE_M(m); 3549 return (NULL); 3550 } 3551 ITEM_DEBUG_CHECKS; 3552 item->el_flags |= NGQF_READER; 3553 NGI_M(item) = m; 3554 return (item); 3555 } 3556 3557 /* 3558 * Allocate a queue item and put items into it.. 3559 * Evaluate the address as this will be needed to queue it and 3560 * to work out what some of the fields should be. 3561 * Hook and node references will be removed when the item is dequeued. 3562 * (or equivalent) 3563 */ 3564 item_p 3565 ng_package_msg(struct ng_mesg *msg, int flags) 3566 { 3567 item_p item; 3568 3569 if ((item = ng_alloc_item(NGQF_MESG, flags)) == NULL) { 3570 NG_FREE_MSG(msg); 3571 return (NULL); 3572 } 3573 ITEM_DEBUG_CHECKS; 3574 /* Messages items count as writers unless explicitly exempted. */ 3575 if (msg->header.cmd & NGM_READONLY) 3576 item->el_flags |= NGQF_READER; 3577 else 3578 item->el_flags |= NGQF_WRITER; 3579 /* 3580 * Set the current lasthook into the queue item 3581 */ 3582 NGI_MSG(item) = msg; 3583 NGI_RETADDR(item) = 0; 3584 return (item); 3585 } 3586 3587 #define SET_RETADDR(item, here, retaddr) \ 3588 do { /* Data or fn items don't have retaddrs */ \ 3589 if ((item->el_flags & NGQF_TYPE) == NGQF_MESG) { \ 3590 if (retaddr) { \ 3591 NGI_RETADDR(item) = retaddr; \ 3592 } else { \ 3593 /* \ 3594 * The old return address should be ok. \ 3595 * If there isn't one, use the address \ 3596 * here. \ 3597 */ \ 3598 if (NGI_RETADDR(item) == 0) { \ 3599 NGI_RETADDR(item) \ 3600 = ng_node2ID(here); \ 3601 } \ 3602 } \ 3603 } \ 3604 } while (0) 3605 3606 int 3607 ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr) 3608 { 3609 hook_p peer; 3610 node_p peernode; 3611 ITEM_DEBUG_CHECKS; 3612 /* 3613 * Quick sanity check.. 3614 * Since a hook holds a reference on its node, once we know 3615 * that the peer is still connected (even if invalid,) we know 3616 * that the peer node is present, though maybe invalid. 3617 */ 3618 TOPOLOGY_RLOCK(); 3619 if ((hook == NULL) || NG_HOOK_NOT_VALID(hook) || 3620 NG_HOOK_NOT_VALID(peer = NG_HOOK_PEER(hook)) || 3621 NG_NODE_NOT_VALID(peernode = NG_PEER_NODE(hook))) { 3622 NG_FREE_ITEM(item); 3623 TRAP_ERROR(); 3624 TOPOLOGY_RUNLOCK(); 3625 return (ENETDOWN); 3626 } 3627 3628 /* 3629 * Transfer our interest to the other (peer) end. 3630 */ 3631 NG_HOOK_REF(peer); 3632 NG_NODE_REF(peernode); 3633 NGI_SET_HOOK(item, peer); 3634 NGI_SET_NODE(item, peernode); 3635 SET_RETADDR(item, here, retaddr); 3636 3637 TOPOLOGY_RUNLOCK(); 3638 3639 return (0); 3640 } 3641 3642 int 3643 ng_address_path(node_p here, item_p item, const char *address, ng_ID_t retaddr) 3644 { 3645 node_p dest = NULL; 3646 hook_p hook = NULL; 3647 int error; 3648 3649 ITEM_DEBUG_CHECKS; 3650 /* 3651 * Note that ng_path2noderef increments the reference count 3652 * on the node for us if it finds one. So we don't have to. 3653 */ 3654 error = ng_path2noderef(here, address, &dest, &hook); 3655 if (error) { 3656 NG_FREE_ITEM(item); 3657 return (error); 3658 } 3659 NGI_SET_NODE(item, dest); 3660 if (hook) 3661 NGI_SET_HOOK(item, hook); 3662 3663 SET_RETADDR(item, here, retaddr); 3664 return (0); 3665 } 3666 3667 int 3668 ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr) 3669 { 3670 node_p dest; 3671 3672 ITEM_DEBUG_CHECKS; 3673 /* 3674 * Find the target node. 3675 */ 3676 dest = ng_ID2noderef(ID); /* GETS REFERENCE! */ 3677 if (dest == NULL) { 3678 NG_FREE_ITEM(item); 3679 TRAP_ERROR(); 3680 return(EINVAL); 3681 } 3682 /* Fill out the contents */ 3683 NGI_SET_NODE(item, dest); 3684 NGI_CLR_HOOK(item); 3685 SET_RETADDR(item, here, retaddr); 3686 return (0); 3687 } 3688 3689 /* 3690 * special case to send a message to self (e.g. destroy node) 3691 * Possibly indicate an arrival hook too. 3692 * Useful for removing that hook :-) 3693 */ 3694 item_p 3695 ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg) 3696 { 3697 item_p item; 3698 3699 /* 3700 * Find the target node. 3701 * If there is a HOOK argument, then use that in preference 3702 * to the address. 3703 */ 3704 if ((item = ng_alloc_item(NGQF_MESG, NG_NOFLAGS)) == NULL) { 3705 NG_FREE_MSG(msg); 3706 return (NULL); 3707 } 3708 3709 /* Fill out the contents */ 3710 item->el_flags |= NGQF_WRITER; 3711 NG_NODE_REF(here); 3712 NGI_SET_NODE(item, here); 3713 if (hook) { 3714 NG_HOOK_REF(hook); 3715 NGI_SET_HOOK(item, hook); 3716 } 3717 NGI_MSG(item) = msg; 3718 NGI_RETADDR(item) = ng_node2ID(here); 3719 return (item); 3720 } 3721 3722 /* 3723 * Send ng_item_fn function call to the specified node. 3724 */ 3725 3726 int 3727 ng_send_fn(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2) 3728 { 3729 3730 return ng_send_fn1(node, hook, fn, arg1, arg2, NG_NOFLAGS); 3731 } 3732 3733 int 3734 ng_send_fn1(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2, 3735 int flags) 3736 { 3737 item_p item; 3738 3739 if ((item = ng_alloc_item(NGQF_FN, flags)) == NULL) { 3740 return (ENOMEM); 3741 } 3742 item->el_flags |= NGQF_WRITER; 3743 NG_NODE_REF(node); /* and one for the item */ 3744 NGI_SET_NODE(item, node); 3745 if (hook) { 3746 NG_HOOK_REF(hook); 3747 NGI_SET_HOOK(item, hook); 3748 } 3749 NGI_FN(item) = fn; 3750 NGI_ARG1(item) = arg1; 3751 NGI_ARG2(item) = arg2; 3752 return(ng_snd_item(item, flags)); 3753 } 3754 3755 /* 3756 * Send ng_item_fn2 function call to the specified node. 3757 * 3758 * If an optional pitem parameter is supplied, its apply 3759 * callback will be copied to the new item. If also NG_REUSE_ITEM 3760 * flag is set, no new item will be allocated, but pitem will 3761 * be used. 3762 */ 3763 int 3764 ng_send_fn2(node_p node, hook_p hook, item_p pitem, ng_item_fn2 *fn, void *arg1, 3765 int arg2, int flags) 3766 { 3767 item_p item; 3768 3769 KASSERT((pitem != NULL || (flags & NG_REUSE_ITEM) == 0), 3770 ("%s: NG_REUSE_ITEM but no pitem", __func__)); 3771 3772 /* 3773 * Allocate a new item if no supplied or 3774 * if we can't use supplied one. 3775 */ 3776 if (pitem == NULL || (flags & NG_REUSE_ITEM) == 0) { 3777 if ((item = ng_alloc_item(NGQF_FN2, flags)) == NULL) 3778 return (ENOMEM); 3779 if (pitem != NULL) 3780 item->apply = pitem->apply; 3781 } else { 3782 if ((item = ng_realloc_item(pitem, NGQF_FN2, flags)) == NULL) 3783 return (ENOMEM); 3784 } 3785 3786 item->el_flags = (item->el_flags & ~NGQF_RW) | NGQF_WRITER; 3787 NG_NODE_REF(node); /* and one for the item */ 3788 NGI_SET_NODE(item, node); 3789 if (hook) { 3790 NG_HOOK_REF(hook); 3791 NGI_SET_HOOK(item, hook); 3792 } 3793 NGI_FN2(item) = fn; 3794 NGI_ARG1(item) = arg1; 3795 NGI_ARG2(item) = arg2; 3796 return(ng_snd_item(item, flags)); 3797 } 3798 3799 /* 3800 * Official timeout routines for Netgraph nodes. 3801 */ 3802 static void 3803 ng_callout_trampoline(void *arg) 3804 { 3805 struct epoch_tracker et; 3806 item_p item = arg; 3807 3808 NET_EPOCH_ENTER(et); 3809 CURVNET_SET(NGI_NODE(item)->nd_vnet); 3810 ng_snd_item(item, 0); 3811 CURVNET_RESTORE(); 3812 NET_EPOCH_EXIT(et); 3813 } 3814 3815 int 3816 ng_callout(struct callout *c, node_p node, hook_p hook, int ticks, 3817 ng_item_fn *fn, void * arg1, int arg2) 3818 { 3819 item_p item, oitem; 3820 3821 if ((item = ng_alloc_item(NGQF_FN, NG_NOFLAGS)) == NULL) 3822 return (ENOMEM); 3823 3824 item->el_flags |= NGQF_WRITER; 3825 NG_NODE_REF(node); /* and one for the item */ 3826 NGI_SET_NODE(item, node); 3827 if (hook) { 3828 NG_HOOK_REF(hook); 3829 NGI_SET_HOOK(item, hook); 3830 } 3831 NGI_FN(item) = fn; 3832 NGI_ARG1(item) = arg1; 3833 NGI_ARG2(item) = arg2; 3834 oitem = c->c_arg; 3835 if (callout_reset(c, ticks, &ng_callout_trampoline, item) == 1 && 3836 oitem != NULL) 3837 NG_FREE_ITEM(oitem); 3838 return (0); 3839 } 3840 3841 /* A special modified version of callout_stop() */ 3842 int 3843 ng_uncallout(struct callout *c, node_p node) 3844 { 3845 item_p item; 3846 int rval; 3847 3848 KASSERT(c != NULL, ("ng_uncallout: NULL callout")); 3849 KASSERT(node != NULL, ("ng_uncallout: NULL node")); 3850 3851 rval = callout_stop(c); 3852 item = c->c_arg; 3853 /* Do an extra check */ 3854 if ((rval > 0) && (c->c_func == &ng_callout_trampoline) && 3855 (item != NULL) && (NGI_NODE(item) == node)) { 3856 /* 3857 * We successfully removed it from the queue before it ran 3858 * So now we need to unreference everything that was 3859 * given extra references. (NG_FREE_ITEM does this). 3860 */ 3861 NG_FREE_ITEM(item); 3862 } 3863 c->c_arg = NULL; 3864 3865 /* 3866 * Callers only want to know if the callout was cancelled and 3867 * not draining or stopped. 3868 */ 3869 return (rval > 0); 3870 } 3871 3872 /* 3873 * Set the address, if none given, give the node here. 3874 */ 3875 void 3876 ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr) 3877 { 3878 if (retaddr) { 3879 NGI_RETADDR(item) = retaddr; 3880 } else { 3881 /* 3882 * The old return address should be ok. 3883 * If there isn't one, use the address here. 3884 */ 3885 NGI_RETADDR(item) = ng_node2ID(here); 3886 } 3887 } 3888