1 /*- 2 * Copyright (c) 1996-1999 Whistle Communications, Inc. 3 * All rights reserved. 4 * 5 * Subject to the following obligations and disclaimer of warranty, use and 6 * redistribution of this software, in source or object code forms, with or 7 * without modifications are expressly permitted by Whistle Communications; 8 * provided, however, that: 9 * 1. Any and all reproductions of the source or object code must include the 10 * copyright notice above and the following disclaimer of warranties; and 11 * 2. No rights are granted, in any manner or form, to use Whistle 12 * Communications, Inc. trademarks, including the mark "WHISTLE 13 * COMMUNICATIONS" on advertising, endorsements, or otherwise except as 14 * such appears in the above copyright notice or in the software. 15 * 16 * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND 17 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO 18 * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE, 19 * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. 21 * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY 22 * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS 23 * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE. 24 * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES 25 * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING 26 * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 27 * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR 28 * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY 32 * OF SUCH DAMAGE. 33 * 34 * Authors: Julian Elischer <[email protected]> 35 * Archie Cobbs <[email protected]> 36 * 37 * $FreeBSD$ 38 * $Whistle: ng_base.c,v 1.39 1999/01/28 23:54:53 julian Exp $ 39 */ 40 41 /* 42 * This file implements the base netgraph code. 43 */ 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/ctype.h> 48 #include <sys/hash.h> 49 #include <sys/kdb.h> 50 #include <sys/kernel.h> 51 #include <sys/kthread.h> 52 #include <sys/ktr.h> 53 #include <sys/limits.h> 54 #include <sys/lock.h> 55 #include <sys/malloc.h> 56 #include <sys/mbuf.h> 57 #include <sys/proc.h> 58 #include <sys/queue.h> 59 #include <sys/refcount.h> 60 #include <sys/rwlock.h> 61 #include <sys/smp.h> 62 #include <sys/sysctl.h> 63 #include <sys/syslog.h> 64 #include <sys/unistd.h> 65 #include <machine/cpu.h> 66 #include <vm/uma.h> 67 68 #include <net/netisr.h> 69 #include <net/vnet.h> 70 71 #include <netgraph/ng_message.h> 72 #include <netgraph/netgraph.h> 73 #include <netgraph/ng_parse.h> 74 75 MODULE_VERSION(netgraph, NG_ABI_VERSION); 76 77 /* Mutex to protect topology events. */ 78 static struct rwlock ng_topo_lock; 79 #define TOPOLOGY_RLOCK() rw_rlock(&ng_topo_lock) 80 #define TOPOLOGY_RUNLOCK() rw_runlock(&ng_topo_lock) 81 #define TOPOLOGY_WLOCK() rw_wlock(&ng_topo_lock) 82 #define TOPOLOGY_WUNLOCK() rw_wunlock(&ng_topo_lock) 83 #define TOPOLOGY_NOTOWNED() rw_assert(&ng_topo_lock, RA_UNLOCKED) 84 85 #ifdef NETGRAPH_DEBUG 86 static struct mtx ng_nodelist_mtx; /* protects global node/hook lists */ 87 static struct mtx ngq_mtx; /* protects the queue item list */ 88 89 static SLIST_HEAD(, ng_node) ng_allnodes; 90 static LIST_HEAD(, ng_node) ng_freenodes; /* in debug, we never free() them */ 91 static SLIST_HEAD(, ng_hook) ng_allhooks; 92 static LIST_HEAD(, ng_hook) ng_freehooks; /* in debug, we never free() them */ 93 94 static void ng_dumpitems(void); 95 static void ng_dumpnodes(void); 96 static void ng_dumphooks(void); 97 98 #endif /* NETGRAPH_DEBUG */ 99 /* 100 * DEAD versions of the structures. 101 * In order to avoid races, it is sometimes necessary to point 102 * at SOMETHING even though theoretically, the current entity is 103 * INVALID. Use these to avoid these races. 104 */ 105 struct ng_type ng_deadtype = { 106 NG_ABI_VERSION, 107 "dead", 108 NULL, /* modevent */ 109 NULL, /* constructor */ 110 NULL, /* rcvmsg */ 111 NULL, /* shutdown */ 112 NULL, /* newhook */ 113 NULL, /* findhook */ 114 NULL, /* connect */ 115 NULL, /* rcvdata */ 116 NULL, /* disconnect */ 117 NULL, /* cmdlist */ 118 }; 119 120 struct ng_node ng_deadnode = { 121 "dead", 122 &ng_deadtype, 123 NGF_INVALID, 124 0, /* numhooks */ 125 NULL, /* private */ 126 0, /* ID */ 127 LIST_HEAD_INITIALIZER(ng_deadnode.nd_hooks), 128 {}, /* all_nodes list entry */ 129 {}, /* id hashtable list entry */ 130 { 0, 131 0, 132 {}, /* should never use! (should hang) */ 133 {}, /* workqueue entry */ 134 STAILQ_HEAD_INITIALIZER(ng_deadnode.nd_input_queue.queue), 135 }, 136 1, /* refs */ 137 NULL, /* vnet */ 138 #ifdef NETGRAPH_DEBUG 139 ND_MAGIC, 140 __FILE__, 141 __LINE__, 142 {NULL} 143 #endif /* NETGRAPH_DEBUG */ 144 }; 145 146 struct ng_hook ng_deadhook = { 147 "dead", 148 NULL, /* private */ 149 HK_INVALID | HK_DEAD, 150 0, /* undefined data link type */ 151 &ng_deadhook, /* Peer is self */ 152 &ng_deadnode, /* attached to deadnode */ 153 {}, /* hooks list */ 154 NULL, /* override rcvmsg() */ 155 NULL, /* override rcvdata() */ 156 1, /* refs always >= 1 */ 157 #ifdef NETGRAPH_DEBUG 158 HK_MAGIC, 159 __FILE__, 160 __LINE__, 161 {NULL} 162 #endif /* NETGRAPH_DEBUG */ 163 }; 164 165 /* 166 * END DEAD STRUCTURES 167 */ 168 /* List nodes with unallocated work */ 169 static STAILQ_HEAD(, ng_node) ng_worklist = STAILQ_HEAD_INITIALIZER(ng_worklist); 170 static struct mtx ng_worklist_mtx; /* MUST LOCK NODE FIRST */ 171 172 /* List of installed types */ 173 static LIST_HEAD(, ng_type) ng_typelist; 174 static struct rwlock ng_typelist_lock; 175 #define TYPELIST_RLOCK() rw_rlock(&ng_typelist_lock) 176 #define TYPELIST_RUNLOCK() rw_runlock(&ng_typelist_lock) 177 #define TYPELIST_WLOCK() rw_wlock(&ng_typelist_lock) 178 #define TYPELIST_WUNLOCK() rw_wunlock(&ng_typelist_lock) 179 180 /* Hash related definitions. */ 181 LIST_HEAD(nodehash, ng_node); 182 static VNET_DEFINE(struct nodehash *, ng_ID_hash); 183 static VNET_DEFINE(u_long, ng_ID_hmask); 184 static VNET_DEFINE(u_long, ng_nodes); 185 static VNET_DEFINE(struct nodehash *, ng_name_hash); 186 static VNET_DEFINE(u_long, ng_name_hmask); 187 static VNET_DEFINE(u_long, ng_named_nodes); 188 #define V_ng_ID_hash VNET(ng_ID_hash) 189 #define V_ng_ID_hmask VNET(ng_ID_hmask) 190 #define V_ng_nodes VNET(ng_nodes) 191 #define V_ng_name_hash VNET(ng_name_hash) 192 #define V_ng_name_hmask VNET(ng_name_hmask) 193 #define V_ng_named_nodes VNET(ng_named_nodes) 194 195 static struct rwlock ng_idhash_lock; 196 #define IDHASH_RLOCK() rw_rlock(&ng_idhash_lock) 197 #define IDHASH_RUNLOCK() rw_runlock(&ng_idhash_lock) 198 #define IDHASH_WLOCK() rw_wlock(&ng_idhash_lock) 199 #define IDHASH_WUNLOCK() rw_wunlock(&ng_idhash_lock) 200 201 /* Method to find a node.. used twice so do it here */ 202 #define NG_IDHASH_FN(ID) ((ID) % (V_ng_ID_hmask + 1)) 203 #define NG_IDHASH_FIND(ID, node) \ 204 do { \ 205 rw_assert(&ng_idhash_lock, RA_LOCKED); \ 206 LIST_FOREACH(node, &V_ng_ID_hash[NG_IDHASH_FN(ID)], \ 207 nd_idnodes) { \ 208 if (NG_NODE_IS_VALID(node) \ 209 && (NG_NODE_ID(node) == ID)) { \ 210 break; \ 211 } \ 212 } \ 213 } while (0) 214 215 static struct rwlock ng_namehash_lock; 216 #define NAMEHASH_RLOCK() rw_rlock(&ng_namehash_lock) 217 #define NAMEHASH_RUNLOCK() rw_runlock(&ng_namehash_lock) 218 #define NAMEHASH_WLOCK() rw_wlock(&ng_namehash_lock) 219 #define NAMEHASH_WUNLOCK() rw_wunlock(&ng_namehash_lock) 220 221 /* Internal functions */ 222 static int ng_add_hook(node_p node, const char *name, hook_p * hookp); 223 static int ng_generic_msg(node_p here, item_p item, hook_p lasthook); 224 static ng_ID_t ng_decodeidname(const char *name); 225 static int ngb_mod_event(module_t mod, int event, void *data); 226 #ifndef FSTACK 227 static void ng_worklist_add(node_p node); 228 static void ngthread(void *); 229 #endif 230 static int ng_apply_item(node_p node, item_p item, int rw); 231 #ifndef FSTACK 232 static void ng_flush_input_queue(node_p node); 233 #endif 234 static node_p ng_ID2noderef(ng_ID_t ID); 235 static int ng_con_nodes(item_p item, node_p node, const char *name, 236 node_p node2, const char *name2); 237 static int ng_con_part2(node_p node, item_p item, hook_p hook); 238 static int ng_con_part3(node_p node, item_p item, hook_p hook); 239 static int ng_mkpeer(node_p node, const char *name, const char *name2, 240 char *type); 241 static void ng_name_rehash(void); 242 static void ng_ID_rehash(void); 243 244 /* Imported, these used to be externally visible, some may go back. */ 245 void ng_destroy_hook(hook_p hook); 246 int ng_path2noderef(node_p here, const char *path, 247 node_p *dest, hook_p *lasthook); 248 int ng_make_node(const char *type, node_p *nodepp); 249 int ng_path_parse(char *addr, char **node, char **path, char **hook); 250 void ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3); 251 void ng_unname(node_p node); 252 253 /* Our own netgraph malloc type */ 254 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages"); 255 MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage"); 256 static MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", 257 "netgraph hook structures"); 258 static MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", 259 "netgraph node structures"); 260 static MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", 261 "netgraph item structures"); 262 263 /* Should not be visible outside this file */ 264 265 #define _NG_ALLOC_HOOK(hook) \ 266 hook = malloc(sizeof(*hook), M_NETGRAPH_HOOK, M_NOWAIT | M_ZERO) 267 #define _NG_ALLOC_NODE(node) \ 268 node = malloc(sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO) 269 270 #define NG_QUEUE_LOCK_INIT(n) \ 271 mtx_init(&(n)->q_mtx, "ng_node", NULL, MTX_DEF) 272 #define NG_QUEUE_LOCK(n) \ 273 mtx_lock(&(n)->q_mtx) 274 #define NG_QUEUE_UNLOCK(n) \ 275 mtx_unlock(&(n)->q_mtx) 276 #define NG_WORKLIST_LOCK_INIT() \ 277 mtx_init(&ng_worklist_mtx, "ng_worklist", NULL, MTX_DEF) 278 #define NG_WORKLIST_LOCK() \ 279 mtx_lock(&ng_worklist_mtx) 280 #define NG_WORKLIST_UNLOCK() \ 281 mtx_unlock(&ng_worklist_mtx) 282 #define NG_WORKLIST_SLEEP() \ 283 mtx_sleep(&ng_worklist, &ng_worklist_mtx, PI_NET, "sleep", 0) 284 #define NG_WORKLIST_WAKEUP() \ 285 wakeup_one(&ng_worklist) 286 287 #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/ 288 /* 289 * In debug mode: 290 * In an attempt to help track reference count screwups 291 * we do not free objects back to the malloc system, but keep them 292 * in a local cache where we can examine them and keep information safely 293 * after they have been freed. 294 * We use this scheme for nodes and hooks, and to some extent for items. 295 */ 296 static __inline hook_p 297 ng_alloc_hook(void) 298 { 299 hook_p hook; 300 SLIST_ENTRY(ng_hook) temp; 301 mtx_lock(&ng_nodelist_mtx); 302 hook = LIST_FIRST(&ng_freehooks); 303 if (hook) { 304 LIST_REMOVE(hook, hk_hooks); 305 bcopy(&hook->hk_all, &temp, sizeof(temp)); 306 bzero(hook, sizeof(struct ng_hook)); 307 bcopy(&temp, &hook->hk_all, sizeof(temp)); 308 mtx_unlock(&ng_nodelist_mtx); 309 hook->hk_magic = HK_MAGIC; 310 } else { 311 mtx_unlock(&ng_nodelist_mtx); 312 _NG_ALLOC_HOOK(hook); 313 if (hook) { 314 hook->hk_magic = HK_MAGIC; 315 mtx_lock(&ng_nodelist_mtx); 316 SLIST_INSERT_HEAD(&ng_allhooks, hook, hk_all); 317 mtx_unlock(&ng_nodelist_mtx); 318 } 319 } 320 return (hook); 321 } 322 323 static __inline node_p 324 ng_alloc_node(void) 325 { 326 node_p node; 327 SLIST_ENTRY(ng_node) temp; 328 mtx_lock(&ng_nodelist_mtx); 329 node = LIST_FIRST(&ng_freenodes); 330 if (node) { 331 LIST_REMOVE(node, nd_nodes); 332 bcopy(&node->nd_all, &temp, sizeof(temp)); 333 bzero(node, sizeof(struct ng_node)); 334 bcopy(&temp, &node->nd_all, sizeof(temp)); 335 mtx_unlock(&ng_nodelist_mtx); 336 node->nd_magic = ND_MAGIC; 337 } else { 338 mtx_unlock(&ng_nodelist_mtx); 339 _NG_ALLOC_NODE(node); 340 if (node) { 341 node->nd_magic = ND_MAGIC; 342 mtx_lock(&ng_nodelist_mtx); 343 SLIST_INSERT_HEAD(&ng_allnodes, node, nd_all); 344 mtx_unlock(&ng_nodelist_mtx); 345 } 346 } 347 return (node); 348 } 349 350 #define NG_ALLOC_HOOK(hook) do { (hook) = ng_alloc_hook(); } while (0) 351 #define NG_ALLOC_NODE(node) do { (node) = ng_alloc_node(); } while (0) 352 353 #define NG_FREE_HOOK(hook) \ 354 do { \ 355 mtx_lock(&ng_nodelist_mtx); \ 356 LIST_INSERT_HEAD(&ng_freehooks, hook, hk_hooks); \ 357 hook->hk_magic = 0; \ 358 mtx_unlock(&ng_nodelist_mtx); \ 359 } while (0) 360 361 #define NG_FREE_NODE(node) \ 362 do { \ 363 mtx_lock(&ng_nodelist_mtx); \ 364 LIST_INSERT_HEAD(&ng_freenodes, node, nd_nodes); \ 365 node->nd_magic = 0; \ 366 mtx_unlock(&ng_nodelist_mtx); \ 367 } while (0) 368 369 #else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 370 371 #define NG_ALLOC_HOOK(hook) _NG_ALLOC_HOOK(hook) 372 #define NG_ALLOC_NODE(node) _NG_ALLOC_NODE(node) 373 374 #define NG_FREE_HOOK(hook) do { free((hook), M_NETGRAPH_HOOK); } while (0) 375 #define NG_FREE_NODE(node) do { free((node), M_NETGRAPH_NODE); } while (0) 376 377 #endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 378 379 /* Set this to kdb_enter("X") to catch all errors as they occur */ 380 #ifndef TRAP_ERROR 381 #define TRAP_ERROR() 382 #endif 383 384 static VNET_DEFINE(ng_ID_t, nextID) = 1; 385 #define V_nextID VNET(nextID) 386 387 #ifdef INVARIANTS 388 #define CHECK_DATA_MBUF(m) do { \ 389 struct mbuf *n; \ 390 int total; \ 391 \ 392 M_ASSERTPKTHDR(m); \ 393 for (total = 0, n = (m); n != NULL; n = n->m_next) { \ 394 total += n->m_len; \ 395 if (n->m_nextpkt != NULL) \ 396 panic("%s: m_nextpkt", __func__); \ 397 } \ 398 \ 399 if ((m)->m_pkthdr.len != total) { \ 400 panic("%s: %d != %d", \ 401 __func__, (m)->m_pkthdr.len, total); \ 402 } \ 403 } while (0) 404 #else 405 #define CHECK_DATA_MBUF(m) 406 #endif 407 408 #define ERROUT(x) do { error = (x); goto done; } while (0) 409 410 /************************************************************************ 411 Parse type definitions for generic messages 412 ************************************************************************/ 413 414 /* Handy structure parse type defining macro */ 415 #define DEFINE_PARSE_STRUCT_TYPE(lo, up, args) \ 416 static const struct ng_parse_struct_field \ 417 ng_ ## lo ## _type_fields[] = NG_GENERIC_ ## up ## _INFO args; \ 418 static const struct ng_parse_type ng_generic_ ## lo ## _type = { \ 419 &ng_parse_struct_type, \ 420 &ng_ ## lo ## _type_fields \ 421 } 422 423 DEFINE_PARSE_STRUCT_TYPE(mkpeer, MKPEER, ()); 424 DEFINE_PARSE_STRUCT_TYPE(connect, CONNECT, ()); 425 DEFINE_PARSE_STRUCT_TYPE(name, NAME, ()); 426 DEFINE_PARSE_STRUCT_TYPE(rmhook, RMHOOK, ()); 427 DEFINE_PARSE_STRUCT_TYPE(nodeinfo, NODEINFO, ()); 428 DEFINE_PARSE_STRUCT_TYPE(typeinfo, TYPEINFO, ()); 429 DEFINE_PARSE_STRUCT_TYPE(linkinfo, LINKINFO, (&ng_generic_nodeinfo_type)); 430 431 /* Get length of an array when the length is stored as a 32 bit 432 value immediately preceding the array -- as with struct namelist 433 and struct typelist. */ 434 static int 435 ng_generic_list_getLength(const struct ng_parse_type *type, 436 const u_char *start, const u_char *buf) 437 { 438 return *((const u_int32_t *)(buf - 4)); 439 } 440 441 /* Get length of the array of struct linkinfo inside a struct hooklist */ 442 static int 443 ng_generic_linkinfo_getLength(const struct ng_parse_type *type, 444 const u_char *start, const u_char *buf) 445 { 446 const struct hooklist *hl = (const struct hooklist *)start; 447 448 return hl->nodeinfo.hooks; 449 } 450 451 /* Array type for a variable length array of struct namelist */ 452 static const struct ng_parse_array_info ng_nodeinfoarray_type_info = { 453 &ng_generic_nodeinfo_type, 454 &ng_generic_list_getLength 455 }; 456 static const struct ng_parse_type ng_generic_nodeinfoarray_type = { 457 &ng_parse_array_type, 458 &ng_nodeinfoarray_type_info 459 }; 460 461 /* Array type for a variable length array of struct typelist */ 462 static const struct ng_parse_array_info ng_typeinfoarray_type_info = { 463 &ng_generic_typeinfo_type, 464 &ng_generic_list_getLength 465 }; 466 static const struct ng_parse_type ng_generic_typeinfoarray_type = { 467 &ng_parse_array_type, 468 &ng_typeinfoarray_type_info 469 }; 470 471 /* Array type for array of struct linkinfo in struct hooklist */ 472 static const struct ng_parse_array_info ng_generic_linkinfo_array_type_info = { 473 &ng_generic_linkinfo_type, 474 &ng_generic_linkinfo_getLength 475 }; 476 static const struct ng_parse_type ng_generic_linkinfo_array_type = { 477 &ng_parse_array_type, 478 &ng_generic_linkinfo_array_type_info 479 }; 480 481 DEFINE_PARSE_STRUCT_TYPE(typelist, TYPELIST, (&ng_generic_typeinfoarray_type)); 482 DEFINE_PARSE_STRUCT_TYPE(hooklist, HOOKLIST, 483 (&ng_generic_nodeinfo_type, &ng_generic_linkinfo_array_type)); 484 DEFINE_PARSE_STRUCT_TYPE(listnodes, LISTNODES, 485 (&ng_generic_nodeinfoarray_type)); 486 487 /* List of commands and how to convert arguments to/from ASCII */ 488 static const struct ng_cmdlist ng_generic_cmds[] = { 489 { 490 NGM_GENERIC_COOKIE, 491 NGM_SHUTDOWN, 492 "shutdown", 493 NULL, 494 NULL 495 }, 496 { 497 NGM_GENERIC_COOKIE, 498 NGM_MKPEER, 499 "mkpeer", 500 &ng_generic_mkpeer_type, 501 NULL 502 }, 503 { 504 NGM_GENERIC_COOKIE, 505 NGM_CONNECT, 506 "connect", 507 &ng_generic_connect_type, 508 NULL 509 }, 510 { 511 NGM_GENERIC_COOKIE, 512 NGM_NAME, 513 "name", 514 &ng_generic_name_type, 515 NULL 516 }, 517 { 518 NGM_GENERIC_COOKIE, 519 NGM_RMHOOK, 520 "rmhook", 521 &ng_generic_rmhook_type, 522 NULL 523 }, 524 { 525 NGM_GENERIC_COOKIE, 526 NGM_NODEINFO, 527 "nodeinfo", 528 NULL, 529 &ng_generic_nodeinfo_type 530 }, 531 { 532 NGM_GENERIC_COOKIE, 533 NGM_LISTHOOKS, 534 "listhooks", 535 NULL, 536 &ng_generic_hooklist_type 537 }, 538 { 539 NGM_GENERIC_COOKIE, 540 NGM_LISTNAMES, 541 "listnames", 542 NULL, 543 &ng_generic_listnodes_type /* same as NGM_LISTNODES */ 544 }, 545 { 546 NGM_GENERIC_COOKIE, 547 NGM_LISTNODES, 548 "listnodes", 549 NULL, 550 &ng_generic_listnodes_type 551 }, 552 { 553 NGM_GENERIC_COOKIE, 554 NGM_LISTTYPES, 555 "listtypes", 556 NULL, 557 &ng_generic_typelist_type 558 }, 559 { 560 NGM_GENERIC_COOKIE, 561 NGM_TEXT_CONFIG, 562 "textconfig", 563 NULL, 564 &ng_parse_string_type 565 }, 566 { 567 NGM_GENERIC_COOKIE, 568 NGM_TEXT_STATUS, 569 "textstatus", 570 NULL, 571 &ng_parse_string_type 572 }, 573 { 574 NGM_GENERIC_COOKIE, 575 NGM_ASCII2BINARY, 576 "ascii2binary", 577 &ng_parse_ng_mesg_type, 578 &ng_parse_ng_mesg_type 579 }, 580 { 581 NGM_GENERIC_COOKIE, 582 NGM_BINARY2ASCII, 583 "binary2ascii", 584 &ng_parse_ng_mesg_type, 585 &ng_parse_ng_mesg_type 586 }, 587 { 0 } 588 }; 589 590 /************************************************************************ 591 Node routines 592 ************************************************************************/ 593 594 /* 595 * Instantiate a node of the requested type 596 */ 597 int 598 ng_make_node(const char *typename, node_p *nodepp) 599 { 600 struct ng_type *type; 601 int error; 602 603 /* Check that the type makes sense */ 604 if (typename == NULL) { 605 TRAP_ERROR(); 606 return (EINVAL); 607 } 608 609 /* Locate the node type. If we fail we return. Do not try to load 610 * module. 611 */ 612 if ((type = ng_findtype(typename)) == NULL) 613 return (ENXIO); 614 615 /* 616 * If we have a constructor, then make the node and 617 * call the constructor to do type specific initialisation. 618 */ 619 if (type->constructor != NULL) { 620 if ((error = ng_make_node_common(type, nodepp)) == 0) { 621 if ((error = ((*type->constructor)(*nodepp))) != 0) { 622 NG_NODE_UNREF(*nodepp); 623 } 624 } 625 } else { 626 /* 627 * Node has no constructor. We cannot ask for one 628 * to be made. It must be brought into existence by 629 * some external agency. The external agency should 630 * call ng_make_node_common() directly to get the 631 * netgraph part initialised. 632 */ 633 TRAP_ERROR(); 634 error = EINVAL; 635 } 636 return (error); 637 } 638 639 /* 640 * Generic node creation. Called by node initialisation for externally 641 * instantiated nodes (e.g. hardware, sockets, etc ). 642 * The returned node has a reference count of 1. 643 */ 644 int 645 ng_make_node_common(struct ng_type *type, node_p *nodepp) 646 { 647 node_p node; 648 649 /* Require the node type to have been already installed */ 650 if (ng_findtype(type->name) == NULL) { 651 TRAP_ERROR(); 652 return (EINVAL); 653 } 654 655 /* Make a node and try attach it to the type */ 656 NG_ALLOC_NODE(node); 657 if (node == NULL) { 658 TRAP_ERROR(); 659 return (ENOMEM); 660 } 661 node->nd_type = type; 662 #ifdef VIMAGE 663 node->nd_vnet = curvnet; 664 #endif 665 NG_NODE_REF(node); /* note reference */ 666 type->refs++; 667 668 NG_QUEUE_LOCK_INIT(&node->nd_input_queue); 669 STAILQ_INIT(&node->nd_input_queue.queue); 670 node->nd_input_queue.q_flags = 0; 671 672 /* Initialize hook list for new node */ 673 LIST_INIT(&node->nd_hooks); 674 675 /* Get an ID and put us in the hash chain. */ 676 IDHASH_WLOCK(); 677 for (;;) { /* wrap protection, even if silly */ 678 node_p node2 = NULL; 679 node->nd_ID = V_nextID++; /* 137/sec for 1 year before wrap */ 680 681 /* Is there a problem with the new number? */ 682 NG_IDHASH_FIND(node->nd_ID, node2); /* already taken? */ 683 if ((node->nd_ID != 0) && (node2 == NULL)) { 684 break; 685 } 686 } 687 V_ng_nodes++; 688 if (V_ng_nodes * 2 > V_ng_ID_hmask) 689 ng_ID_rehash(); 690 LIST_INSERT_HEAD(&V_ng_ID_hash[NG_IDHASH_FN(node->nd_ID)], node, 691 nd_idnodes); 692 IDHASH_WUNLOCK(); 693 694 /* Done */ 695 *nodepp = node; 696 return (0); 697 } 698 699 /* 700 * Forceably start the shutdown process on a node. Either call 701 * its shutdown method, or do the default shutdown if there is 702 * no type-specific method. 703 * 704 * We can only be called from a shutdown message, so we know we have 705 * a writer lock, and therefore exclusive access. It also means 706 * that we should not be on the work queue, but we check anyhow. 707 * 708 * Persistent node types must have a type-specific method which 709 * allocates a new node in which case, this one is irretrievably going away, 710 * or cleans up anything it needs, and just makes the node valid again, 711 * in which case we allow the node to survive. 712 * 713 * XXX We need to think of how to tell a persistent node that we 714 * REALLY need to go away because the hardware has gone or we 715 * are rebooting.... etc. 716 */ 717 void 718 ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3) 719 { 720 hook_p hook; 721 722 /* Check if it's already shutting down */ 723 if ((node->nd_flags & NGF_CLOSING) != 0) 724 return; 725 726 if (node == &ng_deadnode) { 727 printf ("shutdown called on deadnode\n"); 728 return; 729 } 730 731 /* Add an extra reference so it doesn't go away during this */ 732 NG_NODE_REF(node); 733 734 /* 735 * Mark it invalid so any newcomers know not to try use it 736 * Also add our own mark so we can't recurse 737 * note that NGF_INVALID does not do this as it's also set during 738 * creation 739 */ 740 node->nd_flags |= NGF_INVALID|NGF_CLOSING; 741 742 /* If node has its pre-shutdown method, then call it first*/ 743 if (node->nd_type && node->nd_type->close) 744 (*node->nd_type->close)(node); 745 746 /* Notify all remaining connected nodes to disconnect */ 747 while ((hook = LIST_FIRST(&node->nd_hooks)) != NULL) 748 ng_destroy_hook(hook); 749 750 #ifndef FSTACK 751 /* 752 * Drain the input queue forceably. 753 * it has no hooks so what's it going to do, bleed on someone? 754 * Theoretically we came here from a queue entry that was added 755 * Just before the queue was closed, so it should be empty anyway. 756 * Also removes us from worklist if needed. 757 */ 758 ng_flush_input_queue(node); 759 #endif 760 761 /* Ask the type if it has anything to do in this case */ 762 if (node->nd_type && node->nd_type->shutdown) { 763 (*node->nd_type->shutdown)(node); 764 if (NG_NODE_IS_VALID(node)) { 765 /* 766 * Well, blow me down if the node code hasn't declared 767 * that it doesn't want to die. 768 * Presumably it is a persistent node. 769 * If we REALLY want it to go away, 770 * e.g. hardware going away, 771 * Our caller should set NGF_REALLY_DIE in nd_flags. 772 */ 773 node->nd_flags &= ~(NGF_INVALID|NGF_CLOSING); 774 NG_NODE_UNREF(node); /* Assume they still have theirs */ 775 return; 776 } 777 } else { /* do the default thing */ 778 NG_NODE_UNREF(node); 779 } 780 781 ng_unname(node); /* basically a NOP these days */ 782 783 /* 784 * Remove extra reference, possibly the last 785 * Possible other holders of references may include 786 * timeout callouts, but theoretically the node's supposed to 787 * have cancelled them. Possibly hardware dependencies may 788 * force a driver to 'linger' with a reference. 789 */ 790 NG_NODE_UNREF(node); 791 } 792 793 /* 794 * Remove a reference to the node, possibly the last. 795 * deadnode always acts as it it were the last. 796 */ 797 void 798 ng_unref_node(node_p node) 799 { 800 801 if (node == &ng_deadnode) 802 return; 803 804 CURVNET_SET(node->nd_vnet); 805 806 if (refcount_release(&node->nd_refs)) { /* we were the last */ 807 808 node->nd_type->refs--; /* XXX maybe should get types lock? */ 809 NAMEHASH_WLOCK(); 810 if (NG_NODE_HAS_NAME(node)) { 811 V_ng_named_nodes--; 812 LIST_REMOVE(node, nd_nodes); 813 } 814 NAMEHASH_WUNLOCK(); 815 816 IDHASH_WLOCK(); 817 V_ng_nodes--; 818 LIST_REMOVE(node, nd_idnodes); 819 IDHASH_WUNLOCK(); 820 821 mtx_destroy(&node->nd_input_queue.q_mtx); 822 NG_FREE_NODE(node); 823 } 824 CURVNET_RESTORE(); 825 } 826 827 /************************************************************************ 828 Node ID handling 829 ************************************************************************/ 830 static node_p 831 ng_ID2noderef(ng_ID_t ID) 832 { 833 node_p node; 834 835 IDHASH_RLOCK(); 836 NG_IDHASH_FIND(ID, node); 837 if (node) 838 NG_NODE_REF(node); 839 IDHASH_RUNLOCK(); 840 return(node); 841 } 842 843 ng_ID_t 844 ng_node2ID(node_p node) 845 { 846 return (node ? NG_NODE_ID(node) : 0); 847 } 848 849 /************************************************************************ 850 Node name handling 851 ************************************************************************/ 852 853 /* 854 * Assign a node a name. 855 */ 856 int 857 ng_name_node(node_p node, const char *name) 858 { 859 uint32_t hash; 860 node_p node2; 861 int i; 862 863 /* Check the name is valid */ 864 for (i = 0; i < NG_NODESIZ; i++) { 865 if (name[i] == '\0' || name[i] == '.' || name[i] == ':') 866 break; 867 } 868 if (i == 0 || name[i] != '\0') { 869 TRAP_ERROR(); 870 return (EINVAL); 871 } 872 if (ng_decodeidname(name) != 0) { /* valid IDs not allowed here */ 873 TRAP_ERROR(); 874 return (EINVAL); 875 } 876 877 NAMEHASH_WLOCK(); 878 if (V_ng_named_nodes * 2 > V_ng_name_hmask) 879 ng_name_rehash(); 880 881 hash = hash32_str(name, HASHINIT) & V_ng_name_hmask; 882 /* Check the name isn't already being used. */ 883 LIST_FOREACH(node2, &V_ng_name_hash[hash], nd_nodes) 884 if (NG_NODE_IS_VALID(node2) && 885 (strcmp(NG_NODE_NAME(node2), name) == 0)) { 886 NAMEHASH_WUNLOCK(); 887 return (EADDRINUSE); 888 } 889 890 if (NG_NODE_HAS_NAME(node)) 891 LIST_REMOVE(node, nd_nodes); 892 else 893 V_ng_named_nodes++; 894 /* Copy it. */ 895 strlcpy(NG_NODE_NAME(node), name, NG_NODESIZ); 896 /* Update name hash. */ 897 LIST_INSERT_HEAD(&V_ng_name_hash[hash], node, nd_nodes); 898 NAMEHASH_WUNLOCK(); 899 900 return (0); 901 } 902 903 /* 904 * Find a node by absolute name. The name should NOT end with ':' 905 * The name "." means "this node" and "[xxx]" means "the node 906 * with ID (ie, at address) xxx". 907 * 908 * Returns the node if found, else NULL. 909 * Eventually should add something faster than a sequential search. 910 * Note it acquires a reference on the node so you can be sure it's still 911 * there. 912 */ 913 node_p 914 ng_name2noderef(node_p here, const char *name) 915 { 916 node_p node; 917 ng_ID_t temp; 918 int hash; 919 920 /* "." means "this node" */ 921 if (strcmp(name, ".") == 0) { 922 NG_NODE_REF(here); 923 return(here); 924 } 925 926 /* Check for name-by-ID */ 927 if ((temp = ng_decodeidname(name)) != 0) { 928 return (ng_ID2noderef(temp)); 929 } 930 931 /* Find node by name. */ 932 hash = hash32_str(name, HASHINIT) & V_ng_name_hmask; 933 NAMEHASH_RLOCK(); 934 LIST_FOREACH(node, &V_ng_name_hash[hash], nd_nodes) 935 if (NG_NODE_IS_VALID(node) && 936 (strcmp(NG_NODE_NAME(node), name) == 0)) { 937 NG_NODE_REF(node); 938 break; 939 } 940 NAMEHASH_RUNLOCK(); 941 942 return (node); 943 } 944 945 /* 946 * Decode an ID name, eg. "[f03034de]". Returns 0 if the 947 * string is not valid, otherwise returns the value. 948 */ 949 static ng_ID_t 950 ng_decodeidname(const char *name) 951 { 952 const int len = strlen(name); 953 char *eptr; 954 u_long val; 955 956 /* Check for proper length, brackets, no leading junk */ 957 if ((len < 3) || (name[0] != '[') || (name[len - 1] != ']') || 958 (!isxdigit(name[1]))) 959 return ((ng_ID_t)0); 960 961 /* Decode number */ 962 val = strtoul(name + 1, &eptr, 16); 963 if ((eptr - name != len - 1) || (val == ULONG_MAX) || (val == 0)) 964 return ((ng_ID_t)0); 965 966 return ((ng_ID_t)val); 967 } 968 969 /* 970 * Remove a name from a node. This should only be called 971 * when shutting down and removing the node. 972 */ 973 void 974 ng_unname(node_p node) 975 { 976 } 977 978 /* 979 * Allocate a bigger name hash. 980 */ 981 static void 982 ng_name_rehash() 983 { 984 struct nodehash *new; 985 uint32_t hash; 986 u_long hmask; 987 node_p node, node2; 988 int i; 989 990 new = hashinit_flags((V_ng_name_hmask + 1) * 2, M_NETGRAPH_NODE, &hmask, 991 HASH_NOWAIT); 992 if (new == NULL) 993 return; 994 995 for (i = 0; i <= V_ng_name_hmask; i++) 996 LIST_FOREACH_SAFE(node, &V_ng_name_hash[i], nd_nodes, node2) { 997 #ifdef INVARIANTS 998 LIST_REMOVE(node, nd_nodes); 999 #endif 1000 hash = hash32_str(NG_NODE_NAME(node), HASHINIT) & hmask; 1001 LIST_INSERT_HEAD(&new[hash], node, nd_nodes); 1002 } 1003 1004 hashdestroy(V_ng_name_hash, M_NETGRAPH_NODE, V_ng_name_hmask); 1005 V_ng_name_hash = new; 1006 V_ng_name_hmask = hmask; 1007 } 1008 1009 /* 1010 * Allocate a bigger ID hash. 1011 */ 1012 static void 1013 ng_ID_rehash() 1014 { 1015 struct nodehash *new; 1016 uint32_t hash; 1017 u_long hmask; 1018 node_p node, node2; 1019 int i; 1020 1021 new = hashinit_flags((V_ng_ID_hmask + 1) * 2, M_NETGRAPH_NODE, &hmask, 1022 HASH_NOWAIT); 1023 if (new == NULL) 1024 return; 1025 1026 for (i = 0; i <= V_ng_ID_hmask; i++) 1027 LIST_FOREACH_SAFE(node, &V_ng_ID_hash[i], nd_idnodes, node2) { 1028 #ifdef INVARIANTS 1029 LIST_REMOVE(node, nd_idnodes); 1030 #endif 1031 hash = (node->nd_ID % (hmask + 1)); 1032 LIST_INSERT_HEAD(&new[hash], node, nd_idnodes); 1033 } 1034 1035 hashdestroy(V_ng_ID_hash, M_NETGRAPH_NODE, V_ng_name_hmask); 1036 V_ng_ID_hash = new; 1037 V_ng_ID_hmask = hmask; 1038 } 1039 1040 /************************************************************************ 1041 Hook routines 1042 Names are not optional. Hooks are always connected, except for a 1043 brief moment within these routines. On invalidation or during creation 1044 they are connected to the 'dead' hook. 1045 ************************************************************************/ 1046 1047 /* 1048 * Remove a hook reference 1049 */ 1050 void 1051 ng_unref_hook(hook_p hook) 1052 { 1053 1054 if (hook == &ng_deadhook) 1055 return; 1056 1057 if (refcount_release(&hook->hk_refs)) { /* we were the last */ 1058 if (_NG_HOOK_NODE(hook)) /* it'll probably be ng_deadnode */ 1059 _NG_NODE_UNREF((_NG_HOOK_NODE(hook))); 1060 NG_FREE_HOOK(hook); 1061 } 1062 } 1063 1064 /* 1065 * Add an unconnected hook to a node. Only used internally. 1066 * Assumes node is locked. (XXX not yet true ) 1067 */ 1068 static int 1069 ng_add_hook(node_p node, const char *name, hook_p *hookp) 1070 { 1071 hook_p hook; 1072 int error = 0; 1073 1074 /* Check that the given name is good */ 1075 if (name == NULL) { 1076 TRAP_ERROR(); 1077 return (EINVAL); 1078 } 1079 if (ng_findhook(node, name) != NULL) { 1080 TRAP_ERROR(); 1081 return (EEXIST); 1082 } 1083 1084 /* Allocate the hook and link it up */ 1085 NG_ALLOC_HOOK(hook); 1086 if (hook == NULL) { 1087 TRAP_ERROR(); 1088 return (ENOMEM); 1089 } 1090 hook->hk_refs = 1; /* add a reference for us to return */ 1091 hook->hk_flags = HK_INVALID; 1092 hook->hk_peer = &ng_deadhook; /* start off this way */ 1093 hook->hk_node = node; 1094 NG_NODE_REF(node); /* each hook counts as a reference */ 1095 1096 /* Set hook name */ 1097 strlcpy(NG_HOOK_NAME(hook), name, NG_HOOKSIZ); 1098 1099 /* 1100 * Check if the node type code has something to say about it 1101 * If it fails, the unref of the hook will also unref the node. 1102 */ 1103 if (node->nd_type->newhook != NULL) { 1104 if ((error = (*node->nd_type->newhook)(node, hook, name))) { 1105 NG_HOOK_UNREF(hook); /* this frees the hook */ 1106 return (error); 1107 } 1108 } 1109 /* 1110 * The 'type' agrees so far, so go ahead and link it in. 1111 * We'll ask again later when we actually connect the hooks. 1112 */ 1113 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1114 node->nd_numhooks++; 1115 NG_HOOK_REF(hook); /* one for the node */ 1116 1117 if (hookp) 1118 *hookp = hook; 1119 return (0); 1120 } 1121 1122 /* 1123 * Find a hook 1124 * 1125 * Node types may supply their own optimized routines for finding 1126 * hooks. If none is supplied, we just do a linear search. 1127 * XXX Possibly we should add a reference to the hook? 1128 */ 1129 hook_p 1130 ng_findhook(node_p node, const char *name) 1131 { 1132 hook_p hook; 1133 1134 if (node->nd_type->findhook != NULL) 1135 return (*node->nd_type->findhook)(node, name); 1136 LIST_FOREACH(hook, &node->nd_hooks, hk_hooks) { 1137 if (NG_HOOK_IS_VALID(hook) && 1138 (strcmp(NG_HOOK_NAME(hook), name) == 0)) 1139 return (hook); 1140 } 1141 return (NULL); 1142 } 1143 1144 /* 1145 * Destroy a hook 1146 * 1147 * As hooks are always attached, this really destroys two hooks. 1148 * The one given, and the one attached to it. Disconnect the hooks 1149 * from each other first. We reconnect the peer hook to the 'dead' 1150 * hook so that it can still exist after we depart. We then 1151 * send the peer its own destroy message. This ensures that we only 1152 * interact with the peer's structures when it is locked processing that 1153 * message. We hold a reference to the peer hook so we are guaranteed that 1154 * the peer hook and node are still going to exist until 1155 * we are finished there as the hook holds a ref on the node. 1156 * We run this same code again on the peer hook, but that time it is already 1157 * attached to the 'dead' hook. 1158 * 1159 * This routine is called at all stages of hook creation 1160 * on error detection and must be able to handle any such stage. 1161 */ 1162 void 1163 ng_destroy_hook(hook_p hook) 1164 { 1165 hook_p peer; 1166 node_p node; 1167 1168 if (hook == &ng_deadhook) { /* better safe than sorry */ 1169 printf("ng_destroy_hook called on deadhook\n"); 1170 return; 1171 } 1172 1173 /* 1174 * Protect divorce process with mutex, to avoid races on 1175 * simultaneous disconnect. 1176 */ 1177 TOPOLOGY_WLOCK(); 1178 1179 hook->hk_flags |= HK_INVALID; 1180 1181 peer = NG_HOOK_PEER(hook); 1182 node = NG_HOOK_NODE(hook); 1183 1184 if (peer && (peer != &ng_deadhook)) { 1185 /* 1186 * Set the peer to point to ng_deadhook 1187 * from this moment on we are effectively independent it. 1188 * send it an rmhook message of it's own. 1189 */ 1190 peer->hk_peer = &ng_deadhook; /* They no longer know us */ 1191 hook->hk_peer = &ng_deadhook; /* Nor us, them */ 1192 if (NG_HOOK_NODE(peer) == &ng_deadnode) { 1193 /* 1194 * If it's already divorced from a node, 1195 * just free it. 1196 */ 1197 TOPOLOGY_WUNLOCK(); 1198 } else { 1199 TOPOLOGY_WUNLOCK(); 1200 ng_rmhook_self(peer); /* Send it a surprise */ 1201 } 1202 NG_HOOK_UNREF(peer); /* account for peer link */ 1203 NG_HOOK_UNREF(hook); /* account for peer link */ 1204 } else 1205 TOPOLOGY_WUNLOCK(); 1206 1207 TOPOLOGY_NOTOWNED(); 1208 1209 /* 1210 * Remove the hook from the node's list to avoid possible recursion 1211 * in case the disconnection results in node shutdown. 1212 */ 1213 if (node == &ng_deadnode) { /* happens if called from ng_con_nodes() */ 1214 return; 1215 } 1216 LIST_REMOVE(hook, hk_hooks); 1217 node->nd_numhooks--; 1218 if (node->nd_type->disconnect) { 1219 /* 1220 * The type handler may elect to destroy the node so don't 1221 * trust its existence after this point. (except 1222 * that we still hold a reference on it. (which we 1223 * inherrited from the hook we are destroying) 1224 */ 1225 (*node->nd_type->disconnect) (hook); 1226 } 1227 1228 /* 1229 * Note that because we will point to ng_deadnode, the original node 1230 * is not decremented automatically so we do that manually. 1231 */ 1232 _NG_HOOK_NODE(hook) = &ng_deadnode; 1233 NG_NODE_UNREF(node); /* We no longer point to it so adjust count */ 1234 NG_HOOK_UNREF(hook); /* Account for linkage (in list) to node */ 1235 } 1236 1237 /* 1238 * Take two hooks on a node and merge the connection so that the given node 1239 * is effectively bypassed. 1240 */ 1241 int 1242 ng_bypass(hook_p hook1, hook_p hook2) 1243 { 1244 if (hook1->hk_node != hook2->hk_node) { 1245 TRAP_ERROR(); 1246 return (EINVAL); 1247 } 1248 TOPOLOGY_WLOCK(); 1249 if (NG_HOOK_NOT_VALID(hook1) || NG_HOOK_NOT_VALID(hook2)) { 1250 TOPOLOGY_WUNLOCK(); 1251 return (EINVAL); 1252 } 1253 hook1->hk_peer->hk_peer = hook2->hk_peer; 1254 hook2->hk_peer->hk_peer = hook1->hk_peer; 1255 1256 hook1->hk_peer = &ng_deadhook; 1257 hook2->hk_peer = &ng_deadhook; 1258 TOPOLOGY_WUNLOCK(); 1259 1260 NG_HOOK_UNREF(hook1); 1261 NG_HOOK_UNREF(hook2); 1262 1263 /* XXX If we ever cache methods on hooks update them as well */ 1264 ng_destroy_hook(hook1); 1265 ng_destroy_hook(hook2); 1266 return (0); 1267 } 1268 1269 /* 1270 * Install a new netgraph type 1271 */ 1272 int 1273 ng_newtype(struct ng_type *tp) 1274 { 1275 const size_t namelen = strlen(tp->name); 1276 1277 /* Check version and type name fields */ 1278 if ((tp->version != NG_ABI_VERSION) || (namelen == 0) || 1279 (namelen >= NG_TYPESIZ)) { 1280 TRAP_ERROR(); 1281 if (tp->version != NG_ABI_VERSION) { 1282 printf("Netgraph: Node type rejected. ABI mismatch. " 1283 "Suggest recompile\n"); 1284 } 1285 return (EINVAL); 1286 } 1287 1288 /* Check for name collision */ 1289 if (ng_findtype(tp->name) != NULL) { 1290 TRAP_ERROR(); 1291 return (EEXIST); 1292 } 1293 1294 /* Link in new type */ 1295 TYPELIST_WLOCK(); 1296 LIST_INSERT_HEAD(&ng_typelist, tp, types); 1297 tp->refs = 1; /* first ref is linked list */ 1298 TYPELIST_WUNLOCK(); 1299 return (0); 1300 } 1301 1302 /* 1303 * unlink a netgraph type 1304 * If no examples exist 1305 */ 1306 int 1307 ng_rmtype(struct ng_type *tp) 1308 { 1309 /* Check for name collision */ 1310 if (tp->refs != 1) { 1311 TRAP_ERROR(); 1312 return (EBUSY); 1313 } 1314 1315 /* Unlink type */ 1316 TYPELIST_WLOCK(); 1317 LIST_REMOVE(tp, types); 1318 TYPELIST_WUNLOCK(); 1319 return (0); 1320 } 1321 1322 /* 1323 * Look for a type of the name given 1324 */ 1325 struct ng_type * 1326 ng_findtype(const char *typename) 1327 { 1328 struct ng_type *type; 1329 1330 TYPELIST_RLOCK(); 1331 LIST_FOREACH(type, &ng_typelist, types) { 1332 if (strcmp(type->name, typename) == 0) 1333 break; 1334 } 1335 TYPELIST_RUNLOCK(); 1336 return (type); 1337 } 1338 1339 /************************************************************************ 1340 Composite routines 1341 ************************************************************************/ 1342 /* 1343 * Connect two nodes using the specified hooks, using queued functions. 1344 */ 1345 static int 1346 ng_con_part3(node_p node, item_p item, hook_p hook) 1347 { 1348 int error = 0; 1349 1350 /* 1351 * When we run, we know that the node 'node' is locked for us. 1352 * Our caller has a reference on the hook. 1353 * Our caller has a reference on the node. 1354 * (In this case our caller is ng_apply_item() ). 1355 * The peer hook has a reference on the hook. 1356 * We are all set up except for the final call to the node, and 1357 * the clearing of the INVALID flag. 1358 */ 1359 if (NG_HOOK_NODE(hook) == &ng_deadnode) { 1360 /* 1361 * The node must have been freed again since we last visited 1362 * here. ng_destry_hook() has this effect but nothing else does. 1363 * We should just release our references and 1364 * free anything we can think of. 1365 * Since we know it's been destroyed, and it's our caller 1366 * that holds the references, just return. 1367 */ 1368 ERROUT(ENOENT); 1369 } 1370 if (hook->hk_node->nd_type->connect) { 1371 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1372 ng_destroy_hook(hook); /* also zaps peer */ 1373 printf("failed in ng_con_part3()\n"); 1374 ERROUT(error); 1375 } 1376 } 1377 /* 1378 * XXX this is wrong for SMP. Possibly we need 1379 * to separate out 'create' and 'invalid' flags. 1380 * should only set flags on hooks we have locked under our node. 1381 */ 1382 hook->hk_flags &= ~HK_INVALID; 1383 done: 1384 NG_FREE_ITEM(item); 1385 return (error); 1386 } 1387 1388 static int 1389 ng_con_part2(node_p node, item_p item, hook_p hook) 1390 { 1391 hook_p peer; 1392 int error = 0; 1393 1394 /* 1395 * When we run, we know that the node 'node' is locked for us. 1396 * Our caller has a reference on the hook. 1397 * Our caller has a reference on the node. 1398 * (In this case our caller is ng_apply_item() ). 1399 * The peer hook has a reference on the hook. 1400 * our node pointer points to the 'dead' node. 1401 * First check the hook name is unique. 1402 * Should not happen because we checked before queueing this. 1403 */ 1404 if (ng_findhook(node, NG_HOOK_NAME(hook)) != NULL) { 1405 TRAP_ERROR(); 1406 ng_destroy_hook(hook); /* should destroy peer too */ 1407 printf("failed in ng_con_part2()\n"); 1408 ERROUT(EEXIST); 1409 } 1410 /* 1411 * Check if the node type code has something to say about it 1412 * If it fails, the unref of the hook will also unref the attached node, 1413 * however since that node is 'ng_deadnode' this will do nothing. 1414 * The peer hook will also be destroyed. 1415 */ 1416 if (node->nd_type->newhook != NULL) { 1417 if ((error = (*node->nd_type->newhook)(node, hook, 1418 hook->hk_name))) { 1419 ng_destroy_hook(hook); /* should destroy peer too */ 1420 printf("failed in ng_con_part2()\n"); 1421 ERROUT(error); 1422 } 1423 } 1424 1425 /* 1426 * The 'type' agrees so far, so go ahead and link it in. 1427 * We'll ask again later when we actually connect the hooks. 1428 */ 1429 hook->hk_node = node; /* just overwrite ng_deadnode */ 1430 NG_NODE_REF(node); /* each hook counts as a reference */ 1431 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1432 node->nd_numhooks++; 1433 NG_HOOK_REF(hook); /* one for the node */ 1434 1435 /* 1436 * We now have a symmetrical situation, where both hooks have been 1437 * linked to their nodes, the newhook methods have been called 1438 * And the references are all correct. The hooks are still marked 1439 * as invalid, as we have not called the 'connect' methods 1440 * yet. 1441 * We can call the local one immediately as we have the 1442 * node locked, but we need to queue the remote one. 1443 */ 1444 if (hook->hk_node->nd_type->connect) { 1445 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1446 ng_destroy_hook(hook); /* also zaps peer */ 1447 printf("failed in ng_con_part2(A)\n"); 1448 ERROUT(error); 1449 } 1450 } 1451 1452 /* 1453 * Acquire topo mutex to avoid race with ng_destroy_hook(). 1454 */ 1455 TOPOLOGY_RLOCK(); 1456 peer = hook->hk_peer; 1457 if (peer == &ng_deadhook) { 1458 TOPOLOGY_RUNLOCK(); 1459 printf("failed in ng_con_part2(B)\n"); 1460 ng_destroy_hook(hook); 1461 ERROUT(ENOENT); 1462 } 1463 TOPOLOGY_RUNLOCK(); 1464 1465 if ((error = ng_send_fn2(peer->hk_node, peer, item, &ng_con_part3, 1466 NULL, 0, NG_REUSE_ITEM))) { 1467 printf("failed in ng_con_part2(C)\n"); 1468 ng_destroy_hook(hook); /* also zaps peer */ 1469 return (error); /* item was consumed. */ 1470 } 1471 hook->hk_flags &= ~HK_INVALID; /* need both to be able to work */ 1472 return (0); /* item was consumed. */ 1473 done: 1474 NG_FREE_ITEM(item); 1475 return (error); 1476 } 1477 1478 /* 1479 * Connect this node with another node. We assume that this node is 1480 * currently locked, as we are only called from an NGM_CONNECT message. 1481 */ 1482 static int 1483 ng_con_nodes(item_p item, node_p node, const char *name, 1484 node_p node2, const char *name2) 1485 { 1486 int error; 1487 hook_p hook; 1488 hook_p hook2; 1489 1490 if (ng_findhook(node2, name2) != NULL) { 1491 return(EEXIST); 1492 } 1493 if ((error = ng_add_hook(node, name, &hook))) /* gives us a ref */ 1494 return (error); 1495 /* Allocate the other hook and link it up */ 1496 NG_ALLOC_HOOK(hook2); 1497 if (hook2 == NULL) { 1498 TRAP_ERROR(); 1499 ng_destroy_hook(hook); /* XXX check ref counts so far */ 1500 NG_HOOK_UNREF(hook); /* including our ref */ 1501 return (ENOMEM); 1502 } 1503 hook2->hk_refs = 1; /* start with a reference for us. */ 1504 hook2->hk_flags = HK_INVALID; 1505 hook2->hk_peer = hook; /* Link the two together */ 1506 hook->hk_peer = hook2; 1507 NG_HOOK_REF(hook); /* Add a ref for the peer to each*/ 1508 NG_HOOK_REF(hook2); 1509 hook2->hk_node = &ng_deadnode; 1510 strlcpy(NG_HOOK_NAME(hook2), name2, NG_HOOKSIZ); 1511 1512 /* 1513 * Queue the function above. 1514 * Procesing continues in that function in the lock context of 1515 * the other node. 1516 */ 1517 if ((error = ng_send_fn2(node2, hook2, item, &ng_con_part2, NULL, 0, 1518 NG_NOFLAGS))) { 1519 printf("failed in ng_con_nodes(): %d\n", error); 1520 ng_destroy_hook(hook); /* also zaps peer */ 1521 } 1522 1523 NG_HOOK_UNREF(hook); /* Let each hook go if it wants to */ 1524 NG_HOOK_UNREF(hook2); 1525 return (error); 1526 } 1527 1528 /* 1529 * Make a peer and connect. 1530 * We assume that the local node is locked. 1531 * The new node probably doesn't need a lock until 1532 * it has a hook, because it cannot really have any work until then, 1533 * but we should think about it a bit more. 1534 * 1535 * The problem may come if the other node also fires up 1536 * some hardware or a timer or some other source of activation, 1537 * also it may already get a command msg via it's ID. 1538 * 1539 * We could use the same method as ng_con_nodes() but we'd have 1540 * to add ability to remove the node when failing. (Not hard, just 1541 * make arg1 point to the node to remove). 1542 * Unless of course we just ignore failure to connect and leave 1543 * an unconnected node? 1544 */ 1545 static int 1546 ng_mkpeer(node_p node, const char *name, const char *name2, char *type) 1547 { 1548 node_p node2; 1549 hook_p hook1, hook2; 1550 int error; 1551 1552 if ((error = ng_make_node(type, &node2))) { 1553 return (error); 1554 } 1555 1556 if ((error = ng_add_hook(node, name, &hook1))) { /* gives us a ref */ 1557 ng_rmnode(node2, NULL, NULL, 0); 1558 return (error); 1559 } 1560 1561 if ((error = ng_add_hook(node2, name2, &hook2))) { 1562 ng_rmnode(node2, NULL, NULL, 0); 1563 ng_destroy_hook(hook1); 1564 NG_HOOK_UNREF(hook1); 1565 return (error); 1566 } 1567 1568 /* 1569 * Actually link the two hooks together. 1570 */ 1571 hook1->hk_peer = hook2; 1572 hook2->hk_peer = hook1; 1573 1574 /* Each hook is referenced by the other */ 1575 NG_HOOK_REF(hook1); 1576 NG_HOOK_REF(hook2); 1577 1578 /* Give each node the opportunity to veto the pending connection */ 1579 if (hook1->hk_node->nd_type->connect) { 1580 error = (*hook1->hk_node->nd_type->connect) (hook1); 1581 } 1582 1583 if ((error == 0) && hook2->hk_node->nd_type->connect) { 1584 error = (*hook2->hk_node->nd_type->connect) (hook2); 1585 1586 } 1587 1588 /* 1589 * drop the references we were holding on the two hooks. 1590 */ 1591 if (error) { 1592 ng_destroy_hook(hook2); /* also zaps hook1 */ 1593 ng_rmnode(node2, NULL, NULL, 0); 1594 } else { 1595 /* As a last act, allow the hooks to be used */ 1596 hook1->hk_flags &= ~HK_INVALID; 1597 hook2->hk_flags &= ~HK_INVALID; 1598 } 1599 NG_HOOK_UNREF(hook1); 1600 NG_HOOK_UNREF(hook2); 1601 return (error); 1602 } 1603 1604 /************************************************************************ 1605 Utility routines to send self messages 1606 ************************************************************************/ 1607 1608 /* Shut this node down as soon as everyone is clear of it */ 1609 /* Should add arg "immediately" to jump the queue */ 1610 int 1611 ng_rmnode_self(node_p node) 1612 { 1613 int error; 1614 1615 if (node == &ng_deadnode) 1616 return (0); 1617 node->nd_flags |= NGF_INVALID; 1618 if (node->nd_flags & NGF_CLOSING) 1619 return (0); 1620 1621 error = ng_send_fn(node, NULL, &ng_rmnode, NULL, 0); 1622 return (error); 1623 } 1624 1625 static void 1626 ng_rmhook_part2(node_p node, hook_p hook, void *arg1, int arg2) 1627 { 1628 ng_destroy_hook(hook); 1629 return ; 1630 } 1631 1632 int 1633 ng_rmhook_self(hook_p hook) 1634 { 1635 int error; 1636 node_p node = NG_HOOK_NODE(hook); 1637 1638 if (node == &ng_deadnode) 1639 return (0); 1640 1641 error = ng_send_fn(node, hook, &ng_rmhook_part2, NULL, 0); 1642 return (error); 1643 } 1644 1645 /*********************************************************************** 1646 * Parse and verify a string of the form: <NODE:><PATH> 1647 * 1648 * Such a string can refer to a specific node or a specific hook 1649 * on a specific node, depending on how you look at it. In the 1650 * latter case, the PATH component must not end in a dot. 1651 * 1652 * Both <NODE:> and <PATH> are optional. The <PATH> is a string 1653 * of hook names separated by dots. This breaks out the original 1654 * string, setting *nodep to "NODE" (or NULL if none) and *pathp 1655 * to "PATH" (or NULL if degenerate). Also, *hookp will point to 1656 * the final hook component of <PATH>, if any, otherwise NULL. 1657 * 1658 * This returns -1 if the path is malformed. The char ** are optional. 1659 ***********************************************************************/ 1660 int 1661 ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp) 1662 { 1663 char *node, *path, *hook; 1664 int k; 1665 1666 /* 1667 * Extract absolute NODE, if any 1668 */ 1669 for (path = addr; *path && *path != ':'; path++); 1670 if (*path) { 1671 node = addr; /* Here's the NODE */ 1672 *path++ = '\0'; /* Here's the PATH */ 1673 1674 /* Node name must not be empty */ 1675 if (!*node) 1676 return -1; 1677 1678 /* A name of "." is OK; otherwise '.' not allowed */ 1679 if (strcmp(node, ".") != 0) { 1680 for (k = 0; node[k]; k++) 1681 if (node[k] == '.') 1682 return -1; 1683 } 1684 } else { 1685 node = NULL; /* No absolute NODE */ 1686 path = addr; /* Here's the PATH */ 1687 } 1688 1689 /* Snoop for illegal characters in PATH */ 1690 for (k = 0; path[k]; k++) 1691 if (path[k] == ':') 1692 return -1; 1693 1694 /* Check for no repeated dots in PATH */ 1695 for (k = 0; path[k]; k++) 1696 if (path[k] == '.' && path[k + 1] == '.') 1697 return -1; 1698 1699 /* Remove extra (degenerate) dots from beginning or end of PATH */ 1700 if (path[0] == '.') 1701 path++; 1702 if (*path && path[strlen(path) - 1] == '.') 1703 path[strlen(path) - 1] = 0; 1704 1705 /* If PATH has a dot, then we're not talking about a hook */ 1706 if (*path) { 1707 for (hook = path, k = 0; path[k]; k++) 1708 if (path[k] == '.') { 1709 hook = NULL; 1710 break; 1711 } 1712 } else 1713 path = hook = NULL; 1714 1715 /* Done */ 1716 if (nodep) 1717 *nodep = node; 1718 if (pathp) 1719 *pathp = path; 1720 if (hookp) 1721 *hookp = hook; 1722 return (0); 1723 } 1724 1725 /* 1726 * Given a path, which may be absolute or relative, and a starting node, 1727 * return the destination node. 1728 */ 1729 int 1730 ng_path2noderef(node_p here, const char *address, node_p *destp, 1731 hook_p *lasthook) 1732 { 1733 char fullpath[NG_PATHSIZ]; 1734 char *nodename, *path; 1735 node_p node, oldnode; 1736 1737 /* Initialize */ 1738 if (destp == NULL) { 1739 TRAP_ERROR(); 1740 return EINVAL; 1741 } 1742 *destp = NULL; 1743 1744 /* Make a writable copy of address for ng_path_parse() */ 1745 strncpy(fullpath, address, sizeof(fullpath) - 1); 1746 fullpath[sizeof(fullpath) - 1] = '\0'; 1747 1748 /* Parse out node and sequence of hooks */ 1749 if (ng_path_parse(fullpath, &nodename, &path, NULL) < 0) { 1750 TRAP_ERROR(); 1751 return EINVAL; 1752 } 1753 1754 /* 1755 * For an absolute address, jump to the starting node. 1756 * Note that this holds a reference on the node for us. 1757 * Don't forget to drop the reference if we don't need it. 1758 */ 1759 if (nodename) { 1760 node = ng_name2noderef(here, nodename); 1761 if (node == NULL) { 1762 TRAP_ERROR(); 1763 return (ENOENT); 1764 } 1765 } else { 1766 if (here == NULL) { 1767 TRAP_ERROR(); 1768 return (EINVAL); 1769 } 1770 node = here; 1771 NG_NODE_REF(node); 1772 } 1773 1774 if (path == NULL) { 1775 if (lasthook != NULL) 1776 *lasthook = NULL; 1777 *destp = node; 1778 return (0); 1779 } 1780 1781 /* 1782 * Now follow the sequence of hooks 1783 * 1784 * XXXGL: The path may demolish as we go the sequence, but if 1785 * we hold the topology mutex at critical places, then, I hope, 1786 * we would always have valid pointers in hand, although the 1787 * path behind us may no longer exist. 1788 */ 1789 for (;;) { 1790 hook_p hook; 1791 char *segment; 1792 1793 /* 1794 * Break out the next path segment. Replace the dot we just 1795 * found with a NUL; "path" points to the next segment (or the 1796 * NUL at the end). 1797 */ 1798 for (segment = path; *path != '\0'; path++) { 1799 if (*path == '.') { 1800 *path++ = '\0'; 1801 break; 1802 } 1803 } 1804 1805 /* We have a segment, so look for a hook by that name */ 1806 hook = ng_findhook(node, segment); 1807 1808 TOPOLOGY_WLOCK(); 1809 /* Can't get there from here... */ 1810 if (hook == NULL || NG_HOOK_PEER(hook) == NULL || 1811 NG_HOOK_NOT_VALID(hook) || 1812 NG_HOOK_NOT_VALID(NG_HOOK_PEER(hook))) { 1813 TRAP_ERROR(); 1814 NG_NODE_UNREF(node); 1815 TOPOLOGY_WUNLOCK(); 1816 return (ENOENT); 1817 } 1818 1819 /* 1820 * Hop on over to the next node 1821 * XXX 1822 * Big race conditions here as hooks and nodes go away 1823 * *** Idea.. store an ng_ID_t in each hook and use that 1824 * instead of the direct hook in this crawl? 1825 */ 1826 oldnode = node; 1827 if ((node = NG_PEER_NODE(hook))) 1828 NG_NODE_REF(node); /* XXX RACE */ 1829 NG_NODE_UNREF(oldnode); /* XXX another race */ 1830 if (NG_NODE_NOT_VALID(node)) { 1831 NG_NODE_UNREF(node); /* XXX more races */ 1832 TOPOLOGY_WUNLOCK(); 1833 TRAP_ERROR(); 1834 return (ENXIO); 1835 } 1836 1837 if (*path == '\0') { 1838 if (lasthook != NULL) { 1839 if (hook != NULL) { 1840 *lasthook = NG_HOOK_PEER(hook); 1841 NG_HOOK_REF(*lasthook); 1842 } else 1843 *lasthook = NULL; 1844 } 1845 TOPOLOGY_WUNLOCK(); 1846 *destp = node; 1847 return (0); 1848 } 1849 TOPOLOGY_WUNLOCK(); 1850 } 1851 } 1852 1853 #ifndef FSTACK 1854 /***************************************************************\ 1855 * Input queue handling. 1856 * All activities are submitted to the node via the input queue 1857 * which implements a multiple-reader/single-writer gate. 1858 * Items which cannot be handled immediately are queued. 1859 * 1860 * read-write queue locking inline functions * 1861 \***************************************************************/ 1862 1863 static __inline void ng_queue_rw(node_p node, item_p item, int rw); 1864 static __inline item_p ng_dequeue(node_p node, int *rw); 1865 static __inline item_p ng_acquire_read(node_p node, item_p item); 1866 static __inline item_p ng_acquire_write(node_p node, item_p item); 1867 static __inline void ng_leave_read(node_p node); 1868 static __inline void ng_leave_write(node_p node); 1869 1870 /* 1871 * Definition of the bits fields in the ng_queue flag word. 1872 * Defined here rather than in netgraph.h because no-one should fiddle 1873 * with them. 1874 * 1875 * The ordering here may be important! don't shuffle these. 1876 */ 1877 /*- 1878 Safety Barrier--------+ (adjustable to suit taste) (not used yet) 1879 | 1880 V 1881 +-------+-------+-------+-------+-------+-------+-------+-------+ 1882 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | 1883 | |A|c|t|i|v|e| |R|e|a|d|e|r| |C|o|u|n|t| | | | | | | | | |P|A| 1884 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |O|W| 1885 +-------+-------+-------+-------+-------+-------+-------+-------+ 1886 \___________________________ ____________________________/ | | 1887 V | | 1888 [active reader count] | | 1889 | | 1890 Operation Pending -------------------------------+ | 1891 | 1892 Active Writer ---------------------------------------+ 1893 1894 Node queue has such semantics: 1895 - All flags modifications are atomic. 1896 - Reader count can be incremented only if there is no writer or pending flags. 1897 As soon as this can't be done with single operation, it is implemented with 1898 spin loop and atomic_cmpset(). 1899 - Writer flag can be set only if there is no any bits set. 1900 It is implemented with atomic_cmpset(). 1901 - Pending flag can be set any time, but to avoid collision on queue processing 1902 all queue fields are protected by the mutex. 1903 - Queue processing thread reads queue holding the mutex, but releases it while 1904 processing. When queue is empty pending flag is removed. 1905 */ 1906 1907 #define WRITER_ACTIVE 0x00000001 1908 #define OP_PENDING 0x00000002 1909 #define READER_INCREMENT 0x00000004 1910 #define READER_MASK 0xfffffffc /* Not valid if WRITER_ACTIVE is set */ 1911 #define SAFETY_BARRIER 0x00100000 /* 128K items queued should be enough */ 1912 1913 /* Defines of more elaborate states on the queue */ 1914 /* Mask of bits a new read cares about */ 1915 #define NGQ_RMASK (WRITER_ACTIVE|OP_PENDING) 1916 1917 /* Mask of bits a new write cares about */ 1918 #define NGQ_WMASK (NGQ_RMASK|READER_MASK) 1919 1920 /* Test to decide if there is something on the queue. */ 1921 #define QUEUE_ACTIVE(QP) ((QP)->q_flags & OP_PENDING) 1922 1923 /* How to decide what the next queued item is. */ 1924 #define HEAD_IS_READER(QP) NGI_QUEUED_READER(STAILQ_FIRST(&(QP)->queue)) 1925 #define HEAD_IS_WRITER(QP) NGI_QUEUED_WRITER(STAILQ_FIRST(&(QP)->queue)) /* notused */ 1926 1927 /* Read the status to decide if the next item on the queue can now run. */ 1928 #define QUEUED_READER_CAN_PROCEED(QP) \ 1929 (((QP)->q_flags & (NGQ_RMASK & ~OP_PENDING)) == 0) 1930 #define QUEUED_WRITER_CAN_PROCEED(QP) \ 1931 (((QP)->q_flags & (NGQ_WMASK & ~OP_PENDING)) == 0) 1932 1933 /* Is there a chance of getting ANY work off the queue? */ 1934 #define NEXT_QUEUED_ITEM_CAN_PROCEED(QP) \ 1935 ((HEAD_IS_READER(QP)) ? QUEUED_READER_CAN_PROCEED(QP) : \ 1936 QUEUED_WRITER_CAN_PROCEED(QP)) 1937 1938 #define NGQRW_R 0 1939 #define NGQRW_W 1 1940 1941 #define NGQ2_WORKQ 0x00000001 1942 1943 /* 1944 * Taking into account the current state of the queue and node, possibly take 1945 * the next entry off the queue and return it. Return NULL if there was 1946 * nothing we could return, either because there really was nothing there, or 1947 * because the node was in a state where it cannot yet process the next item 1948 * on the queue. 1949 */ 1950 static __inline item_p 1951 ng_dequeue(node_p node, int *rw) 1952 { 1953 item_p item; 1954 struct ng_queue *ngq = &node->nd_input_queue; 1955 1956 /* This MUST be called with the mutex held. */ 1957 mtx_assert(&ngq->q_mtx, MA_OWNED); 1958 1959 /* If there is nothing queued, then just return. */ 1960 if (!QUEUE_ACTIVE(ngq)) { 1961 CTR4(KTR_NET, "%20s: node [%x] (%p) queue empty; " 1962 "queue flags 0x%lx", __func__, 1963 node->nd_ID, node, ngq->q_flags); 1964 return (NULL); 1965 } 1966 1967 /* 1968 * From here, we can assume there is a head item. 1969 * We need to find out what it is and if it can be dequeued, given 1970 * the current state of the node. 1971 */ 1972 if (HEAD_IS_READER(ngq)) { 1973 while (1) { 1974 long t = ngq->q_flags; 1975 if (t & WRITER_ACTIVE) { 1976 /* There is writer, reader can't proceed. */ 1977 CTR4(KTR_NET, "%20s: node [%x] (%p) queued " 1978 "reader can't proceed; queue flags 0x%lx", 1979 __func__, node->nd_ID, node, t); 1980 return (NULL); 1981 } 1982 if (atomic_cmpset_acq_int(&ngq->q_flags, t, 1983 t + READER_INCREMENT)) 1984 break; 1985 cpu_spinwait(); 1986 } 1987 /* We have got reader lock for the node. */ 1988 *rw = NGQRW_R; 1989 } else if (atomic_cmpset_acq_int(&ngq->q_flags, OP_PENDING, 1990 OP_PENDING + WRITER_ACTIVE)) { 1991 /* We have got writer lock for the node. */ 1992 *rw = NGQRW_W; 1993 } else { 1994 /* There is somebody other, writer can't proceed. */ 1995 CTR4(KTR_NET, "%20s: node [%x] (%p) queued writer can't " 1996 "proceed; queue flags 0x%lx", __func__, node->nd_ID, node, 1997 ngq->q_flags); 1998 return (NULL); 1999 } 2000 2001 /* 2002 * Now we dequeue the request (whatever it may be) and correct the 2003 * pending flags and the next and last pointers. 2004 */ 2005 item = STAILQ_FIRST(&ngq->queue); 2006 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 2007 if (STAILQ_EMPTY(&ngq->queue)) 2008 atomic_clear_int(&ngq->q_flags, OP_PENDING); 2009 CTR6(KTR_NET, "%20s: node [%x] (%p) returning item %p as %s; queue " 2010 "flags 0x%lx", __func__, node->nd_ID, node, item, *rw ? "WRITER" : 2011 "READER", ngq->q_flags); 2012 return (item); 2013 } 2014 2015 /* 2016 * Queue a packet to be picked up later by someone else. 2017 * If the queue could be run now, add node to the queue handler's worklist. 2018 */ 2019 static __inline void 2020 ng_queue_rw(node_p node, item_p item, int rw) 2021 { 2022 struct ng_queue *ngq = &node->nd_input_queue; 2023 if (rw == NGQRW_W) 2024 NGI_SET_WRITER(item); 2025 else 2026 NGI_SET_READER(item); 2027 item->depth = 1; 2028 2029 NG_QUEUE_LOCK(ngq); 2030 /* Set OP_PENDING flag and enqueue the item. */ 2031 atomic_set_int(&ngq->q_flags, OP_PENDING); 2032 STAILQ_INSERT_TAIL(&ngq->queue, item, el_next); 2033 2034 CTR5(KTR_NET, "%20s: node [%x] (%p) queued item %p as %s", __func__, 2035 node->nd_ID, node, item, rw ? "WRITER" : "READER" ); 2036 2037 /* 2038 * We can take the worklist lock with the node locked 2039 * BUT NOT THE REVERSE! 2040 */ 2041 if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2042 ng_worklist_add(node); 2043 NG_QUEUE_UNLOCK(ngq); 2044 } 2045 2046 /* Acquire reader lock on node. If node is busy, queue the packet. */ 2047 static __inline item_p 2048 ng_acquire_read(node_p node, item_p item) 2049 { 2050 KASSERT(node != &ng_deadnode, 2051 ("%s: working on deadnode", __func__)); 2052 2053 /* Reader needs node without writer and pending items. */ 2054 for (;;) { 2055 long t = node->nd_input_queue.q_flags; 2056 if (t & NGQ_RMASK) 2057 break; /* Node is not ready for reader. */ 2058 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, t, 2059 t + READER_INCREMENT)) { 2060 /* Successfully grabbed node */ 2061 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 2062 __func__, node->nd_ID, node, item); 2063 return (item); 2064 } 2065 cpu_spinwait(); 2066 } 2067 2068 /* Queue the request for later. */ 2069 ng_queue_rw(node, item, NGQRW_R); 2070 2071 return (NULL); 2072 } 2073 2074 /* Acquire writer lock on node. If node is busy, queue the packet. */ 2075 static __inline item_p 2076 ng_acquire_write(node_p node, item_p item) 2077 { 2078 KASSERT(node != &ng_deadnode, 2079 ("%s: working on deadnode", __func__)); 2080 2081 /* Writer needs completely idle node. */ 2082 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, 0, 2083 WRITER_ACTIVE)) { 2084 /* Successfully grabbed node */ 2085 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 2086 __func__, node->nd_ID, node, item); 2087 return (item); 2088 } 2089 2090 /* Queue the request for later. */ 2091 ng_queue_rw(node, item, NGQRW_W); 2092 2093 return (NULL); 2094 } 2095 2096 #if 0 2097 static __inline item_p 2098 ng_upgrade_write(node_p node, item_p item) 2099 { 2100 struct ng_queue *ngq = &node->nd_input_queue; 2101 KASSERT(node != &ng_deadnode, 2102 ("%s: working on deadnode", __func__)); 2103 2104 NGI_SET_WRITER(item); 2105 2106 NG_QUEUE_LOCK(ngq); 2107 2108 /* 2109 * There will never be no readers as we are there ourselves. 2110 * Set the WRITER_ACTIVE flags ASAP to block out fast track readers. 2111 * The caller we are running from will call ng_leave_read() 2112 * soon, so we must account for that. We must leave again with the 2113 * READER lock. If we find other readers, then 2114 * queue the request for later. However "later" may be rignt now 2115 * if there are no readers. We don't really care if there are queued 2116 * items as we will bypass them anyhow. 2117 */ 2118 atomic_add_int(&ngq->q_flags, WRITER_ACTIVE - READER_INCREMENT); 2119 if ((ngq->q_flags & (NGQ_WMASK & ~OP_PENDING)) == WRITER_ACTIVE) { 2120 NG_QUEUE_UNLOCK(ngq); 2121 2122 /* It's just us, act on the item. */ 2123 /* will NOT drop writer lock when done */ 2124 ng_apply_item(node, item, 0); 2125 2126 /* 2127 * Having acted on the item, atomically 2128 * downgrade back to READER and finish up. 2129 */ 2130 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2131 2132 /* Our caller will call ng_leave_read() */ 2133 return; 2134 } 2135 /* 2136 * It's not just us active, so queue us AT THE HEAD. 2137 * "Why?" I hear you ask. 2138 * Put us at the head of the queue as we've already been 2139 * through it once. If there is nothing else waiting, 2140 * set the correct flags. 2141 */ 2142 if (STAILQ_EMPTY(&ngq->queue)) { 2143 /* We've gone from, 0 to 1 item in the queue */ 2144 atomic_set_int(&ngq->q_flags, OP_PENDING); 2145 2146 CTR3(KTR_NET, "%20s: node [%x] (%p) set OP_PENDING", __func__, 2147 node->nd_ID, node); 2148 }; 2149 STAILQ_INSERT_HEAD(&ngq->queue, item, el_next); 2150 CTR4(KTR_NET, "%20s: node [%x] (%p) requeued item %p as WRITER", 2151 __func__, node->nd_ID, node, item ); 2152 2153 /* Reverse what we did above. That downgrades us back to reader */ 2154 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2155 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2156 ng_worklist_add(node); 2157 NG_QUEUE_UNLOCK(ngq); 2158 2159 return; 2160 } 2161 #endif 2162 2163 /* Release reader lock. */ 2164 static __inline void 2165 ng_leave_read(node_p node) 2166 { 2167 atomic_subtract_rel_int(&node->nd_input_queue.q_flags, READER_INCREMENT); 2168 } 2169 2170 /* Release writer lock. */ 2171 static __inline void 2172 ng_leave_write(node_p node) 2173 { 2174 atomic_clear_rel_int(&node->nd_input_queue.q_flags, WRITER_ACTIVE); 2175 } 2176 2177 /* Purge node queue. Called on node shutdown. */ 2178 static void 2179 ng_flush_input_queue(node_p node) 2180 { 2181 struct ng_queue *ngq = &node->nd_input_queue; 2182 item_p item; 2183 2184 NG_QUEUE_LOCK(ngq); 2185 while ((item = STAILQ_FIRST(&ngq->queue)) != NULL) { 2186 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 2187 if (STAILQ_EMPTY(&ngq->queue)) 2188 atomic_clear_int(&ngq->q_flags, OP_PENDING); 2189 NG_QUEUE_UNLOCK(ngq); 2190 2191 /* If the item is supplying a callback, call it with an error */ 2192 if (item->apply != NULL) { 2193 if (item->depth == 1) 2194 item->apply->error = ENOENT; 2195 if (refcount_release(&item->apply->refs)) { 2196 (*item->apply->apply)(item->apply->context, 2197 item->apply->error); 2198 } 2199 } 2200 NG_FREE_ITEM(item); 2201 NG_QUEUE_LOCK(ngq); 2202 } 2203 NG_QUEUE_UNLOCK(ngq); 2204 } 2205 #endif 2206 2207 /*********************************************************************** 2208 * Externally visible method for sending or queueing messages or data. 2209 ***********************************************************************/ 2210 2211 /* 2212 * The module code should have filled out the item correctly by this stage: 2213 * Common: 2214 * reference to destination node. 2215 * Reference to destination rcv hook if relevant. 2216 * apply pointer must be or NULL or reference valid struct ng_apply_info. 2217 * Data: 2218 * pointer to mbuf 2219 * Control_Message: 2220 * pointer to msg. 2221 * ID of original sender node. (return address) 2222 * Function: 2223 * Function pointer 2224 * void * argument 2225 * integer argument 2226 * 2227 * The nodes have several routines and macros to help with this task: 2228 */ 2229 2230 int 2231 ng_snd_item(item_p item, int flags) 2232 { 2233 hook_p hook; 2234 node_p node; 2235 #ifndef FSTACK 2236 int queue, rw; 2237 struct ng_queue *ngq; 2238 #endif 2239 int error = 0; 2240 2241 /* We are sending item, so it must be present! */ 2242 KASSERT(item != NULL, ("ng_snd_item: item is NULL")); 2243 2244 #ifdef NETGRAPH_DEBUG 2245 _ngi_check(item, __FILE__, __LINE__); 2246 #endif 2247 2248 /* Item was sent once more, postpone apply() call. */ 2249 if (item->apply) 2250 refcount_acquire(&item->apply->refs); 2251 2252 node = NGI_NODE(item); 2253 /* Node is never optional. */ 2254 KASSERT(node != NULL, ("ng_snd_item: node is NULL")); 2255 2256 hook = NGI_HOOK(item); 2257 /* Valid hook and mbuf are mandatory for data. */ 2258 if ((item->el_flags & NGQF_TYPE) == NGQF_DATA) { 2259 KASSERT(hook != NULL, ("ng_snd_item: hook for data is NULL")); 2260 if (NGI_M(item) == NULL) 2261 ERROUT(EINVAL); 2262 CHECK_DATA_MBUF(NGI_M(item)); 2263 } 2264 2265 #ifndef FSTACK 2266 /* 2267 * If the item or the node specifies single threading, force 2268 * writer semantics. Similarly, the node may say one hook always 2269 * produces writers. These are overrides. 2270 */ 2271 if (((item->el_flags & NGQF_RW) == NGQF_WRITER) || 2272 (node->nd_flags & NGF_FORCE_WRITER) || 2273 (hook && (hook->hk_flags & HK_FORCE_WRITER))) { 2274 rw = NGQRW_W; 2275 } else { 2276 rw = NGQRW_R; 2277 } 2278 2279 /* 2280 * If sender or receiver requests queued delivery, or call graph 2281 * loops back from outbound to inbound path, or stack usage 2282 * level is dangerous - enqueue message. 2283 */ 2284 if ((flags & NG_QUEUE) || (hook && (hook->hk_flags & HK_QUEUE))) { 2285 queue = 1; 2286 } else if (hook && (hook->hk_flags & HK_TO_INBOUND) && 2287 curthread->td_ng_outbound) { 2288 queue = 1; 2289 } else { 2290 queue = 0; 2291 #ifdef GET_STACK_USAGE 2292 /* 2293 * Most of netgraph nodes have small stack consumption and 2294 * for them 25% of free stack space is more than enough. 2295 * Nodes/hooks with higher stack usage should be marked as 2296 * HI_STACK. For them 50% of stack will be guaranteed then. 2297 * XXX: Values 25% and 50% are completely empirical. 2298 */ 2299 size_t st, su, sl; 2300 GET_STACK_USAGE(st, su); 2301 sl = st - su; 2302 if ((sl * 4 < st) || ((sl * 2 < st) && 2303 ((node->nd_flags & NGF_HI_STACK) || (hook && 2304 (hook->hk_flags & HK_HI_STACK))))) 2305 queue = 1; 2306 #endif 2307 } 2308 2309 if (queue) { 2310 /* Put it on the queue for that node*/ 2311 ng_queue_rw(node, item, rw); 2312 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2313 } 2314 2315 /* 2316 * We already decided how we will be queueud or treated. 2317 * Try get the appropriate operating permission. 2318 */ 2319 if (rw == NGQRW_R) 2320 item = ng_acquire_read(node, item); 2321 else 2322 item = ng_acquire_write(node, item); 2323 2324 /* Item was queued while trying to get permission. */ 2325 if (item == NULL) 2326 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2327 #endif 2328 2329 NGI_GET_NODE(item, node); /* zaps stored node */ 2330 2331 item->depth++; 2332 #ifndef FSTACK 2333 error = ng_apply_item(node, item, rw); /* drops r/w lock when done */ 2334 #else 2335 error = ng_apply_item(node, item, 0); 2336 #endif 2337 2338 #ifndef FSTACK 2339 /* If something is waiting on queue and ready, schedule it. */ 2340 ngq = &node->nd_input_queue; 2341 if (QUEUE_ACTIVE(ngq)) { 2342 NG_QUEUE_LOCK(ngq); 2343 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2344 ng_worklist_add(node); 2345 NG_QUEUE_UNLOCK(ngq); 2346 } 2347 #endif 2348 2349 /* 2350 * Node may go away as soon as we remove the reference. 2351 * Whatever we do, DO NOT access the node again! 2352 */ 2353 NG_NODE_UNREF(node); 2354 2355 return (error); 2356 2357 done: 2358 /* If was not sent, apply callback here. */ 2359 if (item->apply != NULL) { 2360 if (item->depth == 0 && error != 0) 2361 item->apply->error = error; 2362 if (refcount_release(&item->apply->refs)) { 2363 (*item->apply->apply)(item->apply->context, 2364 item->apply->error); 2365 } 2366 } 2367 2368 NG_FREE_ITEM(item); 2369 return (error); 2370 } 2371 2372 /* 2373 * We have an item that was possibly queued somewhere. 2374 * It should contain all the information needed 2375 * to run it on the appropriate node/hook. 2376 * If there is apply pointer and we own the last reference, call apply(). 2377 */ 2378 static int 2379 ng_apply_item(node_p node, item_p item, int rw) 2380 { 2381 hook_p hook; 2382 ng_rcvdata_t *rcvdata; 2383 ng_rcvmsg_t *rcvmsg; 2384 struct ng_apply_info *apply; 2385 int error = 0, depth; 2386 2387 /* Node and item are never optional. */ 2388 KASSERT(node != NULL, ("ng_apply_item: node is NULL")); 2389 KASSERT(item != NULL, ("ng_apply_item: item is NULL")); 2390 2391 NGI_GET_HOOK(item, hook); /* clears stored hook */ 2392 #ifdef NETGRAPH_DEBUG 2393 _ngi_check(item, __FILE__, __LINE__); 2394 #endif 2395 2396 apply = item->apply; 2397 depth = item->depth; 2398 2399 switch (item->el_flags & NGQF_TYPE) { 2400 case NGQF_DATA: 2401 /* 2402 * Check things are still ok as when we were queued. 2403 */ 2404 KASSERT(hook != NULL, ("ng_apply_item: hook for data is NULL")); 2405 if (NG_HOOK_NOT_VALID(hook) || 2406 NG_NODE_NOT_VALID(node)) { 2407 error = EIO; 2408 NG_FREE_ITEM(item); 2409 break; 2410 } 2411 /* 2412 * If no receive method, just silently drop it. 2413 * Give preference to the hook over-ride method. 2414 */ 2415 if ((!(rcvdata = hook->hk_rcvdata)) && 2416 (!(rcvdata = NG_HOOK_NODE(hook)->nd_type->rcvdata))) { 2417 error = 0; 2418 NG_FREE_ITEM(item); 2419 break; 2420 } 2421 error = (*rcvdata)(hook, item); 2422 break; 2423 case NGQF_MESG: 2424 if (hook && NG_HOOK_NOT_VALID(hook)) { 2425 /* 2426 * The hook has been zapped then we can't use it. 2427 * Immediately drop its reference. 2428 * The message may not need it. 2429 */ 2430 NG_HOOK_UNREF(hook); 2431 hook = NULL; 2432 } 2433 /* 2434 * Similarly, if the node is a zombie there is 2435 * nothing we can do with it, drop everything. 2436 */ 2437 if (NG_NODE_NOT_VALID(node)) { 2438 TRAP_ERROR(); 2439 error = EINVAL; 2440 NG_FREE_ITEM(item); 2441 break; 2442 } 2443 /* 2444 * Call the appropriate message handler for the object. 2445 * It is up to the message handler to free the message. 2446 * If it's a generic message, handle it generically, 2447 * otherwise call the type's message handler (if it exists). 2448 * XXX (race). Remember that a queued message may 2449 * reference a node or hook that has just been 2450 * invalidated. It will exist as the queue code 2451 * is holding a reference, but.. 2452 */ 2453 if ((NGI_MSG(item)->header.typecookie == NGM_GENERIC_COOKIE) && 2454 ((NGI_MSG(item)->header.flags & NGF_RESP) == 0)) { 2455 error = ng_generic_msg(node, item, hook); 2456 break; 2457 } 2458 if (((!hook) || (!(rcvmsg = hook->hk_rcvmsg))) && 2459 (!(rcvmsg = node->nd_type->rcvmsg))) { 2460 TRAP_ERROR(); 2461 error = 0; 2462 NG_FREE_ITEM(item); 2463 break; 2464 } 2465 error = (*rcvmsg)(node, item, hook); 2466 break; 2467 case NGQF_FN: 2468 case NGQF_FN2: 2469 /* 2470 * In the case of the shutdown message we allow it to hit 2471 * even if the node is invalid. 2472 */ 2473 if (NG_NODE_NOT_VALID(node) && 2474 NGI_FN(item) != &ng_rmnode) { 2475 TRAP_ERROR(); 2476 error = EINVAL; 2477 NG_FREE_ITEM(item); 2478 break; 2479 } 2480 /* Same is about some internal functions and invalid hook. */ 2481 if (hook && NG_HOOK_NOT_VALID(hook) && 2482 NGI_FN2(item) != &ng_con_part2 && 2483 NGI_FN2(item) != &ng_con_part3 && 2484 NGI_FN(item) != &ng_rmhook_part2) { 2485 TRAP_ERROR(); 2486 error = EINVAL; 2487 NG_FREE_ITEM(item); 2488 break; 2489 } 2490 2491 if ((item->el_flags & NGQF_TYPE) == NGQF_FN) { 2492 (*NGI_FN(item))(node, hook, NGI_ARG1(item), 2493 NGI_ARG2(item)); 2494 NG_FREE_ITEM(item); 2495 } else /* it is NGQF_FN2 */ 2496 error = (*NGI_FN2(item))(node, item, hook); 2497 break; 2498 } 2499 /* 2500 * We held references on some of the resources 2501 * that we took from the item. Now that we have 2502 * finished doing everything, drop those references. 2503 */ 2504 if (hook) 2505 NG_HOOK_UNREF(hook); 2506 2507 #ifndef FSTACK 2508 if (rw == NGQRW_R) 2509 ng_leave_read(node); 2510 else 2511 ng_leave_write(node); 2512 #endif 2513 2514 /* Apply callback. */ 2515 if (apply != NULL) { 2516 if (depth == 1 && error != 0) 2517 apply->error = error; 2518 if (refcount_release(&apply->refs)) 2519 (*apply->apply)(apply->context, apply->error); 2520 } 2521 2522 return (error); 2523 } 2524 2525 /*********************************************************************** 2526 * Implement the 'generic' control messages 2527 ***********************************************************************/ 2528 static int 2529 ng_generic_msg(node_p here, item_p item, hook_p lasthook) 2530 { 2531 int error = 0; 2532 struct ng_mesg *msg; 2533 struct ng_mesg *resp = NULL; 2534 2535 NGI_GET_MSG(item, msg); 2536 if (msg->header.typecookie != NGM_GENERIC_COOKIE) { 2537 TRAP_ERROR(); 2538 error = EINVAL; 2539 goto out; 2540 } 2541 switch (msg->header.cmd) { 2542 case NGM_SHUTDOWN: 2543 ng_rmnode(here, NULL, NULL, 0); 2544 break; 2545 case NGM_MKPEER: 2546 { 2547 struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data; 2548 2549 if (msg->header.arglen != sizeof(*mkp)) { 2550 TRAP_ERROR(); 2551 error = EINVAL; 2552 break; 2553 } 2554 mkp->type[sizeof(mkp->type) - 1] = '\0'; 2555 mkp->ourhook[sizeof(mkp->ourhook) - 1] = '\0'; 2556 mkp->peerhook[sizeof(mkp->peerhook) - 1] = '\0'; 2557 error = ng_mkpeer(here, mkp->ourhook, mkp->peerhook, mkp->type); 2558 break; 2559 } 2560 case NGM_CONNECT: 2561 { 2562 struct ngm_connect *const con = 2563 (struct ngm_connect *) msg->data; 2564 node_p node2; 2565 2566 if (msg->header.arglen != sizeof(*con)) { 2567 TRAP_ERROR(); 2568 error = EINVAL; 2569 break; 2570 } 2571 con->path[sizeof(con->path) - 1] = '\0'; 2572 con->ourhook[sizeof(con->ourhook) - 1] = '\0'; 2573 con->peerhook[sizeof(con->peerhook) - 1] = '\0'; 2574 /* Don't forget we get a reference.. */ 2575 error = ng_path2noderef(here, con->path, &node2, NULL); 2576 if (error) 2577 break; 2578 error = ng_con_nodes(item, here, con->ourhook, 2579 node2, con->peerhook); 2580 NG_NODE_UNREF(node2); 2581 break; 2582 } 2583 case NGM_NAME: 2584 { 2585 struct ngm_name *const nam = (struct ngm_name *) msg->data; 2586 2587 if (msg->header.arglen != sizeof(*nam)) { 2588 TRAP_ERROR(); 2589 error = EINVAL; 2590 break; 2591 } 2592 nam->name[sizeof(nam->name) - 1] = '\0'; 2593 error = ng_name_node(here, nam->name); 2594 break; 2595 } 2596 case NGM_RMHOOK: 2597 { 2598 struct ngm_rmhook *const rmh = (struct ngm_rmhook *) msg->data; 2599 hook_p hook; 2600 2601 if (msg->header.arglen != sizeof(*rmh)) { 2602 TRAP_ERROR(); 2603 error = EINVAL; 2604 break; 2605 } 2606 rmh->ourhook[sizeof(rmh->ourhook) - 1] = '\0'; 2607 if ((hook = ng_findhook(here, rmh->ourhook)) != NULL) 2608 ng_destroy_hook(hook); 2609 break; 2610 } 2611 case NGM_NODEINFO: 2612 { 2613 struct nodeinfo *ni; 2614 2615 NG_MKRESPONSE(resp, msg, sizeof(*ni), M_NOWAIT); 2616 if (resp == NULL) { 2617 error = ENOMEM; 2618 break; 2619 } 2620 2621 /* Fill in node info */ 2622 ni = (struct nodeinfo *) resp->data; 2623 if (NG_NODE_HAS_NAME(here)) 2624 strcpy(ni->name, NG_NODE_NAME(here)); 2625 strcpy(ni->type, here->nd_type->name); 2626 ni->id = ng_node2ID(here); 2627 ni->hooks = here->nd_numhooks; 2628 break; 2629 } 2630 case NGM_LISTHOOKS: 2631 { 2632 const int nhooks = here->nd_numhooks; 2633 struct hooklist *hl; 2634 struct nodeinfo *ni; 2635 hook_p hook; 2636 2637 /* Get response struct */ 2638 NG_MKRESPONSE(resp, msg, sizeof(*hl) + 2639 (nhooks * sizeof(struct linkinfo)), M_NOWAIT); 2640 if (resp == NULL) { 2641 error = ENOMEM; 2642 break; 2643 } 2644 hl = (struct hooklist *) resp->data; 2645 ni = &hl->nodeinfo; 2646 2647 /* Fill in node info */ 2648 if (NG_NODE_HAS_NAME(here)) 2649 strcpy(ni->name, NG_NODE_NAME(here)); 2650 strcpy(ni->type, here->nd_type->name); 2651 ni->id = ng_node2ID(here); 2652 2653 /* Cycle through the linked list of hooks */ 2654 ni->hooks = 0; 2655 LIST_FOREACH(hook, &here->nd_hooks, hk_hooks) { 2656 struct linkinfo *const link = &hl->link[ni->hooks]; 2657 2658 if (ni->hooks >= nhooks) { 2659 log(LOG_ERR, "%s: number of %s changed\n", 2660 __func__, "hooks"); 2661 break; 2662 } 2663 if (NG_HOOK_NOT_VALID(hook)) 2664 continue; 2665 strcpy(link->ourhook, NG_HOOK_NAME(hook)); 2666 strcpy(link->peerhook, NG_PEER_HOOK_NAME(hook)); 2667 if (NG_PEER_NODE_NAME(hook)[0] != '\0') 2668 strcpy(link->nodeinfo.name, 2669 NG_PEER_NODE_NAME(hook)); 2670 strcpy(link->nodeinfo.type, 2671 NG_PEER_NODE(hook)->nd_type->name); 2672 link->nodeinfo.id = ng_node2ID(NG_PEER_NODE(hook)); 2673 link->nodeinfo.hooks = NG_PEER_NODE(hook)->nd_numhooks; 2674 ni->hooks++; 2675 } 2676 break; 2677 } 2678 2679 case NGM_LISTNODES: 2680 { 2681 struct namelist *nl; 2682 node_p node; 2683 int i; 2684 2685 IDHASH_RLOCK(); 2686 /* Get response struct. */ 2687 NG_MKRESPONSE(resp, msg, sizeof(*nl) + 2688 (V_ng_nodes * sizeof(struct nodeinfo)), M_NOWAIT | M_ZERO); 2689 if (resp == NULL) { 2690 IDHASH_RUNLOCK(); 2691 error = ENOMEM; 2692 break; 2693 } 2694 nl = (struct namelist *) resp->data; 2695 2696 /* Cycle through the lists of nodes. */ 2697 nl->numnames = 0; 2698 for (i = 0; i <= V_ng_ID_hmask; i++) { 2699 LIST_FOREACH(node, &V_ng_ID_hash[i], nd_idnodes) { 2700 struct nodeinfo *const np = 2701 &nl->nodeinfo[nl->numnames]; 2702 2703 if (NG_NODE_NOT_VALID(node)) 2704 continue; 2705 if (NG_NODE_HAS_NAME(node)) 2706 strcpy(np->name, NG_NODE_NAME(node)); 2707 strcpy(np->type, node->nd_type->name); 2708 np->id = ng_node2ID(node); 2709 np->hooks = node->nd_numhooks; 2710 KASSERT(nl->numnames < V_ng_nodes, 2711 ("%s: no space", __func__)); 2712 nl->numnames++; 2713 } 2714 } 2715 IDHASH_RUNLOCK(); 2716 break; 2717 } 2718 case NGM_LISTNAMES: 2719 { 2720 struct namelist *nl; 2721 node_p node; 2722 int i; 2723 2724 NAMEHASH_RLOCK(); 2725 /* Get response struct. */ 2726 NG_MKRESPONSE(resp, msg, sizeof(*nl) + 2727 (V_ng_named_nodes * sizeof(struct nodeinfo)), M_NOWAIT); 2728 if (resp == NULL) { 2729 NAMEHASH_RUNLOCK(); 2730 error = ENOMEM; 2731 break; 2732 } 2733 nl = (struct namelist *) resp->data; 2734 2735 /* Cycle through the lists of nodes. */ 2736 nl->numnames = 0; 2737 for (i = 0; i <= V_ng_name_hmask; i++) { 2738 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 2739 struct nodeinfo *const np = 2740 &nl->nodeinfo[nl->numnames]; 2741 2742 if (NG_NODE_NOT_VALID(node)) 2743 continue; 2744 strcpy(np->name, NG_NODE_NAME(node)); 2745 strcpy(np->type, node->nd_type->name); 2746 np->id = ng_node2ID(node); 2747 np->hooks = node->nd_numhooks; 2748 KASSERT(nl->numnames < V_ng_named_nodes, 2749 ("%s: no space", __func__)); 2750 nl->numnames++; 2751 } 2752 } 2753 NAMEHASH_RUNLOCK(); 2754 break; 2755 } 2756 2757 case NGM_LISTTYPES: 2758 { 2759 struct typelist *tl; 2760 struct ng_type *type; 2761 int num = 0; 2762 2763 TYPELIST_RLOCK(); 2764 /* Count number of types */ 2765 LIST_FOREACH(type, &ng_typelist, types) 2766 num++; 2767 2768 /* Get response struct */ 2769 NG_MKRESPONSE(resp, msg, sizeof(*tl) + 2770 (num * sizeof(struct typeinfo)), M_NOWAIT); 2771 if (resp == NULL) { 2772 TYPELIST_RUNLOCK(); 2773 error = ENOMEM; 2774 break; 2775 } 2776 tl = (struct typelist *) resp->data; 2777 2778 /* Cycle through the linked list of types */ 2779 tl->numtypes = 0; 2780 LIST_FOREACH(type, &ng_typelist, types) { 2781 struct typeinfo *const tp = &tl->typeinfo[tl->numtypes]; 2782 2783 strcpy(tp->type_name, type->name); 2784 tp->numnodes = type->refs - 1; /* don't count list */ 2785 KASSERT(tl->numtypes < num, ("%s: no space", __func__)); 2786 tl->numtypes++; 2787 } 2788 TYPELIST_RUNLOCK(); 2789 break; 2790 } 2791 2792 case NGM_BINARY2ASCII: 2793 { 2794 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2795 const struct ng_parse_type *argstype; 2796 const struct ng_cmdlist *c; 2797 struct ng_mesg *binary, *ascii; 2798 2799 /* Data area must contain a valid netgraph message */ 2800 binary = (struct ng_mesg *)msg->data; 2801 if (msg->header.arglen < sizeof(struct ng_mesg) || 2802 (msg->header.arglen - sizeof(struct ng_mesg) < 2803 binary->header.arglen)) { 2804 TRAP_ERROR(); 2805 error = EINVAL; 2806 break; 2807 } 2808 2809 /* Get a response message with lots of room */ 2810 NG_MKRESPONSE(resp, msg, sizeof(*ascii) + bufSize, M_NOWAIT); 2811 if (resp == NULL) { 2812 error = ENOMEM; 2813 break; 2814 } 2815 ascii = (struct ng_mesg *)resp->data; 2816 2817 /* Copy binary message header to response message payload */ 2818 bcopy(binary, ascii, sizeof(*binary)); 2819 2820 /* Find command by matching typecookie and command number */ 2821 for (c = here->nd_type->cmdlist; c != NULL && c->name != NULL; 2822 c++) { 2823 if (binary->header.typecookie == c->cookie && 2824 binary->header.cmd == c->cmd) 2825 break; 2826 } 2827 if (c == NULL || c->name == NULL) { 2828 for (c = ng_generic_cmds; c->name != NULL; c++) { 2829 if (binary->header.typecookie == c->cookie && 2830 binary->header.cmd == c->cmd) 2831 break; 2832 } 2833 if (c->name == NULL) { 2834 NG_FREE_MSG(resp); 2835 error = ENOSYS; 2836 break; 2837 } 2838 } 2839 2840 /* Convert command name to ASCII */ 2841 snprintf(ascii->header.cmdstr, sizeof(ascii->header.cmdstr), 2842 "%s", c->name); 2843 2844 /* Convert command arguments to ASCII */ 2845 argstype = (binary->header.flags & NGF_RESP) ? 2846 c->respType : c->mesgType; 2847 if (argstype == NULL) { 2848 *ascii->data = '\0'; 2849 } else { 2850 if ((error = ng_unparse(argstype, 2851 (u_char *)binary->data, 2852 ascii->data, bufSize)) != 0) { 2853 NG_FREE_MSG(resp); 2854 break; 2855 } 2856 } 2857 2858 /* Return the result as struct ng_mesg plus ASCII string */ 2859 bufSize = strlen(ascii->data) + 1; 2860 ascii->header.arglen = bufSize; 2861 resp->header.arglen = sizeof(*ascii) + bufSize; 2862 break; 2863 } 2864 2865 case NGM_ASCII2BINARY: 2866 { 2867 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2868 const struct ng_cmdlist *c; 2869 const struct ng_parse_type *argstype; 2870 struct ng_mesg *ascii, *binary; 2871 int off = 0; 2872 2873 /* Data area must contain at least a struct ng_mesg + '\0' */ 2874 ascii = (struct ng_mesg *)msg->data; 2875 if ((msg->header.arglen < sizeof(*ascii) + 1) || 2876 (ascii->header.arglen < 1) || 2877 (msg->header.arglen < sizeof(*ascii) + 2878 ascii->header.arglen)) { 2879 TRAP_ERROR(); 2880 error = EINVAL; 2881 break; 2882 } 2883 ascii->data[ascii->header.arglen - 1] = '\0'; 2884 2885 /* Get a response message with lots of room */ 2886 NG_MKRESPONSE(resp, msg, sizeof(*binary) + bufSize, M_NOWAIT); 2887 if (resp == NULL) { 2888 error = ENOMEM; 2889 break; 2890 } 2891 binary = (struct ng_mesg *)resp->data; 2892 2893 /* Copy ASCII message header to response message payload */ 2894 bcopy(ascii, binary, sizeof(*ascii)); 2895 2896 /* Find command by matching ASCII command string */ 2897 for (c = here->nd_type->cmdlist; 2898 c != NULL && c->name != NULL; c++) { 2899 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2900 break; 2901 } 2902 if (c == NULL || c->name == NULL) { 2903 for (c = ng_generic_cmds; c->name != NULL; c++) { 2904 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2905 break; 2906 } 2907 if (c->name == NULL) { 2908 NG_FREE_MSG(resp); 2909 error = ENOSYS; 2910 break; 2911 } 2912 } 2913 2914 /* Convert command name to binary */ 2915 binary->header.cmd = c->cmd; 2916 binary->header.typecookie = c->cookie; 2917 2918 /* Convert command arguments to binary */ 2919 argstype = (binary->header.flags & NGF_RESP) ? 2920 c->respType : c->mesgType; 2921 if (argstype == NULL) { 2922 bufSize = 0; 2923 } else { 2924 if ((error = ng_parse(argstype, ascii->data, &off, 2925 (u_char *)binary->data, &bufSize)) != 0) { 2926 NG_FREE_MSG(resp); 2927 break; 2928 } 2929 } 2930 2931 /* Return the result */ 2932 binary->header.arglen = bufSize; 2933 resp->header.arglen = sizeof(*binary) + bufSize; 2934 break; 2935 } 2936 2937 case NGM_TEXT_CONFIG: 2938 case NGM_TEXT_STATUS: 2939 /* 2940 * This one is tricky as it passes the command down to the 2941 * actual node, even though it is a generic type command. 2942 * This means we must assume that the item/msg is already freed 2943 * when control passes back to us. 2944 */ 2945 if (here->nd_type->rcvmsg != NULL) { 2946 NGI_MSG(item) = msg; /* put it back as we found it */ 2947 return((*here->nd_type->rcvmsg)(here, item, lasthook)); 2948 } 2949 /* Fall through if rcvmsg not supported */ 2950 default: 2951 TRAP_ERROR(); 2952 error = EINVAL; 2953 } 2954 /* 2955 * Sometimes a generic message may be statically allocated 2956 * to avoid problems with allocating when in tight memory situations. 2957 * Don't free it if it is so. 2958 * I break them apart here, because erros may cause a free if the item 2959 * in which case we'd be doing it twice. 2960 * they are kept together above, to simplify freeing. 2961 */ 2962 out: 2963 NG_RESPOND_MSG(error, here, item, resp); 2964 NG_FREE_MSG(msg); 2965 return (error); 2966 } 2967 2968 /************************************************************************ 2969 Queue element get/free routines 2970 ************************************************************************/ 2971 2972 uma_zone_t ng_qzone; 2973 uma_zone_t ng_qdzone; 2974 #ifndef FSTACK 2975 static int numthreads = 0; /* number of queue threads */ 2976 #endif 2977 static int maxalloc = 4096;/* limit the damage of a leak */ 2978 static int maxdata = 4096; /* limit the damage of a DoS */ 2979 2980 #ifndef FSTACK 2981 SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads, 2982 0, "Number of queue processing threads"); 2983 #endif 2984 SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc, 2985 0, "Maximum number of non-data queue items to allocate"); 2986 SYSCTL_INT(_net_graph, OID_AUTO, maxdata, CTLFLAG_RDTUN, &maxdata, 2987 0, "Maximum number of data queue items to allocate"); 2988 2989 #ifdef NETGRAPH_DEBUG 2990 static TAILQ_HEAD(, ng_item) ng_itemlist = TAILQ_HEAD_INITIALIZER(ng_itemlist); 2991 static int allocated; /* number of items malloc'd */ 2992 #endif 2993 2994 /* 2995 * Get a queue entry. 2996 * This is usually called when a packet first enters netgraph. 2997 * By definition, this is usually from an interrupt, or from a user. 2998 * Users are not so important, but try be quick for the times that it's 2999 * an interrupt. 3000 */ 3001 static __inline item_p 3002 ng_alloc_item(int type, int flags) 3003 { 3004 item_p item; 3005 3006 KASSERT(((type & ~NGQF_TYPE) == 0), 3007 ("%s: incorrect item type: %d", __func__, type)); 3008 3009 item = uma_zalloc((type == NGQF_DATA) ? ng_qdzone : ng_qzone, 3010 ((flags & NG_WAITOK) ? M_WAITOK : M_NOWAIT) | M_ZERO); 3011 3012 if (item) { 3013 item->el_flags = type; 3014 #ifdef NETGRAPH_DEBUG 3015 mtx_lock(&ngq_mtx); 3016 TAILQ_INSERT_TAIL(&ng_itemlist, item, all); 3017 allocated++; 3018 mtx_unlock(&ngq_mtx); 3019 #endif 3020 } 3021 3022 return (item); 3023 } 3024 3025 /* 3026 * Release a queue entry 3027 */ 3028 void 3029 ng_free_item(item_p item) 3030 { 3031 /* 3032 * The item may hold resources on it's own. We need to free 3033 * these before we can free the item. What they are depends upon 3034 * what kind of item it is. it is important that nodes zero 3035 * out pointers to resources that they remove from the item 3036 * or we release them again here. 3037 */ 3038 switch (item->el_flags & NGQF_TYPE) { 3039 case NGQF_DATA: 3040 /* If we have an mbuf still attached.. */ 3041 NG_FREE_M(_NGI_M(item)); 3042 break; 3043 case NGQF_MESG: 3044 _NGI_RETADDR(item) = 0; 3045 NG_FREE_MSG(_NGI_MSG(item)); 3046 break; 3047 case NGQF_FN: 3048 case NGQF_FN2: 3049 /* nothing to free really, */ 3050 _NGI_FN(item) = NULL; 3051 _NGI_ARG1(item) = NULL; 3052 _NGI_ARG2(item) = 0; 3053 break; 3054 } 3055 /* If we still have a node or hook referenced... */ 3056 _NGI_CLR_NODE(item); 3057 _NGI_CLR_HOOK(item); 3058 3059 #ifdef NETGRAPH_DEBUG 3060 mtx_lock(&ngq_mtx); 3061 TAILQ_REMOVE(&ng_itemlist, item, all); 3062 allocated--; 3063 mtx_unlock(&ngq_mtx); 3064 #endif 3065 uma_zfree(((item->el_flags & NGQF_TYPE) == NGQF_DATA) ? 3066 ng_qdzone : ng_qzone, item); 3067 } 3068 3069 /* 3070 * Change type of the queue entry. 3071 * Possibly reallocates it from another UMA zone. 3072 */ 3073 static __inline item_p 3074 ng_realloc_item(item_p pitem, int type, int flags) 3075 { 3076 item_p item; 3077 int from, to; 3078 3079 KASSERT((pitem != NULL), ("%s: can't reallocate NULL", __func__)); 3080 KASSERT(((type & ~NGQF_TYPE) == 0), 3081 ("%s: incorrect item type: %d", __func__, type)); 3082 3083 from = ((pitem->el_flags & NGQF_TYPE) == NGQF_DATA); 3084 to = (type == NGQF_DATA); 3085 if (from != to) { 3086 /* If reallocation is required do it and copy item. */ 3087 if ((item = ng_alloc_item(type, flags)) == NULL) { 3088 ng_free_item(pitem); 3089 return (NULL); 3090 } 3091 *item = *pitem; 3092 ng_free_item(pitem); 3093 } else 3094 item = pitem; 3095 item->el_flags = (item->el_flags & ~NGQF_TYPE) | type; 3096 3097 return (item); 3098 } 3099 3100 /************************************************************************ 3101 Module routines 3102 ************************************************************************/ 3103 3104 /* 3105 * Handle the loading/unloading of a netgraph node type module 3106 */ 3107 int 3108 ng_mod_event(module_t mod, int event, void *data) 3109 { 3110 struct ng_type *const type = data; 3111 int error = 0; 3112 3113 switch (event) { 3114 case MOD_LOAD: 3115 3116 /* Register new netgraph node type */ 3117 if ((error = ng_newtype(type)) != 0) 3118 break; 3119 3120 /* Call type specific code */ 3121 if (type->mod_event != NULL) 3122 if ((error = (*type->mod_event)(mod, event, data))) { 3123 TYPELIST_WLOCK(); 3124 type->refs--; /* undo it */ 3125 LIST_REMOVE(type, types); 3126 TYPELIST_WUNLOCK(); 3127 } 3128 break; 3129 3130 case MOD_UNLOAD: 3131 if (type->refs > 1) { /* make sure no nodes exist! */ 3132 error = EBUSY; 3133 } else { 3134 if (type->refs == 0) /* failed load, nothing to undo */ 3135 break; 3136 if (type->mod_event != NULL) { /* check with type */ 3137 error = (*type->mod_event)(mod, event, data); 3138 if (error != 0) /* type refuses.. */ 3139 break; 3140 } 3141 TYPELIST_WLOCK(); 3142 LIST_REMOVE(type, types); 3143 TYPELIST_WUNLOCK(); 3144 } 3145 break; 3146 3147 default: 3148 if (type->mod_event != NULL) 3149 error = (*type->mod_event)(mod, event, data); 3150 else 3151 error = EOPNOTSUPP; /* XXX ? */ 3152 break; 3153 } 3154 return (error); 3155 } 3156 3157 static void 3158 vnet_netgraph_init(const void *unused __unused) 3159 { 3160 3161 /* We start with small hashes, but they can grow. */ 3162 V_ng_ID_hash = hashinit(16, M_NETGRAPH_NODE, &V_ng_ID_hmask); 3163 V_ng_name_hash = hashinit(16, M_NETGRAPH_NODE, &V_ng_name_hmask); 3164 } 3165 VNET_SYSINIT(vnet_netgraph_init, SI_SUB_NETGRAPH, SI_ORDER_FIRST, 3166 vnet_netgraph_init, NULL); 3167 3168 #ifdef VIMAGE 3169 static void 3170 vnet_netgraph_uninit(const void *unused __unused) 3171 { 3172 node_p node = NULL, last_killed = NULL; 3173 int i; 3174 3175 do { 3176 /* Find a node to kill */ 3177 IDHASH_RLOCK(); 3178 for (i = 0; i <= V_ng_ID_hmask; i++) { 3179 LIST_FOREACH(node, &V_ng_ID_hash[i], nd_idnodes) { 3180 if (node != &ng_deadnode) { 3181 NG_NODE_REF(node); 3182 break; 3183 } 3184 } 3185 if (node != NULL) 3186 break; 3187 } 3188 IDHASH_RUNLOCK(); 3189 3190 /* Attempt to kill it only if it is a regular node */ 3191 if (node != NULL) { 3192 if (node == last_killed) { 3193 /* This should never happen */ 3194 printf("ng node %s needs NGF_REALLY_DIE\n", 3195 node->nd_name); 3196 if (node->nd_flags & NGF_REALLY_DIE) 3197 panic("ng node %s won't die", 3198 node->nd_name); 3199 node->nd_flags |= NGF_REALLY_DIE; 3200 } 3201 ng_rmnode(node, NULL, NULL, 0); 3202 NG_NODE_UNREF(node); 3203 last_killed = node; 3204 } 3205 } while (node != NULL); 3206 3207 hashdestroy(V_ng_name_hash, M_NETGRAPH_NODE, V_ng_name_hmask); 3208 hashdestroy(V_ng_ID_hash, M_NETGRAPH_NODE, V_ng_ID_hmask); 3209 } 3210 VNET_SYSUNINIT(vnet_netgraph_uninit, SI_SUB_NETGRAPH, SI_ORDER_FIRST, 3211 vnet_netgraph_uninit, NULL); 3212 #endif /* VIMAGE */ 3213 3214 /* 3215 * Handle loading and unloading for this code. 3216 * The only thing we need to link into is the NETISR strucure. 3217 */ 3218 static int 3219 ngb_mod_event(module_t mod, int event, void *data) 3220 { 3221 #ifndef FSTACK 3222 struct proc *p; 3223 struct thread *td; 3224 #endif 3225 int i, error = 0; 3226 3227 switch (event) { 3228 case MOD_LOAD: 3229 /* Initialize everything. */ 3230 NG_WORKLIST_LOCK_INIT(); 3231 rw_init(&ng_typelist_lock, "netgraph types"); 3232 rw_init(&ng_idhash_lock, "netgraph idhash"); 3233 rw_init(&ng_namehash_lock, "netgraph namehash"); 3234 rw_init(&ng_topo_lock, "netgraph topology mutex"); 3235 #ifdef NETGRAPH_DEBUG 3236 mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", NULL, 3237 MTX_DEF); 3238 mtx_init(&ngq_mtx, "netgraph item list mutex", NULL, 3239 MTX_DEF); 3240 #endif 3241 ng_qzone = uma_zcreate("NetGraph items", sizeof(struct ng_item), 3242 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 3243 uma_zone_set_max(ng_qzone, maxalloc); 3244 ng_qdzone = uma_zcreate("NetGraph data items", 3245 sizeof(struct ng_item), NULL, NULL, NULL, NULL, 3246 UMA_ALIGN_CACHE, 0); 3247 uma_zone_set_max(ng_qdzone, maxdata); 3248 #ifndef FSTACK 3249 /* Autoconfigure number of threads. */ 3250 if (numthreads <= 0) 3251 numthreads = mp_ncpus; 3252 /* Create threads. */ 3253 p = NULL; /* start with no process */ 3254 for (i = 0; i < numthreads; i++) { 3255 if (kproc_kthread_add(ngthread, NULL, &p, &td, 3256 RFHIGHPID, 0, "ng_queue", "ng_queue%d", i)) { 3257 numthreads = i; 3258 break; 3259 } 3260 } 3261 #endif 3262 break; 3263 case MOD_UNLOAD: 3264 /* You can't unload it because an interface may be using it. */ 3265 error = EBUSY; 3266 break; 3267 default: 3268 error = EOPNOTSUPP; 3269 break; 3270 } 3271 return (error); 3272 } 3273 3274 static moduledata_t netgraph_mod = { 3275 "netgraph", 3276 ngb_mod_event, 3277 (NULL) 3278 }; 3279 DECLARE_MODULE(netgraph, netgraph_mod, SI_SUB_NETGRAPH, SI_ORDER_FIRST); 3280 SYSCTL_NODE(_net, OID_AUTO, graph, CTLFLAG_RW, 0, "netgraph Family"); 3281 SYSCTL_INT(_net_graph, OID_AUTO, abi_version, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, NG_ABI_VERSION,""); 3282 SYSCTL_INT(_net_graph, OID_AUTO, msg_version, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, NG_VERSION, ""); 3283 3284 #ifdef NETGRAPH_DEBUG 3285 void 3286 dumphook (hook_p hook, char *file, int line) 3287 { 3288 printf("hook: name %s, %d refs, Last touched:\n", 3289 _NG_HOOK_NAME(hook), hook->hk_refs); 3290 printf(" Last active @ %s, line %d\n", 3291 hook->lastfile, hook->lastline); 3292 if (line) { 3293 printf(" problem discovered at file %s, line %d\n", file, line); 3294 #ifdef KDB 3295 kdb_backtrace(); 3296 #endif 3297 } 3298 } 3299 3300 void 3301 dumpnode(node_p node, char *file, int line) 3302 { 3303 printf("node: ID [%x]: type '%s', %d hooks, flags 0x%x, %d refs, %s:\n", 3304 _NG_NODE_ID(node), node->nd_type->name, 3305 node->nd_numhooks, node->nd_flags, 3306 node->nd_refs, node->nd_name); 3307 printf(" Last active @ %s, line %d\n", 3308 node->lastfile, node->lastline); 3309 if (line) { 3310 printf(" problem discovered at file %s, line %d\n", file, line); 3311 #ifdef KDB 3312 kdb_backtrace(); 3313 #endif 3314 } 3315 } 3316 3317 void 3318 dumpitem(item_p item, char *file, int line) 3319 { 3320 printf(" ACTIVE item, last used at %s, line %d", 3321 item->lastfile, item->lastline); 3322 switch(item->el_flags & NGQF_TYPE) { 3323 case NGQF_DATA: 3324 printf(" - [data]\n"); 3325 break; 3326 case NGQF_MESG: 3327 printf(" - retaddr[%d]:\n", _NGI_RETADDR(item)); 3328 break; 3329 case NGQF_FN: 3330 printf(" - fn@%p (%p, %p, %p, %d (%x))\n", 3331 _NGI_FN(item), 3332 _NGI_NODE(item), 3333 _NGI_HOOK(item), 3334 item->body.fn.fn_arg1, 3335 item->body.fn.fn_arg2, 3336 item->body.fn.fn_arg2); 3337 break; 3338 case NGQF_FN2: 3339 printf(" - fn2@%p (%p, %p, %p, %d (%x))\n", 3340 _NGI_FN2(item), 3341 _NGI_NODE(item), 3342 _NGI_HOOK(item), 3343 item->body.fn.fn_arg1, 3344 item->body.fn.fn_arg2, 3345 item->body.fn.fn_arg2); 3346 break; 3347 } 3348 if (line) { 3349 printf(" problem discovered at file %s, line %d\n", file, line); 3350 if (_NGI_NODE(item)) { 3351 printf("node %p ([%x])\n", 3352 _NGI_NODE(item), ng_node2ID(_NGI_NODE(item))); 3353 } 3354 } 3355 } 3356 3357 static void 3358 ng_dumpitems(void) 3359 { 3360 item_p item; 3361 int i = 1; 3362 TAILQ_FOREACH(item, &ng_itemlist, all) { 3363 printf("[%d] ", i++); 3364 dumpitem(item, NULL, 0); 3365 } 3366 } 3367 3368 static void 3369 ng_dumpnodes(void) 3370 { 3371 node_p node; 3372 int i = 1; 3373 mtx_lock(&ng_nodelist_mtx); 3374 SLIST_FOREACH(node, &ng_allnodes, nd_all) { 3375 printf("[%d] ", i++); 3376 dumpnode(node, NULL, 0); 3377 } 3378 mtx_unlock(&ng_nodelist_mtx); 3379 } 3380 3381 static void 3382 ng_dumphooks(void) 3383 { 3384 hook_p hook; 3385 int i = 1; 3386 mtx_lock(&ng_nodelist_mtx); 3387 SLIST_FOREACH(hook, &ng_allhooks, hk_all) { 3388 printf("[%d] ", i++); 3389 dumphook(hook, NULL, 0); 3390 } 3391 mtx_unlock(&ng_nodelist_mtx); 3392 } 3393 3394 static int 3395 sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS) 3396 { 3397 int error; 3398 int val; 3399 int i; 3400 3401 val = allocated; 3402 i = 1; 3403 error = sysctl_handle_int(oidp, &val, 0, req); 3404 if (error != 0 || req->newptr == NULL) 3405 return (error); 3406 if (val == 42) { 3407 ng_dumpitems(); 3408 ng_dumpnodes(); 3409 ng_dumphooks(); 3410 } 3411 return (0); 3412 } 3413 3414 SYSCTL_PROC(_debug, OID_AUTO, ng_dump_items, CTLTYPE_INT | CTLFLAG_RW, 3415 0, sizeof(int), sysctl_debug_ng_dump_items, "I", "Number of allocated items"); 3416 #endif /* NETGRAPH_DEBUG */ 3417 3418 #ifndef FSTACK 3419 /*********************************************************************** 3420 * Worklist routines 3421 **********************************************************************/ 3422 /* 3423 * Pick a node off the list of nodes with work, 3424 * try get an item to process off it. Remove the node from the list. 3425 */ 3426 static void 3427 ngthread(void *arg) 3428 { 3429 for (;;) { 3430 node_p node; 3431 3432 /* Get node from the worklist. */ 3433 NG_WORKLIST_LOCK(); 3434 while ((node = STAILQ_FIRST(&ng_worklist)) == NULL) 3435 NG_WORKLIST_SLEEP(); 3436 STAILQ_REMOVE_HEAD(&ng_worklist, nd_input_queue.q_work); 3437 NG_WORKLIST_UNLOCK(); 3438 CURVNET_SET(node->nd_vnet); 3439 CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist", 3440 __func__, node->nd_ID, node); 3441 /* 3442 * We have the node. We also take over the reference 3443 * that the list had on it. 3444 * Now process as much as you can, until it won't 3445 * let you have another item off the queue. 3446 * All this time, keep the reference 3447 * that lets us be sure that the node still exists. 3448 * Let the reference go at the last minute. 3449 */ 3450 for (;;) { 3451 item_p item; 3452 int rw; 3453 3454 NG_QUEUE_LOCK(&node->nd_input_queue); 3455 item = ng_dequeue(node, &rw); 3456 if (item == NULL) { 3457 node->nd_input_queue.q_flags2 &= ~NGQ2_WORKQ; 3458 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3459 break; /* go look for another node */ 3460 } else { 3461 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3462 NGI_GET_NODE(item, node); /* zaps stored node */ 3463 ng_apply_item(node, item, rw); 3464 NG_NODE_UNREF(node); 3465 } 3466 } 3467 NG_NODE_UNREF(node); 3468 CURVNET_RESTORE(); 3469 } 3470 } 3471 3472 /* 3473 * XXX 3474 * It's posible that a debugging NG_NODE_REF may need 3475 * to be outside the mutex zone 3476 */ 3477 static void 3478 ng_worklist_add(node_p node) 3479 { 3480 3481 mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED); 3482 3483 if ((node->nd_input_queue.q_flags2 & NGQ2_WORKQ) == 0) { 3484 /* 3485 * If we are not already on the work queue, 3486 * then put us on. 3487 */ 3488 node->nd_input_queue.q_flags2 |= NGQ2_WORKQ; 3489 NG_NODE_REF(node); /* XXX safe in mutex? */ 3490 NG_WORKLIST_LOCK(); 3491 STAILQ_INSERT_TAIL(&ng_worklist, node, nd_input_queue.q_work); 3492 NG_WORKLIST_UNLOCK(); 3493 CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__, 3494 node->nd_ID, node); 3495 NG_WORKLIST_WAKEUP(); 3496 } else { 3497 CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist", 3498 __func__, node->nd_ID, node); 3499 } 3500 } 3501 #endif 3502 3503 /*********************************************************************** 3504 * Externally useable functions to set up a queue item ready for sending 3505 ***********************************************************************/ 3506 3507 #ifdef NETGRAPH_DEBUG 3508 #define ITEM_DEBUG_CHECKS \ 3509 do { \ 3510 if (NGI_NODE(item) ) { \ 3511 printf("item already has node"); \ 3512 kdb_enter(KDB_WHY_NETGRAPH, "has node"); \ 3513 NGI_CLR_NODE(item); \ 3514 } \ 3515 if (NGI_HOOK(item) ) { \ 3516 printf("item already has hook"); \ 3517 kdb_enter(KDB_WHY_NETGRAPH, "has hook"); \ 3518 NGI_CLR_HOOK(item); \ 3519 } \ 3520 } while (0) 3521 #else 3522 #define ITEM_DEBUG_CHECKS 3523 #endif 3524 3525 /* 3526 * Put mbuf into the item. 3527 * Hook and node references will be removed when the item is dequeued. 3528 * (or equivalent) 3529 * (XXX) Unsafe because no reference held by peer on remote node. 3530 * remote node might go away in this timescale. 3531 * We know the hooks can't go away because that would require getting 3532 * a writer item on both nodes and we must have at least a reader 3533 * here to be able to do this. 3534 * Note that the hook loaded is the REMOTE hook. 3535 * 3536 * This is possibly in the critical path for new data. 3537 */ 3538 item_p 3539 ng_package_data(struct mbuf *m, int flags) 3540 { 3541 item_p item; 3542 3543 if ((item = ng_alloc_item(NGQF_DATA, flags)) == NULL) { 3544 NG_FREE_M(m); 3545 return (NULL); 3546 } 3547 ITEM_DEBUG_CHECKS; 3548 item->el_flags |= NGQF_READER; 3549 NGI_M(item) = m; 3550 return (item); 3551 } 3552 3553 /* 3554 * Allocate a queue item and put items into it.. 3555 * Evaluate the address as this will be needed to queue it and 3556 * to work out what some of the fields should be. 3557 * Hook and node references will be removed when the item is dequeued. 3558 * (or equivalent) 3559 */ 3560 item_p 3561 ng_package_msg(struct ng_mesg *msg, int flags) 3562 { 3563 item_p item; 3564 3565 if ((item = ng_alloc_item(NGQF_MESG, flags)) == NULL) { 3566 NG_FREE_MSG(msg); 3567 return (NULL); 3568 } 3569 ITEM_DEBUG_CHECKS; 3570 /* Messages items count as writers unless explicitly exempted. */ 3571 if (msg->header.cmd & NGM_READONLY) 3572 item->el_flags |= NGQF_READER; 3573 else 3574 item->el_flags |= NGQF_WRITER; 3575 /* 3576 * Set the current lasthook into the queue item 3577 */ 3578 NGI_MSG(item) = msg; 3579 NGI_RETADDR(item) = 0; 3580 return (item); 3581 } 3582 3583 #define SET_RETADDR(item, here, retaddr) \ 3584 do { /* Data or fn items don't have retaddrs */ \ 3585 if ((item->el_flags & NGQF_TYPE) == NGQF_MESG) { \ 3586 if (retaddr) { \ 3587 NGI_RETADDR(item) = retaddr; \ 3588 } else { \ 3589 /* \ 3590 * The old return address should be ok. \ 3591 * If there isn't one, use the address \ 3592 * here. \ 3593 */ \ 3594 if (NGI_RETADDR(item) == 0) { \ 3595 NGI_RETADDR(item) \ 3596 = ng_node2ID(here); \ 3597 } \ 3598 } \ 3599 } \ 3600 } while (0) 3601 3602 int 3603 ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr) 3604 { 3605 hook_p peer; 3606 node_p peernode; 3607 ITEM_DEBUG_CHECKS; 3608 /* 3609 * Quick sanity check.. 3610 * Since a hook holds a reference on it's node, once we know 3611 * that the peer is still connected (even if invalid,) we know 3612 * that the peer node is present, though maybe invalid. 3613 */ 3614 TOPOLOGY_RLOCK(); 3615 if ((hook == NULL) || NG_HOOK_NOT_VALID(hook) || 3616 NG_HOOK_NOT_VALID(peer = NG_HOOK_PEER(hook)) || 3617 NG_NODE_NOT_VALID(peernode = NG_PEER_NODE(hook))) { 3618 NG_FREE_ITEM(item); 3619 TRAP_ERROR(); 3620 TOPOLOGY_RUNLOCK(); 3621 return (ENETDOWN); 3622 } 3623 3624 /* 3625 * Transfer our interest to the other (peer) end. 3626 */ 3627 NG_HOOK_REF(peer); 3628 NG_NODE_REF(peernode); 3629 NGI_SET_HOOK(item, peer); 3630 NGI_SET_NODE(item, peernode); 3631 SET_RETADDR(item, here, retaddr); 3632 3633 TOPOLOGY_RUNLOCK(); 3634 3635 return (0); 3636 } 3637 3638 int 3639 ng_address_path(node_p here, item_p item, const char *address, ng_ID_t retaddr) 3640 { 3641 node_p dest = NULL; 3642 hook_p hook = NULL; 3643 int error; 3644 3645 ITEM_DEBUG_CHECKS; 3646 /* 3647 * Note that ng_path2noderef increments the reference count 3648 * on the node for us if it finds one. So we don't have to. 3649 */ 3650 error = ng_path2noderef(here, address, &dest, &hook); 3651 if (error) { 3652 NG_FREE_ITEM(item); 3653 return (error); 3654 } 3655 NGI_SET_NODE(item, dest); 3656 if (hook) 3657 NGI_SET_HOOK(item, hook); 3658 3659 SET_RETADDR(item, here, retaddr); 3660 return (0); 3661 } 3662 3663 int 3664 ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr) 3665 { 3666 node_p dest; 3667 3668 ITEM_DEBUG_CHECKS; 3669 /* 3670 * Find the target node. 3671 */ 3672 dest = ng_ID2noderef(ID); /* GETS REFERENCE! */ 3673 if (dest == NULL) { 3674 NG_FREE_ITEM(item); 3675 TRAP_ERROR(); 3676 return(EINVAL); 3677 } 3678 /* Fill out the contents */ 3679 NGI_SET_NODE(item, dest); 3680 NGI_CLR_HOOK(item); 3681 SET_RETADDR(item, here, retaddr); 3682 return (0); 3683 } 3684 3685 /* 3686 * special case to send a message to self (e.g. destroy node) 3687 * Possibly indicate an arrival hook too. 3688 * Useful for removing that hook :-) 3689 */ 3690 item_p 3691 ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg) 3692 { 3693 item_p item; 3694 3695 /* 3696 * Find the target node. 3697 * If there is a HOOK argument, then use that in preference 3698 * to the address. 3699 */ 3700 if ((item = ng_alloc_item(NGQF_MESG, NG_NOFLAGS)) == NULL) { 3701 NG_FREE_MSG(msg); 3702 return (NULL); 3703 } 3704 3705 /* Fill out the contents */ 3706 item->el_flags |= NGQF_WRITER; 3707 NG_NODE_REF(here); 3708 NGI_SET_NODE(item, here); 3709 if (hook) { 3710 NG_HOOK_REF(hook); 3711 NGI_SET_HOOK(item, hook); 3712 } 3713 NGI_MSG(item) = msg; 3714 NGI_RETADDR(item) = ng_node2ID(here); 3715 return (item); 3716 } 3717 3718 /* 3719 * Send ng_item_fn function call to the specified node. 3720 */ 3721 3722 int 3723 ng_send_fn(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2) 3724 { 3725 3726 return ng_send_fn1(node, hook, fn, arg1, arg2, NG_NOFLAGS); 3727 } 3728 3729 int 3730 ng_send_fn1(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2, 3731 int flags) 3732 { 3733 item_p item; 3734 3735 if ((item = ng_alloc_item(NGQF_FN, flags)) == NULL) { 3736 return (ENOMEM); 3737 } 3738 item->el_flags |= NGQF_WRITER; 3739 NG_NODE_REF(node); /* and one for the item */ 3740 NGI_SET_NODE(item, node); 3741 if (hook) { 3742 NG_HOOK_REF(hook); 3743 NGI_SET_HOOK(item, hook); 3744 } 3745 NGI_FN(item) = fn; 3746 NGI_ARG1(item) = arg1; 3747 NGI_ARG2(item) = arg2; 3748 return(ng_snd_item(item, flags)); 3749 } 3750 3751 /* 3752 * Send ng_item_fn2 function call to the specified node. 3753 * 3754 * If an optional pitem parameter is supplied, its apply 3755 * callback will be copied to the new item. If also NG_REUSE_ITEM 3756 * flag is set, no new item will be allocated, but pitem will 3757 * be used. 3758 */ 3759 int 3760 ng_send_fn2(node_p node, hook_p hook, item_p pitem, ng_item_fn2 *fn, void *arg1, 3761 int arg2, int flags) 3762 { 3763 item_p item; 3764 3765 KASSERT((pitem != NULL || (flags & NG_REUSE_ITEM) == 0), 3766 ("%s: NG_REUSE_ITEM but no pitem", __func__)); 3767 3768 /* 3769 * Allocate a new item if no supplied or 3770 * if we can't use supplied one. 3771 */ 3772 if (pitem == NULL || (flags & NG_REUSE_ITEM) == 0) { 3773 if ((item = ng_alloc_item(NGQF_FN2, flags)) == NULL) 3774 return (ENOMEM); 3775 if (pitem != NULL) 3776 item->apply = pitem->apply; 3777 } else { 3778 if ((item = ng_realloc_item(pitem, NGQF_FN2, flags)) == NULL) 3779 return (ENOMEM); 3780 } 3781 3782 item->el_flags = (item->el_flags & ~NGQF_RW) | NGQF_WRITER; 3783 NG_NODE_REF(node); /* and one for the item */ 3784 NGI_SET_NODE(item, node); 3785 if (hook) { 3786 NG_HOOK_REF(hook); 3787 NGI_SET_HOOK(item, hook); 3788 } 3789 NGI_FN2(item) = fn; 3790 NGI_ARG1(item) = arg1; 3791 NGI_ARG2(item) = arg2; 3792 return(ng_snd_item(item, flags)); 3793 } 3794 3795 /* 3796 * Official timeout routines for Netgraph nodes. 3797 */ 3798 static void 3799 ng_callout_trampoline(void *arg) 3800 { 3801 item_p item = arg; 3802 3803 CURVNET_SET(NGI_NODE(item)->nd_vnet); 3804 ng_snd_item(item, 0); 3805 CURVNET_RESTORE(); 3806 } 3807 3808 int 3809 ng_callout(struct callout *c, node_p node, hook_p hook, int ticks, 3810 ng_item_fn *fn, void * arg1, int arg2) 3811 { 3812 item_p item, oitem; 3813 3814 if ((item = ng_alloc_item(NGQF_FN, NG_NOFLAGS)) == NULL) 3815 return (ENOMEM); 3816 3817 item->el_flags |= NGQF_WRITER; 3818 NG_NODE_REF(node); /* and one for the item */ 3819 NGI_SET_NODE(item, node); 3820 if (hook) { 3821 NG_HOOK_REF(hook); 3822 NGI_SET_HOOK(item, hook); 3823 } 3824 NGI_FN(item) = fn; 3825 NGI_ARG1(item) = arg1; 3826 NGI_ARG2(item) = arg2; 3827 oitem = c->c_arg; 3828 if (callout_reset(c, ticks, &ng_callout_trampoline, item) == 1 && 3829 oitem != NULL) 3830 NG_FREE_ITEM(oitem); 3831 return (0); 3832 } 3833 3834 /* A special modified version of untimeout() */ 3835 int 3836 ng_uncallout(struct callout *c, node_p node) 3837 { 3838 item_p item; 3839 int rval; 3840 3841 KASSERT(c != NULL, ("ng_uncallout: NULL callout")); 3842 KASSERT(node != NULL, ("ng_uncallout: NULL node")); 3843 3844 rval = callout_stop(c); 3845 item = c->c_arg; 3846 /* Do an extra check */ 3847 if ((rval > 0) && (c->c_func == &ng_callout_trampoline) && 3848 (NGI_NODE(item) == node)) { 3849 /* 3850 * We successfully removed it from the queue before it ran 3851 * So now we need to unreference everything that was 3852 * given extra references. (NG_FREE_ITEM does this). 3853 */ 3854 NG_FREE_ITEM(item); 3855 } 3856 c->c_arg = NULL; 3857 3858 return (rval); 3859 } 3860 3861 /* 3862 * Set the address, if none given, give the node here. 3863 */ 3864 void 3865 ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr) 3866 { 3867 if (retaddr) { 3868 NGI_RETADDR(item) = retaddr; 3869 } else { 3870 /* 3871 * The old return address should be ok. 3872 * If there isn't one, use the address here. 3873 */ 3874 NGI_RETADDR(item) = ng_node2ID(here); 3875 } 3876 } 3877