141 static struct inpcb	*in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
353 	struct inpcbinfo *pcbinfo;  in in_pcbinslbgrouphash()  local
358 	pcbinfo = inp->inp_pcbinfo;  in in_pcbinslbgrouphash()
361 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbinslbgrouphash()
374 	idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);  in in_pcbinslbgrouphash()
375 	hdr = &pcbinfo->ipi_lbgrouphashbase[idx];  in in_pcbinslbgrouphash()
424 	struct inpcbinfo *pcbinfo;  in in_pcbremlbgrouphash()  local
429 	pcbinfo = inp->inp_pcbinfo;  in in_pcbremlbgrouphash()
433 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbremlbgrouphash()
435 	hdr = &pcbinfo->ipi_lbgrouphashbase[  in in_pcbremlbgrouphash()
436 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];  in in_pcbremlbgrouphash()
459 	struct inpcbinfo *pcbinfo;  in in_pcblbgroup_numa()  local
479 	pcbinfo = inp->inp_pcbinfo;  in in_pcblbgroup_numa()
481 	INP_HASH_WLOCK(pcbinfo);  in in_pcblbgroup_numa()
482 	hdr = &pcbinfo->ipi_lbgrouphashbase[  in in_pcblbgroup_numa()
483 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];  in in_pcblbgroup_numa()
503 	INP_HASH_WUNLOCK(pcbinfo);  in in_pcblbgroup_numa()
514 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,  in in_pcbinfo_init()  argument
518 	mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);  in in_pcbinfo_init()
519 	mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,  in in_pcbinfo_init()
522 	pcbinfo->ipi_vnet = curvnet;  in in_pcbinfo_init()
524 	CK_LIST_INIT(&pcbinfo->ipi_listhead);  in in_pcbinfo_init()
525 	pcbinfo->ipi_count = 0;  in in_pcbinfo_init()
526 	pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB,  in in_pcbinfo_init()
527 	    &pcbinfo->ipi_hashmask);  in in_pcbinfo_init()
528 	pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB,  in in_pcbinfo_init()
529 	    &pcbinfo->ipi_hashmask);  in in_pcbinfo_init()
531 	pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,  in in_pcbinfo_init()
532 	    &pcbinfo->ipi_porthashmask);  in in_pcbinfo_init()
533 	pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,  in in_pcbinfo_init()
534 	    &pcbinfo->ipi_lbgrouphashmask);  in in_pcbinfo_init()
535 	pcbinfo->ipi_zone = pcbstor->ips_zone;  in in_pcbinfo_init()
536 	pcbinfo->ipi_portzone = pcbstor->ips_portzone;  in in_pcbinfo_init()
537 	pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);  in in_pcbinfo_init()
544 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)  in in_pcbinfo_destroy()  argument
547 	KASSERT(pcbinfo->ipi_count == 0,  in in_pcbinfo_destroy()
548 	    ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));  in in_pcbinfo_destroy()
550 	hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask);  in in_pcbinfo_destroy()
551 	hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask);  in in_pcbinfo_destroy()
552 	hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,  in in_pcbinfo_destroy()
553 	    pcbinfo->ipi_porthashmask);  in in_pcbinfo_destroy()
554 	hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,  in in_pcbinfo_destroy()
555 	    pcbinfo->ipi_lbgrouphashmask);  in in_pcbinfo_destroy()
556 	mtx_destroy(&pcbinfo->ipi_hash_lock);  in in_pcbinfo_destroy()
557 	mtx_destroy(&pcbinfo->ipi_lock);  in in_pcbinfo_destroy()
595 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)  in in_pcballoc()  argument
602 	inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);  in in_pcballoc()
609 	inp->inp_pcbinfo = pcbinfo;  in in_pcballoc()
657 	INP_INFO_WLOCK(pcbinfo);  in in_pcballoc()
658 	pcbinfo->ipi_count++;  in in_pcballoc()
659 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;  in in_pcballoc()
660 	CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);  in in_pcballoc()
661 	INP_INFO_WUNLOCK(pcbinfo);  in in_pcballoc()
672 	uma_zfree_smr(pcbinfo->ipi_zone, inp);  in in_pcballoc()
719 	struct inpcbinfo *pcbinfo;  in in_pcb_lport_dest()  local
731 	pcbinfo = inp->inp_pcbinfo;  in in_pcb_lport_dest()
738 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcb_lport_dest()
743 		lastport = &pcbinfo->ipi_lasthi;  in in_pcb_lport_dest()
750 		lastport = &pcbinfo->ipi_lastlow;  in in_pcb_lport_dest()
754 		lastport = &pcbinfo->ipi_lastport;  in in_pcb_lport_dest()
806 				tmpinp = in_pcblookup_hash_locked(pcbinfo,  in in_pcb_lport_dest()
813 				tmpinp = in6_pcblookup_hash_locked(pcbinfo,  in in_pcb_lport_dest()
821 				tmpinp = in6_pcblookup_local(pcbinfo,  in in_pcb_lport_dest()
826 					tmpinp = in_pcblookup_local(pcbinfo,  in in_pcb_lport_dest()
835 				tmpinp = in_pcblookup_local(pcbinfo, laddr,  in in_pcb_lport_dest()
880 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbbind_setup()  local
896 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcbbind_setup()
962 				t = in_pcblookup_local(pcbinfo, sin->sin_addr,  in in_pcbbind_setup()
979 			t = in_pcblookup_local(pcbinfo, sin->sin_addr,  in in_pcbbind_setup()
1723 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbfree()  local
1743 	INP_INFO_WLOCK(pcbinfo);  in in_pcbfree()
1744 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;  in in_pcbfree()
1745 	pcbinfo->ipi_count--;  in in_pcbfree()
1747 	INP_INFO_WUNLOCK(pcbinfo);  in in_pcbfree()
1888 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,  in in_pcbnotifyall()  argument
1893 	INP_INFO_WLOCK(pcbinfo);  in in_pcbnotifyall()
1894 	CK_LIST_FOREACH_SAFE(inp, &pcbinfo->ipi_listhead, inp_list, inp_temp) {  in in_pcbnotifyall()
1910 	INP_INFO_WUNLOCK(pcbinfo);  in in_pcbnotifyall()
1924 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)  in in_pcbpurgeif0()  argument
1926 	struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,  in in_pcbpurgeif0()
1972 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,  in in_pcblookup_local()  argument
1985 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_local()
1993 		head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,  in in_pcblookup_local()
1994 		    pcbinfo->ipi_hashmask)];  in in_pcblookup_local()
2026 		porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,  in in_pcblookup_local()
2027 		    pcbinfo->ipi_porthashmask)];  in in_pcblookup_local()
2093 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,  in in_pcblookup_lbgroup()  argument
2101 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_lbgroup()
2103 	hdr = &pcbinfo->ipi_lbgrouphashbase[  in in_pcblookup_lbgroup()
2104 	    INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];  in in_pcblookup_lbgroup()
2183 in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_exact()  argument
2189 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_hash_exact()
2191 	head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,  in in_pcblookup_hash_exact()
2192 	    pcbinfo->ipi_hashmask)];  in in_pcblookup_hash_exact()
2227 in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_wild_smr()  argument
2234 	KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),  in in_pcblookup_hash_wild_smr()
2237 	head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,  in in_pcblookup_hash_wild_smr()
2238 	    pcbinfo->ipi_hashmask)];  in in_pcblookup_hash_wild_smr()
2265 in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_wild_locked()  argument
2274 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_hash_wild_locked()
2283 	head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,  in in_pcblookup_hash_wild_locked()
2284 	    pcbinfo->ipi_hashmask)];  in in_pcblookup_hash_wild_locked()
2344 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_locked()  argument
2357 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcblookup_hash_locked()
2359 	inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);  in in_pcblookup_hash_locked()
2364 		inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,  in in_pcblookup_hash_locked()
2367 			inp = in_pcblookup_hash_wild_locked(pcbinfo, faddr,  in in_pcblookup_hash_locked()
2376 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash()  argument
2386 	INP_HASH_WLOCK(pcbinfo);  in in_pcblookup_hash()
2387 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup_hash()
2391 		INP_HASH_WUNLOCK(pcbinfo);  in in_pcblookup_hash()
2397 		INP_HASH_WUNLOCK(pcbinfo);  in in_pcblookup_hash()
2403 in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_smr()  argument
2416 	smr_enter(pcbinfo->ipi_smr);  in in_pcblookup_hash_smr()
2417 	inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);  in in_pcblookup_hash_smr()
2434 		return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup_hash_smr()
2439 		inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,  in in_pcblookup_hash_smr()
2450 			inp = in_pcblookup_hash_wild_smr(pcbinfo, faddr, fport,  in in_pcblookup_hash_smr()
2454 			return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr,  in in_pcblookup_hash_smr()
2460 		smr_exit(pcbinfo->ipi_smr);  in in_pcblookup_hash_smr()
2470 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,  in in_pcblookup()  argument
2474 	return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup()
2479 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_mbuf()  argument
2483 	return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup_mbuf()
2610 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbinshash()  local
2616 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbinshash()
2623 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbinshash()
2629 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbinshash()
2634 		pcbhash = &pcbinfo->ipi_hash_exact[hash];  in in_pcbinshash()
2636 		pcbhash = &pcbinfo->ipi_hash_wild[hash];  in in_pcbinshash()
2638 	pcbporthash = &pcbinfo->ipi_porthashbase[  in in_pcbinshash()
2639 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];  in in_pcbinshash()
2663 		phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT);  in in_pcbinshash()
2682 		smr_wait(pcbinfo->ipi_smr, inp->inp_smr);  in in_pcbinshash()
2751 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbrehash()  local
2757 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbrehash()
2766 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbrehash()
2772 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbrehash()
2786 		head = &pcbinfo->ipi_hash_exact[hash];  in in_pcbrehash()
2789 		head = &pcbinfo->ipi_hash_wild[hash];  in in_pcbrehash()
2874 inp_apply_all(struct inpcbinfo *pcbinfo,  in inp_apply_all()  argument
2877 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,  in inp_apply_all()
2968 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,  in sysctl_setsockopt()  argument
2972 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,  in sysctl_setsockopt()
3016 			    pcbinfo->ipi_hashmask);  in sysctl_setsockopt()
3023 			    pcbinfo->ipi_hashmask);  in sysctl_setsockopt()