Lines Matching refs:so
168 static int soreceive_generic_locked(struct socket *so,
171 static int soreceive_rcvoob(struct socket *so, struct uio *uio,
173 static int soreceive_stream_locked(struct socket *so, struct sockbuf *sb,
176 static int sosend_generic_locked(struct socket *so, struct sockaddr *addr,
191 static int inline hhook_run_socket(struct socket *so, void *hctx, int32_t h_id);
215 #define VNET_SO_ASSERT(so) \ argument
217 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
333 static int so_unsplice(struct socket *so, bool timeout);
784 struct socket *so; in soalloc() local
786 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); in soalloc()
787 if (so == NULL) in soalloc()
790 if (mac_socket_init(so, M_NOWAIT) != 0) { in soalloc()
791 uma_zfree(socket_zone, so); in soalloc()
795 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { in soalloc()
796 uma_zfree(socket_zone, so); in soalloc()
805 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); in soalloc()
806 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); in soalloc()
807 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); in soalloc()
808 so->so_rcv.sb_sel = &so->so_rdsel; in soalloc()
809 so->so_snd.sb_sel = &so->so_wrsel; in soalloc()
810 sx_init(&so->so_snd_sx, "so_snd_sx"); in soalloc()
811 sx_init(&so->so_rcv_sx, "so_rcv_sx"); in soalloc()
812 TAILQ_INIT(&so->so_snd.sb_aiojobq); in soalloc()
813 TAILQ_INIT(&so->so_rcv.sb_aiojobq); in soalloc()
814 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); in soalloc()
815 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); in soalloc()
818 __func__, __LINE__, so)); in soalloc()
819 so->so_vnet = vnet; in soalloc()
822 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { in soalloc()
824 uma_zfree(socket_zone, so); in soalloc()
828 so->so_gencnt = ++so_gencnt; in soalloc()
835 return (so); in soalloc()
844 sodealloc(struct socket *so) in sodealloc() argument
847 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); in sodealloc()
848 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); in sodealloc()
851 so->so_gencnt = ++so_gencnt; in sodealloc()
854 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", in sodealloc()
855 __func__, __LINE__, so)); in sodealloc()
856 so->so_vnet->vnet_sockcnt--; in sodealloc()
860 mac_socket_destroy(so); in sodealloc()
862 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); in sodealloc()
864 khelp_destroy_osd(&so->osd); in sodealloc()
865 if (SOLISTENING(so)) { in sodealloc()
866 if (so->sol_accept_filter != NULL) in sodealloc()
867 accept_filt_setopt(so, NULL); in sodealloc()
869 if (so->so_rcv.sb_hiwat) in sodealloc()
870 (void)chgsbsize(so->so_cred->cr_uidinfo, in sodealloc()
871 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); in sodealloc()
872 if (so->so_snd.sb_hiwat) in sodealloc()
873 (void)chgsbsize(so->so_cred->cr_uidinfo, in sodealloc()
874 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); in sodealloc()
875 sx_destroy(&so->so_snd_sx); in sodealloc()
876 sx_destroy(&so->so_rcv_sx); in sodealloc()
877 mtx_destroy(&so->so_snd_mtx); in sodealloc()
878 mtx_destroy(&so->so_rcv_mtx); in sodealloc()
880 crfree(so->so_cred); in sodealloc()
881 mtx_destroy(&so->so_lock); in sodealloc()
882 uma_zfree(socket_zone, so); in sodealloc()
894 struct socket *so; in socreate() local
931 so = soalloc(CRED_TO_VNET(cred)); in socreate()
932 if (so == NULL) in socreate()
935 so->so_type = type; in socreate()
936 so->so_cred = crhold(cred); in socreate()
940 so->so_fibnum = td->td_proc->p_fibnum; in socreate()
942 so->so_fibnum = 0; in socreate()
943 so->so_proto = prp; in socreate()
945 mac_socket_create(cred, so); in socreate()
947 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, in socreate()
949 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, in socreate()
952 so->so_snd.sb_mtx = &so->so_snd_mtx; in socreate()
953 so->so_rcv.sb_mtx = &so->so_rcv_mtx; in socreate()
959 CURVNET_SET(so->so_vnet); in socreate()
960 error = prp->pr_attach(so, proto, td); in socreate()
963 sodealloc(so); in socreate()
966 soref(so); in socreate()
967 *aso = so; in socreate()
1000 struct socket *so; local
1140 so = soalloc(head->so_vnet);
1141 if (so == NULL) {
1147 so->so_listen = head;
1148 so->so_type = head->so_type;
1160 so->so_options = head->so_options & (SO_KEEPALIVE | SO_DONTROUTE |
1162 so->so_linger = head->so_linger;
1163 so->so_state = head->so_state;
1164 so->so_fibnum = head->so_fibnum;
1165 so->so_proto = head->so_proto;
1166 so->so_cred = crhold(head->so_cred);
1168 mac_socket_newconn(head, so);
1170 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
1172 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
1175 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) {
1176 sodealloc(so);
1181 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat;
1182 so->so_snd.sb_lowat = head->sol_sbsnd_lowat;
1183 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo;
1184 so->so_snd.sb_timeo = head->sol_sbsnd_timeo;
1185 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE;
1186 so->so_snd.sb_flags = head->sol_sbsnd_flags & SB_AUTOSIZE;
1187 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
1188 so->so_snd.sb_mtx = &so->so_snd_mtx;
1189 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
1192 return (so);
1199 struct socket *so; local
1201 if ((so = solisten_clone(head)) == NULL)
1204 if (so->so_proto->pr_attach(so, 0, NULL) != 0) {
1205 sodealloc(so);
1211 (void)solisten_enqueue(so, connstatus);
1213 return (so);
1223 solisten_enqueue(struct socket *so, int connstatus) argument
1225 struct socket *head = so->so_listen;
1227 MPASS(refcount_load(&so->so_count) == 0);
1228 refcount_init(&so->so_count, 1);
1233 so->so_state |= connstatus;
1236 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
1237 so->so_qstate = SQ_COMP;
1263 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list);
1264 so->so_qstate = SQ_INCOMP;
1281 struct socket *so; local
1285 so = soalloc(head->so_vnet);
1286 if (so == NULL) {
1292 so->so_type = head->so_type;
1293 so->so_options = head->so_options;
1294 so->so_linger = head->so_linger;
1295 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED;
1296 so->so_fibnum = head->so_fibnum;
1297 so->so_proto = head->so_proto;
1298 so->so_cred = crhold(head->so_cred);
1300 mac_socket_newconn(head, so);
1302 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
1304 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
1307 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
1308 sodealloc(so);
1313 if ((*so->so_proto->pr_attach)(so, 0, NULL)) {
1314 sodealloc(so);
1319 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
1320 so->so_snd.sb_lowat = head->so_snd.sb_lowat;
1321 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
1322 so->so_snd.sb_timeo = head->so_snd.sb_timeo;
1323 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE;
1324 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE;
1325 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
1326 so->so_snd.sb_mtx = &so->so_snd_mtx;
1327 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
1330 soref(so);
1332 return (so);
1337 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) argument
1341 CURVNET_SET(so->so_vnet);
1342 error = so->so_proto->pr_bind(so, nam, td);
1348 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) argument
1352 CURVNET_SET(so->so_vnet);
1353 error = so->so_proto->pr_bindat(fd, so, nam, td);
1371 solisten(struct socket *so, int backlog, struct thread *td) argument
1375 CURVNET_SET(so->so_vnet);
1376 error = so->so_proto->pr_listen(so, backlog, td);
1386 solisten_proto_check(struct socket *so) argument
1388 SOCK_LOCK_ASSERT(so);
1390 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1400 if (!sx_try_xlock(&so->so_snd_sx))
1402 if (!sx_try_xlock(&so->so_rcv_sx)) {
1403 sx_xunlock(&so->so_snd_sx);
1406 mtx_lock(&so->so_snd_mtx);
1407 mtx_lock(&so->so_rcv_mtx);
1410 if (!SOLISTENING(so)) {
1414 ktls = so->so_snd.sb_tls_info != NULL ||
1415 so->so_rcv.sb_tls_info != NULL;
1420 (so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 ||
1421 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0) {
1422 solisten_proto_abort(so);
1434 solisten_proto_abort(struct socket *so) argument
1436 mtx_unlock(&so->so_snd_mtx);
1437 mtx_unlock(&so->so_rcv_mtx);
1438 sx_xunlock(&so->so_snd_sx);
1439 sx_xunlock(&so->so_rcv_sx);
1443 solisten_proto(struct socket *so, int backlog) argument
1450 SOCK_LOCK_ASSERT(so);
1451 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1453 ("%s: bad socket state %p", __func__, so));
1455 if (SOLISTENING(so))
1461 sbrcv_lowat = so->so_rcv.sb_lowat;
1462 sbsnd_lowat = so->so_snd.sb_lowat;
1463 sbrcv_hiwat = so->so_rcv.sb_hiwat;
1464 sbsnd_hiwat = so->so_snd.sb_hiwat;
1465 sbrcv_flags = so->so_rcv.sb_flags;
1466 sbsnd_flags = so->so_snd.sb_flags;
1467 sbrcv_timeo = so->so_rcv.sb_timeo;
1468 sbsnd_timeo = so->so_snd.sb_timeo;
1471 mac_socketpeer_label_free(so->so_peerlabel);
1474 sbdestroy(so, SO_SND);
1475 sbdestroy(so, SO_RCV);
1478 bzero(&so->so_rcv,
1482 so->sol_sbrcv_lowat = sbrcv_lowat;
1483 so->sol_sbsnd_lowat = sbsnd_lowat;
1484 so->sol_sbrcv_hiwat = sbrcv_hiwat;
1485 so->sol_sbsnd_hiwat = sbsnd_hiwat;
1486 so->sol_sbrcv_flags = sbrcv_flags;
1487 so->sol_sbsnd_flags = sbsnd_flags;
1488 so->sol_sbrcv_timeo = sbrcv_timeo;
1489 so->sol_sbsnd_timeo = sbsnd_timeo;
1491 so->sol_qlen = so->sol_incqlen = 0;
1492 TAILQ_INIT(&so->sol_incomp);
1493 TAILQ_INIT(&so->sol_comp);
1495 so->sol_accept_filter = NULL;
1496 so->sol_accept_filter_arg = NULL;
1497 so->sol_accept_filter_str = NULL;
1499 so->sol_upcall = NULL;
1500 so->sol_upcallarg = NULL;
1502 so->so_options |= SO_ACCEPTCONN;
1507 so->sol_qlimit = backlog;
1509 mtx_unlock(&so->so_snd_mtx);
1510 mtx_unlock(&so->so_rcv_mtx);
1511 sx_xunlock(&so->so_snd_sx);
1512 sx_xunlock(&so->so_rcv_sx);
1548 struct socket *so; local
1573 so = TAILQ_FIRST(&head->sol_comp);
1574 SOCK_LOCK(so);
1575 KASSERT(so->so_qstate == SQ_COMP,
1576 ("%s: so %p not SQ_COMP", __func__, so));
1578 so->so_qstate = SQ_NONE;
1579 so->so_listen = NULL;
1580 TAILQ_REMOVE(&head->sol_comp, so, so_list);
1582 so->so_state |= (head->so_state & SS_NBIO);
1584 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
1585 SOCK_UNLOCK(so);
1588 *ret = so;
1632 so_splice(struct socket *so, struct socket *so2, struct splice *splice) argument
1640 if (so->so_proto->pr_protocol != IPPROTO_TCP ||
1643 if (so->so_vnet != so2->so_vnet)
1647 KASSERT(so->so_proto->pr_sosend == sosend_generic,
1654 so->so_splice_sent = 0;
1655 sp->src = so;
1659 SOCK_LOCK(so);
1660 if (SOLISTENING(so))
1662 else if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0)
1664 else if (so->so_splice != NULL)
1667 SOCK_UNLOCK(so);
1671 soref(so);
1672 so->so_splice = sp;
1673 SOCK_RECVBUF_LOCK(so);
1674 so->so_rcv.sb_flags |= SB_SPLICED;
1675 SOCK_RECVBUF_UNLOCK(so);
1676 SOCK_UNLOCK(so);
1688 SOCK_LOCK(so);
1689 so->so_splice = NULL;
1690 SOCK_RECVBUF_LOCK(so);
1691 so->so_rcv.sb_flags &= ~SB_SPLICED;
1692 SOCK_RECVBUF_UNLOCK(so);
1693 SOCK_UNLOCK(so);
1694 sorele(so);
1720 so_unsplice(struct socket *so, bool timeout) argument
1731 SOCK_LOCK(so);
1732 if (SOLISTENING(so)) {
1733 SOCK_UNLOCK(so);
1736 SOCK_RECVBUF_LOCK(so);
1737 if ((so->so_rcv.sb_flags & SB_SPLICED) == 0) {
1738 SOCK_RECVBUF_UNLOCK(so);
1739 SOCK_UNLOCK(so);
1742 so->so_rcv.sb_flags &= ~SB_SPLICED;
1743 sp = so->so_splice;
1744 so->so_splice = NULL;
1745 SOCK_RECVBUF_UNLOCK(so);
1746 SOCK_UNLOCK(so);
1795 sorwakeup(so);
1796 CURVNET_SET(so->so_vnet);
1797 sorele(so);
1809 sofree(struct socket *so) argument
1811 struct protosw *pr = so->so_proto;
1813 SOCK_LOCK_ASSERT(so);
1814 KASSERT(refcount_load(&so->so_count) == 0,
1815 ("%s: so %p has references", __func__, so));
1816 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE,
1817 ("%s: so %p is on listen queue", __func__, so));
1818 KASSERT(SOLISTENING(so) || (so->so_rcv.sb_flags & SB_SPLICED) == 0,
1819 ("%s: so %p rcvbuf is spliced", __func__, so));
1820 KASSERT(SOLISTENING(so) || (so->so_snd.sb_flags & SB_SPLICED) == 0,
1821 ("%s: so %p sndbuf is spliced", __func__, so));
1822 KASSERT(so->so_splice == NULL && so->so_splice_back == NULL,
1823 ("%s: so %p has spliced data", __func__, so));
1825 SOCK_UNLOCK(so);
1827 if (so->so_dtor != NULL)
1828 so->so_dtor(so);
1830 VNET_SO_ASSERT(so);
1831 if ((pr->pr_flags & PR_RIGHTS) && !SOLISTENING(so)) {
1833 (*pr->pr_domain->dom_dispose)(so);
1836 pr->pr_detach(so);
1843 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) {
1844 sbdestroy(so, SO_SND);
1845 sbdestroy(so, SO_RCV);
1847 seldrain(&so->so_rdsel);
1848 seldrain(&so->so_wrsel);
1849 knlist_destroy(&so->so_rdsel.si_note);
1850 knlist_destroy(&so->so_wrsel.si_note);
1851 sodealloc(so);
1859 sorele_locked(struct socket *so) argument
1861 SOCK_LOCK_ASSERT(so);
1862 if (refcount_release(&so->so_count))
1863 sofree(so);
1865 SOCK_UNLOCK(so);
1877 soclose(struct socket *so) argument
1883 CURVNET_SET(so->so_vnet);
1884 funsetown(&so->so_sigio);
1885 if (so->so_state & SS_ISCONNECTED) {
1886 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
1887 error = sodisconnect(so);
1895 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) {
1896 if ((so->so_state & SS_ISDISCONNECTING) &&
1897 (so->so_state & SS_NBIO))
1899 while (so->so_state & SS_ISCONNECTED) {
1900 error = tsleep(&so->so_timeo,
1902 so->so_linger * hz);
1910 if (so->so_proto->pr_close != NULL)
1911 so->so_proto->pr_close(so);
1913 SOCK_LOCK(so);
1914 if ((listening = SOLISTENING(so))) {
1918 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list);
1919 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list);
1921 so->sol_qlen = so->sol_incqlen = 0;
1928 last = refcount_release(&so->so_count);
1930 __func__, so));
1933 sorele_locked(so);
1966 soabort(struct socket *so) argument
1969 VNET_SO_ASSERT(so);
1971 if (so->so_proto->pr_abort != NULL)
1972 so->so_proto->pr_abort(so);
1973 SOCK_LOCK(so);
1974 sorele_locked(so);
1978 soaccept(struct socket *so, struct sockaddr **nam) argument
1982 CURVNET_SET(so->so_vnet);
1983 error = so->so_proto->pr_accept(so, nam);
1989 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) argument
1992 return (soconnectat(AT_FDCWD, so, nam, td));
1996 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) argument
2000 CURVNET_SET(so->so_vnet);
2010 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
2011 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
2012 (error = sodisconnect(so)))) {
2019 so->so_error = 0;
2021 error = so->so_proto->pr_connect(so, nam, td);
2023 error = so->so_proto->pr_connectat(fd, so, nam, td);
2043 sodisconnect(struct socket *so) argument
2047 if ((so->so_state & SS_ISCONNECTED) == 0)
2049 if (so->so_state & SS_ISDISCONNECTING)
2051 VNET_SO_ASSERT(so);
2052 error = so->so_proto->pr_disconnect(so);
2057 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2064 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM"));
2065 KASSERT(so->so_proto->pr_flags & PR_ATOMIC,
2085 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0;
2091 SOCKBUF_LOCK(&so->so_snd);
2092 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2093 SOCKBUF_UNLOCK(&so->so_snd);
2097 if (so->so_error) {
2098 error = so->so_error;
2099 so->so_error = 0;
2100 SOCKBUF_UNLOCK(&so->so_snd);
2103 if ((so->so_state & SS_ISCONNECTED) == 0) {
2109 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
2110 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
2111 if ((so->so_state & SS_ISCONFIRMING) == 0 &&
2113 SOCKBUF_UNLOCK(&so->so_snd);
2118 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
2122 SOCKBUF_UNLOCK(&so->so_snd);
2131 space = sbspace(&so->so_snd);
2135 SOCKBUF_UNLOCK(&so->so_snd);
2165 SOCK_LOCK(so);
2166 so->so_options |= SO_DONTROUTE;
2167 SOCK_UNLOCK(so);
2177 VNET_SO_ASSERT(so);
2178 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB :
2184 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
2192 SOCK_LOCK(so);
2193 so->so_options &= ~SO_DONTROUTE;
2194 SOCK_UNLOCK(so);
2221 sosend_generic_locked(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2227 int atomic = sosendallatonce(so) || top;
2238 SOCK_IO_SEND_ASSERT_LOCKED(so);
2256 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
2262 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
2263 (so->so_proto->pr_flags & PR_ATOMIC);
2271 tls = ktls_hold(so->so_snd.sb_tls_info);
2298 SOCKBUF_LOCK(&so->so_snd);
2299 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2300 SOCKBUF_UNLOCK(&so->so_snd);
2304 if (so->so_error) {
2305 error = so->so_error;
2306 so->so_error = 0;
2307 SOCKBUF_UNLOCK(&so->so_snd);
2310 if ((so->so_state & SS_ISCONNECTED) == 0) {
2317 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
2318 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
2319 if ((so->so_state & SS_ISCONFIRMING) == 0 &&
2321 SOCKBUF_UNLOCK(&so->so_snd);
2326 SOCKBUF_UNLOCK(&so->so_snd);
2327 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
2334 space = sbspace(&so->so_snd);
2337 if ((atomic && resid > so->so_snd.sb_hiwat) ||
2338 clen > so->so_snd.sb_hiwat) {
2339 SOCKBUF_UNLOCK(&so->so_snd);
2344 (atomic || space < so->so_snd.sb_lowat || space < clen)) {
2345 if ((so->so_state & SS_NBIO) ||
2347 SOCKBUF_UNLOCK(&so->so_snd);
2351 error = sbwait(so, SO_SND);
2352 SOCKBUF_UNLOCK(&so->so_snd);
2357 SOCKBUF_UNLOCK(&so->so_snd);
2405 SOCK_LOCK(so);
2406 so->so_options |= SO_DONTROUTE;
2407 SOCK_UNLOCK(so);
2419 VNET_SO_ASSERT(so);
2428 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
2439 error = so->so_proto->pr_send(so, pr_send_flag, top,
2443 SOCK_LOCK(so);
2444 so->so_options &= ~SO_DONTROUTE;
2445 SOCK_UNLOCK(so);
2454 soref(so);
2455 ktls_enqueue(top, so, tls_enq_cnt);
2480 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2485 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags));
2488 error = sosend_generic_locked(so, addr, uio, top, control, flags, td);
2489 SOCK_IO_SEND_UNLOCK(so);
2502 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2507 CURVNET_SET(so->so_vnet);
2508 error = so->so_proto->pr_sosend(so, addr, uio,
2518 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2527 CURVNET_SET(so->so_vnet);
2528 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags,
2539 (so->so_proto->pr_flags & PR_ATOMIC) == 0 &&
2545 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 &&
2571 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) argument
2573 struct protosw *pr = so->so_proto;
2578 VNET_SO_ASSERT(so);
2581 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK);
2647 soreceive_generic_locked(struct socket *so, struct sockaddr **psa, argument
2653 struct protosw *pr = so->so_proto;
2659 SOCK_IO_RECV_ASSERT_LOCKED(so);
2670 SOCKBUF_LOCK(&so->so_rcv);
2671 m = so->so_rcv.sb_mb;
2679 sbavail(&so->so_rcv) < uio->uio_resid) &&
2680 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat &&
2682 KASSERT(m != NULL || !sbavail(&so->so_rcv),
2684 m, sbavail(&so->so_rcv)));
2685 if (so->so_error || so->so_rerror) {
2688 if (so->so_error)
2689 error = so->so_error;
2691 error = so->so_rerror;
2693 if (so->so_error)
2694 so->so_error = 0;
2696 so->so_rerror = 0;
2698 SOCKBUF_UNLOCK(&so->so_rcv);
2701 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2702 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
2706 else if (so->so_rcv.sb_tlsdcc == 0 &&
2707 so->so_rcv.sb_tlscc == 0) {
2711 SOCKBUF_UNLOCK(&so->so_rcv);
2717 m = so->so_rcv.sb_mb;
2720 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED |
2722 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
2723 SOCKBUF_UNLOCK(&so->so_rcv);
2728 SOCKBUF_UNLOCK(&so->so_rcv);
2731 if ((so->so_state & SS_NBIO) ||
2733 SOCKBUF_UNLOCK(&so->so_rcv);
2737 SBLASTRECORDCHK(&so->so_rcv);
2738 SBLASTMBUFCHK(&so->so_rcv);
2739 error = sbwait(so, SO_RCV);
2740 SOCKBUF_UNLOCK(&so->so_rcv);
2761 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2764 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb"));
2765 SBLASTRECORDCHK(&so->so_rcv);
2766 SBLASTMBUFCHK(&so->so_rcv);
2778 sbfree(&so->so_rcv, m);
2779 so->so_rcv.sb_mb = m_free(m);
2780 m = so->so_rcv.sb_mb;
2781 sockbuf_pushsync(&so->so_rcv, nextrecord);
2813 SOCKBUF_UNLOCK(&so->so_rcv);
2830 sbfree(&so->so_rcv, m);
2831 so->so_rcv.sb_mb = m->m_next;
2835 m = so->so_rcv.sb_mb;
2839 sockbuf_pushsync(&so->so_rcv, nextrecord);
2844 SOCKBUF_UNLOCK(&so->so_rcv);
2845 VNET_SO_ASSERT(so);
2848 SOCKBUF_LOCK(&so->so_rcv);
2860 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
2862 nextrecord = so->so_rcv.sb_mb;
2870 KASSERT(so->so_rcv.sb_mb == m,
2872 KASSERT(so->so_rcv.sb_lastrecord == m,
2881 KASSERT(so->so_rcv.sb_mb == nextrecord,
2883 if (so->so_rcv.sb_mb == NULL) {
2884 KASSERT(so->so_rcv.sb_lastrecord == NULL,
2889 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2890 SBLASTRECORDCHK(&so->so_rcv);
2891 SBLASTMBUFCHK(&so->so_rcv);
2909 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2918 so->so_rcv.sb_state &= ~SBS_RCVATMARK;
2920 if (so->so_oobmark && len > so->so_oobmark - offset)
2921 len = so->so_oobmark - offset;
2932 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2933 SBLASTRECORDCHK(&so->so_rcv);
2934 SBLASTMBUFCHK(&so->so_rcv);
2935 SOCKBUF_UNLOCK(&so->so_rcv);
2942 SOCKBUF_LOCK(&so->so_rcv);
2954 (void)sbdroprecord_locked(&so->so_rcv);
2955 SOCKBUF_UNLOCK(&so->so_rcv);
2960 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2969 sbfree(&so->so_rcv, m);
2974 so->so_rcv.sb_mb = m = m->m_next;
2977 so->so_rcv.sb_mb = m_free(m);
2978 m = so->so_rcv.sb_mb;
2980 sockbuf_pushsync(&so->so_rcv, nextrecord);
2981 SBLASTRECORDCHK(&so->so_rcv);
2982 SBLASTMBUFCHK(&so->so_rcv);
3006 SOCKBUF_UNLOCK(&so->so_rcv);
3009 SOCKBUF_LOCK(&so->so_rcv);
3012 sbcut_locked(&so->so_rcv, len);
3015 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3016 if (so->so_oobmark) {
3018 so->so_oobmark -= len;
3019 if (so->so_oobmark == 0) {
3020 so->so_rcv.sb_state |= SBS_RCVATMARK;
3025 if (offset == so->so_oobmark)
3039 !sosendallatonce(so) && nextrecord == NULL) {
3040 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3041 if (so->so_error || so->so_rerror ||
3042 so->so_rcv.sb_state & SBS_CANTRCVMORE)
3049 SOCKBUF_UNLOCK(&so->so_rcv);
3050 VNET_SO_ASSERT(so);
3051 pr->pr_rcvd(so, flags);
3052 SOCKBUF_LOCK(&so->so_rcv);
3053 if (__predict_false(so->so_rcv.sb_mb == NULL &&
3054 (so->so_error || so->so_rerror ||
3055 so->so_rcv.sb_state & SBS_CANTRCVMORE)))
3058 SBLASTRECORDCHK(&so->so_rcv);
3059 SBLASTMBUFCHK(&so->so_rcv);
3064 if (so->so_rcv.sb_mb == NULL) {
3065 error = sbwait(so, SO_RCV);
3067 SOCKBUF_UNLOCK(&so->so_rcv);
3071 m = so->so_rcv.sb_mb;
3077 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3083 (void) sbdroprecord_locked(&so->so_rcv);
3092 so->so_rcv.sb_mb = nextrecord;
3093 if (so->so_rcv.sb_mb == NULL) {
3094 so->so_rcv.sb_mbtail = NULL;
3095 so->so_rcv.sb_lastrecord = NULL;
3097 so->so_rcv.sb_lastrecord = nextrecord;
3099 SBLASTRECORDCHK(&so->so_rcv);
3100 SBLASTMBUFCHK(&so->so_rcv);
3108 SOCKBUF_UNLOCK(&so->so_rcv);
3109 VNET_SO_ASSERT(so);
3110 pr->pr_rcvd(so, flags);
3111 SOCKBUF_LOCK(&so->so_rcv);
3114 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3116 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
3117 SOCKBUF_UNLOCK(&so->so_rcv);
3120 SOCKBUF_UNLOCK(&so->so_rcv);
3129 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3141 return (soreceive_rcvoob(so, uio, flags));
3147 if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
3148 (so->so_state & SS_ISCONFIRMING) && uio->uio_resid) {
3149 VNET_SO_ASSERT(so);
3150 so->so_proto->pr_rcvd(so, 0);
3153 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
3156 error = soreceive_generic_locked(so, psa, uio, mp, controlp, flagsp);
3157 SOCK_IO_RECV_UNLOCK(so);
3165 soreceive_stream_locked(struct socket *so, struct sockbuf *sb, argument
3172 SOCK_IO_RECV_ASSERT_LOCKED(so);
3181 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
3187 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3190 if (so->so_error) {
3195 error = so->so_error;
3197 so->so_error = 0;
3211 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) {
3218 ((so->so_state & SS_NBIO) ||
3235 error = sbwait(so, SO_RCV);
3241 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3313 if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
3317 VNET_SO_ASSERT(so);
3318 so->so_proto->pr_rcvd(so, flags);
3337 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3343 sb = &so->so_rcv;
3346 if (so->so_type != SOCK_STREAM)
3357 return (soreceive_rcvoob(so, uio, flags));
3370 return (soreceive_generic(so, psa, uio, mp0, controlp,
3378 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
3383 SOCK_IO_RECV_UNLOCK(so);
3384 return (soreceive_generic(so, psa, uio, mp0, controlp,
3388 error = soreceive_stream_locked(so, sb, psa, uio, mp0, controlp, flags);
3389 SOCK_IO_RECV_UNLOCK(so);
3400 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3406 struct protosw *pr = so->so_proto;
3423 return (soreceive_generic(so, psa, uio, mp0, controlp,
3432 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0,
3434 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0,
3440 SOCKBUF_LOCK(&so->so_rcv);
3441 while ((m = so->so_rcv.sb_mb) == NULL) {
3442 KASSERT(sbavail(&so->so_rcv) == 0,
3444 sbavail(&so->so_rcv)));
3445 if (so->so_error) {
3446 error = so->so_error;
3447 so->so_error = 0;
3448 SOCKBUF_UNLOCK(&so->so_rcv);
3451 if (so->so_rcv.sb_state & SBS_CANTRCVMORE ||
3453 SOCKBUF_UNLOCK(&so->so_rcv);
3456 if ((so->so_state & SS_NBIO) ||
3458 SOCKBUF_UNLOCK(&so->so_rcv);
3461 SBLASTRECORDCHK(&so->so_rcv);
3462 SBLASTMBUFCHK(&so->so_rcv);
3463 error = sbwait(so, SO_RCV);
3465 SOCKBUF_UNLOCK(&so->so_rcv);
3469 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3473 SBLASTRECORDCHK(&so->so_rcv);
3474 SBLASTMBUFCHK(&so->so_rcv);
3477 KASSERT(so->so_rcv.sb_lastrecord == m,
3481 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord,
3487 so->so_rcv.sb_mb = NULL;
3488 sockbuf_pushsync(&so->so_rcv, nextrecord);
3494 sbfree(&so->so_rcv, m2);
3499 SBLASTRECORDCHK(&so->so_rcv);
3500 SBLASTMBUFCHK(&so->so_rcv);
3501 SOCKBUF_UNLOCK(&so->so_rcv);
3582 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3587 CURVNET_SET(so->so_vnet);
3588 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp);
3594 soshutdown(struct socket *so, int how) argument
3603 SOCK_LOCK(so);
3604 if ((so->so_state &
3615 if (so->so_type != SOCK_DGRAM && !SOLISTENING(so)) {
3616 SOCK_UNLOCK(so);
3622 if (SOLISTENING(so)) {
3624 so->so_error = ECONNABORTED;
3625 solisten_wakeup(so); /* unlocks so */
3627 SOCK_UNLOCK(so);
3631 SOCK_UNLOCK(so);
3633 CURVNET_SET(so->so_vnet);
3634 pr = so->so_proto;
3636 pr->pr_flush(so, how);
3638 sorflush(so);
3640 error = pr->pr_shutdown(so);
3641 wakeup(&so->so_timeo);
3645 wakeup(&so->so_timeo);
3653 sorflush(struct socket *so) argument
3658 VNET_SO_ASSERT(so);
3666 socantrcvmore(so);
3668 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR);
3670 KASSERT(SOLISTENING(so),
3671 ("%s: soiolock(%p) failed", __func__, so));
3675 pr = so->so_proto;
3678 (*pr->pr_domain->dom_dispose)(so);
3680 sbrelease(so, SO_RCV);
3681 SOCK_IO_RECV_UNLOCK(so);
3691 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) argument
3694 .so = so,
3700 CURVNET_SET(so->so_vnet);
3701 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd);
3744 so_setsockopt(struct socket *so, int level, int optname, void *optval, argument
3755 return (sosetopt(so, &sopt));
3759 sosetopt(struct socket *so, struct sockopt *sopt) argument
3770 CURVNET_SET(so->so_vnet);
3773 if (so->so_proto->pr_ctloutput != NULL)
3774 error = (*so->so_proto->pr_ctloutput)(so, sopt);
3780 error = accept_filt_setopt(so, sopt);
3795 SOCK_LOCK(so);
3796 so->so_linger = l.l_linger;
3798 so->so_options |= SO_LINGER;
3800 so->so_options &= ~SO_LINGER;
3801 SOCK_UNLOCK(so);
3823 SOCK_LOCK(so);
3825 so->so_options |= sopt->sopt_name;
3827 so->so_options &= ~sopt->sopt_name;
3828 SOCK_UNLOCK(so);
3841 if (((so->so_proto->pr_domain->dom_family == PF_INET) ||
3842 (so->so_proto->pr_domain->dom_family == PF_INET6) ||
3843 (so->so_proto->pr_domain->dom_family == PF_ROUTE)))
3844 so->so_fibnum = optval;
3846 so->so_fibnum = 0;
3854 so->so_user_cookie = val32;
3861 error = so->so_proto->pr_setsbopt(so, sopt);
3891 SOCK_LOCK(so);
3893 (SOLISTENING(so) ? &so->sol_sbsnd_timeo :
3894 &so->so_snd.sb_timeo) :
3895 (SOLISTENING(so) ? &so->sol_sbrcv_timeo :
3896 &so->so_rcv.sb_timeo);
3898 SOCK_UNLOCK(so);
3908 so, &extmac);
3923 so->so_ts_clock = optval;
3931 so->so_max_pacing_rate = val32;
3983 error = so_splice(so, so2, &splice);
3986 error = so_unsplice(so, false);
3992 error = hhook_run_socket(so, sopt,
3998 if (error == 0 && so->so_proto->pr_ctloutput != NULL)
3999 (void)(*so->so_proto->pr_ctloutput)(so, sopt);
4037 sogetopt(struct socket *so, struct sockopt *sopt) argument
4046 CURVNET_SET(so->so_vnet);
4049 if (so->so_proto->pr_ctloutput != NULL)
4050 error = (*so->so_proto->pr_ctloutput)(so, sopt);
4058 error = accept_filt_getopt(so, sopt);
4062 SOCK_LOCK(so);
4063 l.l_onoff = so->so_options & SO_LINGER;
4064 l.l_linger = so->so_linger;
4065 SOCK_UNLOCK(so);
4085 optval = so->so_options & sopt->sopt_name;
4091 optval = so->so_proto->pr_domain->dom_family;
4095 optval = so->so_type;
4099 optval = so->so_proto->pr_protocol;
4103 SOCK_LOCK(so);
4104 if (so->so_error) {
4105 optval = so->so_error;
4106 so->so_error = 0;
4108 optval = so->so_rerror;
4109 so->so_rerror = 0;
4111 SOCK_UNLOCK(so);
4115 SOCK_LOCK(so);
4116 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat :
4117 so->so_snd.sb_hiwat;
4118 SOCK_UNLOCK(so);
4122 SOCK_LOCK(so);
4123 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat :
4124 so->so_rcv.sb_hiwat;
4125 SOCK_UNLOCK(so);
4129 SOCK_LOCK(so);
4130 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat :
4131 so->so_snd.sb_lowat;
4132 SOCK_UNLOCK(so);
4136 SOCK_LOCK(so);
4137 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat :
4138 so->so_rcv.sb_lowat;
4139 SOCK_UNLOCK(so);
4144 SOCK_LOCK(so);
4146 (SOLISTENING(so) ? so->sol_sbsnd_timeo :
4147 so->so_snd.sb_timeo) :
4148 (SOLISTENING(so) ? so->sol_sbrcv_timeo :
4149 so->so_rcv.sb_timeo));
4150 SOCK_UNLOCK(so);
4170 so, &extmac);
4186 sopt->sopt_td->td_ucred, so, &extmac);
4196 SOCK_LOCK(so);
4197 optval = SOLISTENING(so) ? so->sol_qlimit : 0;
4198 SOCK_UNLOCK(so);
4202 SOCK_LOCK(so);
4203 optval = SOLISTENING(so) ? so->sol_qlen : 0;
4204 SOCK_UNLOCK(so);
4208 SOCK_LOCK(so);
4209 optval = SOLISTENING(so) ? so->sol_incqlen : 0;
4210 SOCK_UNLOCK(so);
4214 optval = so->so_ts_clock;
4218 optval = so->so_max_pacing_rate;
4233 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT);
4236 SOCK_LOCK(so);
4237 if (SOLISTENING(so)) {
4240 n = so->so_splice_sent;
4242 SOCK_UNLOCK(so);
4243 SOCK_IO_RECV_UNLOCK(so);
4250 error = hhook_run_socket(so, sopt,
4377 sohasoutofband(struct socket *so) argument
4380 if (so->so_sigio != NULL)
4381 pgsigio(&so->so_sigio, SIGURG, 0);
4382 selwakeuppri(&so->so_rdsel, PSOCK);
4386 sopoll(struct socket *so, int events, struct ucred *active_cred, argument
4394 return (so->so_proto->pr_sopoll(so, events, active_cred, td));
4398 sopoll_generic(struct socket *so, int events, struct ucred *active_cred, argument
4403 SOCK_LOCK(so);
4404 if (SOLISTENING(so)) {
4407 else if (!TAILQ_EMPTY(&so->sol_comp))
4409 else if ((events & POLLINIGNEOF) == 0 && so->so_error)
4412 selrecord(td, &so->so_rdsel);
4417 SOCK_SENDBUF_LOCK(so);
4418 SOCK_RECVBUF_LOCK(so);
4420 if (soreadabledata(so) && !isspliced(so))
4423 if (sowriteable(so) && !issplicedback(so))
4426 if (so->so_oobmark ||
4427 (so->so_rcv.sb_state & SBS_RCVATMARK))
4430 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
4432 if (so->so_snd.sb_state & SBS_CANTSENDMORE)
4436 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
4441 selrecord(td, &so->so_rdsel);
4442 so->so_rcv.sb_flags |= SB_SEL;
4445 selrecord(td, &so->so_wrsel);
4446 so->so_snd.sb_flags |= SB_SEL;
4449 SOCK_RECVBUF_UNLOCK(so);
4450 SOCK_SENDBUF_UNLOCK(so);
4452 SOCK_UNLOCK(so);
4459 struct socket *so = kn->kn_fp->f_data; local
4467 knl = &so->so_rdsel.si_note;
4468 sb = &so->so_rcv;
4473 knl = &so->so_wrsel.si_note;
4474 sb = &so->so_snd;
4479 knl = &so->so_wrsel.si_note;
4480 sb = &so->so_snd;
4487 SOCK_LOCK(so);
4488 if (SOLISTENING(so)) {
4491 SOCK_BUF_LOCK(so, which);
4494 SOCK_BUF_UNLOCK(so, which);
4496 SOCK_UNLOCK(so);
4503 struct socket *so = kn->kn_fp->f_data; local
4505 so_rdknl_lock(so);
4506 knlist_remove(&so->so_rdsel.si_note, kn, 1);
4507 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note))
4508 so->so_rcv.sb_flags &= ~SB_KNOTE;
4509 so_rdknl_unlock(so);
4516 struct socket *so; local
4518 so = kn->kn_fp->f_data;
4520 if (SOLISTENING(so)) {
4521 SOCK_LOCK_ASSERT(so);
4522 kn->kn_data = so->sol_qlen;
4523 if (so->so_error) {
4525 kn->kn_fflags = so->so_error;
4528 return (!TAILQ_EMPTY(&so->sol_comp));
4531 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0)
4534 SOCK_RECVBUF_LOCK_ASSERT(so);
4536 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl;
4537 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
4539 kn->kn_fflags = so->so_error;
4541 } else if (so->so_error || so->so_rerror)
4547 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat)
4551 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD));
4557 struct socket *so = kn->kn_fp->f_data; local
4559 so_wrknl_lock(so);
4560 knlist_remove(&so->so_wrsel.si_note, kn, 1);
4561 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note))
4562 so->so_snd.sb_flags &= ~SB_KNOTE;
4563 so_wrknl_unlock(so);
4570 struct socket *so; local
4572 so = kn->kn_fp->f_data;
4574 if (SOLISTENING(so))
4577 SOCK_SENDBUF_LOCK_ASSERT(so);
4578 kn->kn_data = sbspace(&so->so_snd);
4580 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE);
4582 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
4584 kn->kn_fflags = so->so_error;
4586 } else if (so->so_error) /* temporary udp error */
4588 else if (((so->so_state & SS_ISCONNECTED) == 0) &&
4589 (so->so_proto->pr_flags & PR_CONNREQUIRED))
4594 return (kn->kn_data >= so->so_snd.sb_lowat);
4600 struct socket *so; local
4602 so = kn->kn_fp->f_data;
4604 if (SOLISTENING(so))
4607 SOCK_SENDBUF_LOCK_ASSERT(so);
4608 kn->kn_data = sbused(&so->so_snd);
4617 socheckuid(struct socket *so, uid_t uid) argument
4620 if (so == NULL)
4622 if (so->so_cred->cr_uid != uid)
4660 soisconnecting(struct socket *so) argument
4663 SOCK_LOCK(so);
4664 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
4665 so->so_state |= SS_ISCONNECTING;
4666 SOCK_UNLOCK(so);
4670 soisconnected(struct socket *so) argument
4674 SOCK_LOCK(so);
4675 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
4676 so->so_state |= SS_ISCONNECTED;
4678 if (so->so_qstate == SQ_INCOMP) {
4679 struct socket *head = so->so_listen;
4682 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so));
4692 SOCK_UNLOCK(so);
4694 SOCK_LOCK(so);
4695 if (__predict_false(head != so->so_listen)) {
4701 SOCK_UNLOCK(so);
4710 if ((so->so_options & SO_ACCEPTFILTER) == 0) {
4711 TAILQ_REMOVE(&head->sol_incomp, so, so_list);
4713 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
4715 so->so_qstate = SQ_COMP;
4716 SOCK_UNLOCK(so);
4719 SOCK_RECVBUF_LOCK(so);
4720 soupcall_set(so, SO_RCV,
4723 so->so_options &= ~SO_ACCEPTFILTER;
4724 ret = head->sol_accept_filter->accf_callback(so,
4727 soupcall_clear(so, SO_RCV);
4728 SOCK_RECVBUF_UNLOCK(so);
4731 SOCK_RECVBUF_UNLOCK(so);
4732 SOCK_UNLOCK(so);
4737 SOCK_UNLOCK(so);
4738 wakeup(&so->so_timeo);
4739 sorwakeup(so);
4740 sowwakeup(so);
4744 soisdisconnecting(struct socket *so) argument
4747 SOCK_LOCK(so);
4748 so->so_state &= ~SS_ISCONNECTING;
4749 so->so_state |= SS_ISDISCONNECTING;
4751 if (!SOLISTENING(so)) {
4752 SOCK_RECVBUF_LOCK(so);
4753 socantrcvmore_locked(so);
4754 SOCK_SENDBUF_LOCK(so);
4755 socantsendmore_locked(so);
4757 SOCK_UNLOCK(so);
4758 wakeup(&so->so_timeo);
4762 soisdisconnected(struct socket *so) argument
4765 SOCK_LOCK(so);
4774 so->so_state |= SS_ISDISCONNECTED;
4776 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
4778 if (!SOLISTENING(so)) {
4779 SOCK_UNLOCK(so);
4780 SOCK_RECVBUF_LOCK(so);
4781 socantrcvmore_locked(so);
4782 SOCK_SENDBUF_LOCK(so);
4783 sbdrop_locked(&so->so_snd, sbused(&so->so_snd));
4784 socantsendmore_locked(so);
4786 SOCK_UNLOCK(so);
4787 wakeup(&so->so_timeo);
4791 soiolock(struct socket *so, struct sx *sx, int flags) argument
4810 if (__predict_false(SOLISTENING(so))) {
4841 sodtor_set(struct socket *so, so_dtor_t *func) argument
4844 SOCK_LOCK_ASSERT(so);
4845 so->so_dtor = func;
4852 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg) argument
4856 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4860 sb = &so->so_rcv;
4863 sb = &so->so_snd;
4866 SOCK_BUF_LOCK_ASSERT(so, which);
4873 soupcall_clear(struct socket *so, sb_which which) argument
4877 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4881 sb = &so->so_rcv;
4884 sb = &so->so_snd;
4887 SOCK_BUF_LOCK_ASSERT(so, which);
4889 ("%s: so %p no upcall to clear", __func__, so));
4896 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) argument
4899 SOLISTEN_LOCK_ASSERT(so);
4900 so->sol_upcall = func;
4901 so->sol_upcallarg = arg;
4907 struct socket *so = arg; local
4910 if (SOLISTENING(so)) {
4911 SOLISTEN_LOCK(so);
4913 SOCK_RECVBUF_LOCK(so);
4914 if (__predict_false(SOLISTENING(so))) {
4915 SOCK_RECVBUF_UNLOCK(so);
4924 struct socket *so = arg; local
4926 if (SOLISTENING(so))
4927 SOLISTEN_UNLOCK(so);
4929 SOCK_RECVBUF_UNLOCK(so);
4935 struct socket *so = arg; local
4938 if (SOLISTENING(so))
4939 SOLISTEN_LOCK_ASSERT(so);
4941 SOCK_RECVBUF_LOCK_ASSERT(so);
4943 if (SOLISTENING(so))
4944 SOLISTEN_UNLOCK_ASSERT(so);
4946 SOCK_RECVBUF_UNLOCK_ASSERT(so);
4953 struct socket *so = arg; local
4956 if (SOLISTENING(so)) {
4957 SOLISTEN_LOCK(so);
4959 SOCK_SENDBUF_LOCK(so);
4960 if (__predict_false(SOLISTENING(so))) {
4961 SOCK_SENDBUF_UNLOCK(so);
4970 struct socket *so = arg; local
4972 if (SOLISTENING(so))
4973 SOLISTEN_UNLOCK(so);
4975 SOCK_SENDBUF_UNLOCK(so);
4981 struct socket *so = arg; local
4984 if (SOLISTENING(so))
4985 SOLISTEN_LOCK_ASSERT(so);
4987 SOCK_SENDBUF_LOCK_ASSERT(so);
4989 if (SOLISTENING(so))
4990 SOLISTEN_UNLOCK_ASSERT(so);
4992 SOCK_SENDBUF_UNLOCK_ASSERT(so);
5005 sotoxsocket(struct socket *so, struct xsocket *xso) argument
5010 xso->xso_so = (uintptr_t)so;
5011 xso->so_type = so->so_type;
5012 xso->so_options = so->so_options;
5013 xso->so_linger = so->so_linger;
5014 xso->so_state = so->so_state;
5015 xso->so_pcb = (uintptr_t)so->so_pcb;
5016 xso->xso_protocol = so->so_proto->pr_protocol;
5017 xso->xso_family = so->so_proto->pr_domain->dom_family;
5018 xso->so_timeo = so->so_timeo;
5019 xso->so_error = so->so_error;
5020 xso->so_uid = so->so_cred->cr_uid;
5021 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0;
5022 SOCK_LOCK(so);
5023 if (SOLISTENING(so)) {
5024 xso->so_qlen = so->sol_qlen;
5025 xso->so_incqlen = so->sol_incqlen;
5026 xso->so_qlimit = so->sol_qlimit;
5029 xso->so_state |= so->so_qstate;
5031 xso->so_oobmark = so->so_oobmark;
5032 sbtoxsockbuf(&so->so_snd, &xso->so_snd);
5033 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv);
5034 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0)
5035 xso->so_splice_so = (uintptr_t)so->so_splice->dst;
5037 SOCK_UNLOCK(so);
5041 so_sockbuf_rcv(struct socket *so) argument
5044 return (&so->so_rcv);
5048 so_sockbuf_snd(struct socket *so) argument
5051 return (&so->so_snd);
5055 so_state_get(const struct socket *so) argument
5058 return (so->so_state);
5062 so_state_set(struct socket *so, int val) argument
5065 so->so_state = val;
5069 so_options_get(const struct socket *so) argument
5072 return (so->so_options);
5076 so_options_set(struct socket *so, int val) argument
5079 so->so_options = val;
5083 so_error_get(const struct socket *so) argument
5086 return (so->so_error);
5090 so_error_set(struct socket *so, int val) argument
5093 so->so_error = val;
5097 so_linger_get(const struct socket *so) argument
5100 return (so->so_linger);
5104 so_linger_set(struct socket *so, int val) argument
5110 so->so_linger = val;
5114 so_protosw_get(const struct socket *so) argument
5117 return (so->so_proto);
5121 so_protosw_set(struct socket *so, struct protosw *val) argument
5124 so->so_proto = val;
5128 so_sorwakeup(struct socket *so) argument
5131 sorwakeup(so);
5135 so_sowwakeup(struct socket *so) argument
5138 sowwakeup(so);
5142 so_sorwakeup_locked(struct socket *so) argument
5145 sorwakeup_locked(so);
5149 so_sowwakeup_locked(struct socket *so) argument
5152 sowwakeup_locked(so);
5156 so_lock(struct socket *so) argument
5159 SOCK_LOCK(so);
5163 so_unlock(struct socket *so) argument
5166 SOCK_UNLOCK(so);