279 hhook_run_tcp_est_in(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to)  in hhook_run_tcp_est_in()  argument
285 		hhook_data.th = th;  in hhook_run_tcp_est_in()
298 cc_ack_received(struct tcpcb *tp, struct tcphdr *th, uint16_t nsegs,  in cc_ack_received()  argument
304 	tp->ccv->bytes_this_ack = BYTES_THIS_ACK(tp, th);  in cc_ack_received()
326 		tp->ccv->curack = th->th_ack;  in cc_ack_received()
405 cc_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type)  in cc_cong_signal()  argument
453 		if (th != NULL)  in cc_cong_signal()
454 			tp->ccv->curack = th->th_ack;  in cc_cong_signal()
460 cc_post_recovery(struct tcpcb *tp, struct tcphdr *th)  in cc_post_recovery()  argument
467 		tp->ccv->curack = th->th_ack;  in cc_post_recovery()
490 cc_ecnpkt_handler(struct tcpcb *tp, struct tcphdr *th, uint8_t iptos)  in cc_ecnpkt_handler()  argument
507 		if (th->th_flags & TH_CWR)  in cc_ecnpkt_handler()
569 	struct tcphdr *th = NULL;  in tcp_input()  local
621 		if (m->m_len < (sizeof(*ip6) + sizeof(*th))) {  in tcp_input()
622 			m = m_pullup(m, sizeof(*ip6) + sizeof(*th));  in tcp_input()
630 		th = (struct tcphdr *)((caddr_t)ip6 + off0);  in tcp_input()
634 				th->th_sum = m->m_pkthdr.csum_data;  in tcp_input()
636 				th->th_sum = in6_cksum_pseudo(ip6, tlen,  in tcp_input()
638 			th->th_sum ^= 0xffff;  in tcp_input()
640 			th->th_sum = in6_cksum(m, IPPROTO_TCP, off0, tlen);  in tcp_input()
641 		if (th->th_sum) {  in tcp_input()
682 		th = (struct tcphdr *)((caddr_t)ip + off0);  in tcp_input()
688 				th->th_sum = m->m_pkthdr.csum_data;  in tcp_input()
690 				th->th_sum = in_pseudo(ip->ip_src.s_addr,  in tcp_input()
694 			th->th_sum ^= 0xffff;  in tcp_input()
705 			th->th_sum = in_cksum(m, len);  in tcp_input()
716 		if (th->th_sum) {  in tcp_input()
727 	off = th->th_off << 2;  in tcp_input()
738 			th = (struct tcphdr *)((caddr_t)ip6 + off0);  in tcp_input()
753 				th = (struct tcphdr *)((caddr_t)ip + off0);  in tcp_input()
758 		optp = (u_char *)(th + 1);  in tcp_input()
760 	thflags = th->th_flags;  in tcp_input()
765 	tcp_fields_to_host(th);  in tcp_input()
819 		    &ip6->ip6_src, th->th_sport, &ip6->ip6_dst, th->th_dport,  in tcp_input()
828 			    th->th_sport, &next_hop6->sin6_addr,  in tcp_input()
830 			    th->th_dport, INPLOOKUP_WILDCARD |  in tcp_input()
835 		    th->th_sport, &ip6->ip6_dst, th->th_dport,  in tcp_input()
852 		inp = in_pcblookup_mbuf(&V_tcbinfo, ip->ip_src, th->th_sport,  in tcp_input()
853 		    ip->ip_dst, th->th_dport, INPLOOKUP_WLOCKPCB,  in tcp_input()
862 			    th->th_sport, next_hop->sin_addr,  in tcp_input()
864 			    th->th_dport, INPLOOKUP_WILDCARD |  in tcp_input()
869 		    th->th_sport, ip->ip_dst, th->th_dport,  in tcp_input()
886 			if ((s = tcp_log_vain(NULL, th, (void *)ip, ip6)))  in tcp_input()
979 		if (tcp_twcheck(inp, &to, th, m, tlen))  in tcp_input()
1040 		tcp_savetcp = *th;  in tcp_input()
1067 		inc.inc_fport = th->th_sport;  in tcp_input()
1068 		inc.inc_lport = th->th_dport;  in tcp_input()
1089 			rstreason = syncache_expand(&inc, &to, th, &so, m);  in tcp_input()
1120 				if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1153 			TCP_PROBE5(receive, NULL, tp, m, tp, th);  in tcp_input()
1154 			tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen,  in tcp_input()
1171 			syncache_chkrst(&inc, th, m);  in tcp_input()
1178 			if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1189 			if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1210 			if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1266 				if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1290 			if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1298 			if (th->th_dport == th->th_sport &&  in tcp_input()
1300 				if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1308 				if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1321 			if (th->th_dport == th->th_sport &&  in tcp_input()
1323 				if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1333 				if ((s = tcp_log_addrs(&inc, th, NULL, NULL)))  in tcp_input()
1351 		TCP_PROBE3(debug__input, tp, th, m);  in tcp_input()
1353 		if (syncache_add(&inc, &to, th, inp, &so, m, NULL, NULL))  in tcp_input()
1384 		    TCPMD5_INPUT(m, th, to.to_signature) != 0)  in tcp_input()
1388 	TCP_PROBE5(receive, NULL, tp, m, tp, th);  in tcp_input()
1395 	tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, iptos);  in tcp_input()
1401 	TCP_PROBE5(receive, NULL, tp, m, tp, th);  in tcp_input()
1416 		tcp_dropwithreset(m, th, tp, tlen, rstreason);  in tcp_input()
1419 		tcp_dropwithreset(m, th, NULL, tlen, rstreason);  in tcp_input()
1425 		TCP_PROBE5(receive, NULL, tp, m, tp, th);  in tcp_input()
1487 tcp_autorcvbuf(struct mbuf *m, struct tcphdr *th, struct socket *so,  in tcp_autorcvbuf()  argument
1501 		TCP_PROBE6(receive__autoresize, NULL, tp, m, tp, th, newsize);  in tcp_autorcvbuf()
1513 tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,  in tcp_do_segment()  argument
1535 	thflags = th->th_flags;  in tcp_do_segment()
1558 	tcp_pcap_add(th, m, &(tp->t_inpkts));  in tcp_do_segment()
1560 	TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0,  in tcp_do_segment()
1564 		if ((s = tcp_log_addrs(inc, th, NULL, NULL))) {  in tcp_do_segment()
1578 	    (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {  in tcp_do_segment()
1595 	tiwin = th->th_win << tp->snd_scale;  in tcp_do_segment()
1617 		cc_ecnpkt_handler(tp, th, iptos);  in tcp_do_segment()
1621 			cc_cong_signal(tp, th, CC_ECN);  in tcp_do_segment()
1628 	tcp_dooptions(&to, (u_char *)(th + 1),  in tcp_do_segment()
1629 	    (th->th_off << 2) - sizeof(struct tcphdr),  in tcp_do_segment()
1651 			cc_cong_signal(tp, th, CC_RTO_ERR);  in tcp_do_segment()
1670 		tp->snd_wnd = th->th_win;  in tcp_do_segment()
1704 		if ((s = tcp_log_addrs(inc, th, NULL, NULL))) {  in tcp_do_segment()
1711 		if ((s = tcp_log_addrs(inc, th, NULL, NULL))) {  in tcp_do_segment()
1736 	    th->th_seq == tp->rcv_nxt &&  in tcp_do_segment()
1752 		    SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {  in tcp_do_segment()
1758 			if (SEQ_GT(th->th_ack, tp->snd_una) &&  in tcp_do_segment()
1759 			    SEQ_LEQ(th->th_ack, tp->snd_max) &&  in tcp_do_segment()
1775 					cc_cong_signal(tp, th, CC_RTO_ERR);  in tcp_do_segment()
1796 				    SEQ_GT(th->th_ack, tp->t_rtseq)) {  in tcp_do_segment()
1803 				acked = BYTES_THIS_ACK(tp, th);  in tcp_do_segment()
1807 				hhook_run_tcp_est_in(tp, th, &to);  in tcp_do_segment()
1814 				    SEQ_LEQ(th->th_ack, tp->snd_recover))  in tcp_do_segment()
1815 					tp->snd_recover = th->th_ack - 1;  in tcp_do_segment()
1823 				cc_ack_received(tp, th, nsegs, CC_ACK);  in tcp_do_segment()
1825 				tp->snd_una = th->th_ack;  in tcp_do_segment()
1830 				tp->snd_wl2 = th->th_ack;  in tcp_do_segment()
1849 				TCP_PROBE3(debug__input, tp, th, m);  in tcp_do_segment()
1860 		} else if (th->th_ack == tp->snd_una &&  in tcp_do_segment()
1878 			tp->snd_wl1 = th->th_seq;  in tcp_do_segment()
1891 			TCP_PROBE3(debug__input, tp, th, m);  in tcp_do_segment()
1893 			newsize = tcp_autorcvbuf(m, th, so, tp, tlen);  in tcp_do_segment()
1942 		    (SEQ_LEQ(th->th_ack, tp->snd_una) ||  in tcp_do_segment()
1943 		     SEQ_GT(th->th_ack, tp->snd_max))) {  in tcp_do_segment()
1987 			    m, tp, th);  in tcp_do_segment()
1995 		tp->irs = th->th_seq;  in tcp_do_segment()
2019 				tp->snd_nxt = th->th_ack;  in tcp_do_segment()
2052 				    m, tp, th);  in tcp_do_segment()
2079 		th->th_seq++;  in tcp_do_segment()
2088 		tp->snd_wl1 = th->th_seq - 1;  in tcp_do_segment()
2089 		tp->rcv_up = th->th_seq;  in tcp_do_segment()
2140 		if ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&  in tcp_do_segment()
2141 		    SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) ||  in tcp_do_segment()
2142 		    (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) {  in tcp_do_segment()
2147 			    __func__, th, tp));  in tcp_do_segment()
2150 			    tp->last_ack_sent == th->th_seq) {  in tcp_do_segment()
2172 				tcp_respond(tp, mtod(m, void *), th, m,  in tcp_do_segment()
2191 		    SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&  in tcp_do_segment()
2192 		    SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) {  in tcp_do_segment()
2197 			tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt,  in tcp_do_segment()
2243 	if (tp->t_state == TCPS_SYN_RECEIVED && SEQ_LT(th->th_seq, tp->irs)) {  in tcp_do_segment()
2248 	todrop = tp->rcv_nxt - th->th_seq;  in tcp_do_segment()
2252 			th->th_seq++;  in tcp_do_segment()
2253 			if (th->th_urp > 1)  in tcp_do_segment()
2254 				th->th_urp--;  in tcp_do_segment()
2287 			tcp_update_sack_list(tp, th->th_seq,  in tcp_do_segment()
2288 			    th->th_seq + todrop);  in tcp_do_segment()
2296 		th->th_seq += todrop;  in tcp_do_segment()
2298 		if (th->th_urp > todrop)  in tcp_do_segment()
2299 			th->th_urp -= todrop;  in tcp_do_segment()
2302 			th->th_urp = 0;  in tcp_do_segment()
2314 		if ((s = tcp_log_addrs(inc, th, NULL, NULL))) {  in tcp_do_segment()
2331 	todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd);  in tcp_do_segment()
2343 			if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) {  in tcp_do_segment()
2374 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in tcp_do_segment()
2375 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in tcp_do_segment()
2443 			    m, tp, th);  in tcp_do_segment()
2462 		tp->snd_wl1 = th->th_seq - 1;  in tcp_do_segment()
2479 		if (SEQ_GT(th->th_ack, tp->snd_max)) {  in tcp_do_segment()
2486 			sack_changed = tcp_sack_doack(tp, &to, th->th_ack);  in tcp_do_segment()
2496 		hhook_run_tcp_est_in(tp, th, &to);  in tcp_do_segment()
2499 		if (SEQ_LEQ(th->th_ack, tp->snd_una)) {  in tcp_do_segment()
2556 				if (th->th_ack != tp->snd_una ||  in tcp_do_segment()
2564 					cc_ack_received(tp, th, nsegs,  in tcp_do_segment()
2607 						if (SEQ_LEQ(th->th_ack,  in tcp_do_segment()
2614 					cc_cong_signal(tp, th, CC_NDUPACK);  in tcp_do_segment()
2615 					cc_ack_received(tp, th, nsegs,  in tcp_do_segment()
2627 					tp->snd_nxt = th->th_ack;  in tcp_do_segment()
2650 					cc_ack_received(tp, th, nsegs,  in tcp_do_segment()
2708 		KASSERT(SEQ_GT(th->th_ack, tp->snd_una),  in tcp_do_segment()
2716 			if (SEQ_LT(th->th_ack, tp->snd_recover)) {  in tcp_do_segment()
2718 					tcp_sack_partialack(tp, th);  in tcp_do_segment()
2720 					tcp_newreno_partial_ack(tp, th);  in tcp_do_segment()
2722 				cc_post_recovery(tp, th);  in tcp_do_segment()
2749 		acked = BYTES_THIS_ACK(tp, th);  in tcp_do_segment()
2752 		    tp->snd_una, th->th_ack, tp, m));  in tcp_do_segment()
2767 			cc_cong_signal(tp, th, CC_RTO_ERR);  in tcp_do_segment()
2790 		} else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) {  in tcp_do_segment()
2802 		if (th->th_ack == tp->snd_max) {  in tcp_do_segment()
2820 		cc_ack_received(tp, th, nsegs, CC_ACK);  in tcp_do_segment()
2845 		    SEQ_LEQ(th->th_ack, tp->snd_recover))  in tcp_do_segment()
2846 			tp->snd_recover = th->th_ack - 1;  in tcp_do_segment()
2849 		    SEQ_GEQ(th->th_ack, tp->snd_recover)) {  in tcp_do_segment()
2852 		tp->snd_una = th->th_ack;  in tcp_do_segment()
2930 	    (SEQ_LT(tp->snd_wl1, th->th_seq) ||  in tcp_do_segment()
2931 	    (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||  in tcp_do_segment()
2932 	     (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {  in tcp_do_segment()
2935 		    tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)  in tcp_do_segment()
2938 		tp->snd_wl1 = th->th_seq;  in tcp_do_segment()
2939 		tp->snd_wl2 = th->th_ack;  in tcp_do_segment()
2948 	if ((thflags & TH_URG) && th->th_urp &&  in tcp_do_segment()
2957 		if (th->th_urp + sbavail(&so->so_rcv) > sb_max) {  in tcp_do_segment()
2958 			th->th_urp = 0;			/* XXX */  in tcp_do_segment()
2977 		if (SEQ_GT(th->th_seq+th->th_urp, tp->rcv_up)) {  in tcp_do_segment()
2978 			tp->rcv_up = th->th_seq + th->th_urp;  in tcp_do_segment()
2993 		if (th->th_urp <= (uint32_t)tlen &&  in tcp_do_segment()
2996 			tcp_pulloutofband(so, th, m, drop_hdrlen);  in tcp_do_segment()
3022 		tcp_seq save_start = th->th_seq;  in tcp_do_segment()
3038 		if (th->th_seq == tp->rcv_nxt &&  in tcp_do_segment()
3047 			thflags = th->th_flags & TH_FIN;  in tcp_do_segment()
3065 			thflags = tcp_reass(tp, th, &temp, &tlen, m);  in tcp_do_segment()
3174 	TCP_PROBE3(debug__input, tp, th, m);  in tcp_do_segment()
3209 	    (SEQ_GT(tp->snd_una, th->th_ack) ||  in tcp_do_segment()
3210 	     SEQ_GT(th->th_ack, tp->snd_max)) ) {  in tcp_do_segment()
3219 	TCP_PROBE3(debug__input, tp, th, m);  in tcp_do_segment()
3228 		tcp_dropwithreset(m, th, tp, tlen, rstreason);  in tcp_do_segment()
3231 		tcp_dropwithreset(m, th, NULL, tlen, rstreason);  in tcp_do_segment()
3243 	TCP_PROBE3(debug__input, tp, th, m);  in tcp_do_segment()
3255 tcp_dropwithreset(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp,  in tcp_dropwithreset()  argument
3270 	if ((th->th_flags & TH_RST) || m->m_flags & (M_BCAST|M_MCAST))  in tcp_dropwithreset()
3300 	if (th->th_flags & TH_ACK) {  in tcp_dropwithreset()
3301 		tcp_respond(tp, mtod(m, void *), th, m, (tcp_seq)0,  in tcp_dropwithreset()
3302 		    th->th_ack, TH_RST);  in tcp_dropwithreset()
3304 		if (th->th_flags & TH_SYN)  in tcp_dropwithreset()
3306 		if (th->th_flags & TH_FIN)  in tcp_dropwithreset()
3308 		tcp_respond(tp, mtod(m, void *), th, m, th->th_seq+tlen,  in tcp_dropwithreset()
3428 tcp_pulloutofband(struct socket *so, struct tcphdr *th, struct mbuf *m,  in tcp_pulloutofband()  argument
3431 	int cnt = off + th->th_urp - 1;  in tcp_pulloutofband()
3829 tcp_newreno_partial_ack(struct tcpcb *tp, struct tcphdr *th)  in tcp_newreno_partial_ack()  argument
3839 	tp->snd_nxt = th->th_ack;  in tcp_newreno_partial_ack()
3844 	tp->snd_cwnd = maxseg + BYTES_THIS_ACK(tp, th);  in tcp_newreno_partial_ack()
3854 	if (tp->snd_cwnd > BYTES_THIS_ACK(tp, th))  in tcp_newreno_partial_ack()
3855 		tp->snd_cwnd -= BYTES_THIS_ACK(tp, th);  in tcp_newreno_partial_ack()