Lines Matching refs:tp
297 tcp_is_sack_recovery(struct tcpcb *tp, struct tcpopt *to) in tcp_is_sack_recovery() argument
299 return ((tp->t_flags & TF_SACK_PERMIT) && in tcp_is_sack_recovery()
301 (!TAILQ_EMPTY(&tp->snd_holes)))); in tcp_is_sack_recovery()
309 hhook_run_tcp_est_in(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to) in hhook_run_tcp_est_in() argument
314 hhook_data.tp = tp; in hhook_run_tcp_est_in()
319 tp->osd); in hhook_run_tcp_est_in()
328 cc_ack_received(struct tcpcb *tp, struct tcphdr *th, uint16_t nsegs, in cc_ack_received() argument
335 INP_WLOCK_ASSERT(tp->t_inpcb); in cc_ack_received()
337 tp->ccv->nsegs = nsegs; in cc_ack_received()
338 tp->ccv->bytes_this_ack = BYTES_THIS_ACK(tp, th); in cc_ack_received()
339 if ((!V_tcp_do_newcwv && (tp->snd_cwnd <= tp->snd_wnd)) || in cc_ack_received()
340 (V_tcp_do_newcwv && (tp->snd_cwnd <= tp->snd_wnd) && in cc_ack_received()
341 (tp->snd_cwnd < (tcp_compute_pipe(tp) * 2)))) in cc_ack_received()
342 tp->ccv->flags |= CCF_CWND_LIMITED; in cc_ack_received()
344 tp->ccv->flags &= ~CCF_CWND_LIMITED; in cc_ack_received()
348 stats_voi_update_abs_s32(tp->t_stats, VOI_TCP_CALCFRWINDIFF, in cc_ack_received()
349 ((int32_t)tp->snd_cwnd) - tp->snd_wnd); in cc_ack_received()
350 if (!IN_RECOVERY(tp->t_flags)) in cc_ack_received()
351 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_ACKLEN, in cc_ack_received()
352 tp->ccv->bytes_this_ack / (tcp_maxseg(tp) * nsegs)); in cc_ack_received()
353 if ((tp->t_flags & TF_GPUTINPROG) && in cc_ack_received()
354 SEQ_GEQ(th->th_ack, tp->gput_ack)) { in cc_ack_received()
358 gput = (((int64_t)(th->th_ack - tp->gput_seq)) << 3) / in cc_ack_received()
359 max(1, tcp_ts_getticks() - tp->gput_ts); in cc_ack_received()
360 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT, in cc_ack_received()
367 if (tp->t_stats_gput_prev > 0) in cc_ack_received()
368 stats_voi_update_abs_s32(tp->t_stats, in cc_ack_received()
370 ((gput - tp->t_stats_gput_prev) * 100) / in cc_ack_received()
371 tp->t_stats_gput_prev); in cc_ack_received()
372 tp->t_flags &= ~TF_GPUTINPROG; in cc_ack_received()
373 tp->t_stats_gput_prev = gput; in cc_ack_received()
376 if (tp->snd_cwnd > tp->snd_ssthresh) { in cc_ack_received()
377 tp->t_bytes_acked += tp->ccv->bytes_this_ack; in cc_ack_received()
378 if (tp->t_bytes_acked >= tp->snd_cwnd) { in cc_ack_received()
379 tp->t_bytes_acked -= tp->snd_cwnd; in cc_ack_received()
380 tp->ccv->flags |= CCF_ABC_SENTAWND; in cc_ack_received()
383 tp->ccv->flags &= ~CCF_ABC_SENTAWND; in cc_ack_received()
384 tp->t_bytes_acked = 0; in cc_ack_received()
388 if (CC_ALGO(tp)->ack_received != NULL) { in cc_ack_received()
390 tp->ccv->curack = th->th_ack; in cc_ack_received()
391 CC_ALGO(tp)->ack_received(tp->ccv, type); in cc_ack_received()
394 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_LCWIN, tp->snd_cwnd); in cc_ack_received()
399 cc_conn_init(struct tcpcb *tp) in cc_conn_init() argument
402 struct inpcb *inp = tp->t_inpcb; in cc_conn_init()
406 INP_WLOCK_ASSERT(tp->t_inpcb); in cc_conn_init()
409 maxseg = tcp_maxseg(tp); in cc_conn_init()
411 if (tp->t_srtt == 0 && (rtt = metrics.rmx_rtt)) { in cc_conn_init()
412 tp->t_srtt = rtt; in cc_conn_init()
413 tp->t_rttbest = tp->t_srtt + TCP_RTT_SCALE; in cc_conn_init()
416 tp->t_rttvar = metrics.rmx_rttvar; in cc_conn_init()
420 tp->t_rttvar = in cc_conn_init()
421 tp->t_srtt * TCP_RTTVAR_SCALE / TCP_RTT_SCALE; in cc_conn_init()
423 TCPT_RANGESET(tp->t_rxtcur, in cc_conn_init()
424 ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1, in cc_conn_init()
425 tp->t_rttmin, TCPTV_REXMTMAX); in cc_conn_init()
434 tp->snd_ssthresh = max(2 * maxseg, metrics.rmx_ssthresh); in cc_conn_init()
445 if (tp->snd_cwnd == 1) in cc_conn_init()
446 tp->snd_cwnd = maxseg; /* SYN(-ACK) lost */ in cc_conn_init()
448 tp->snd_cwnd = tcp_compute_initwnd(maxseg); in cc_conn_init()
450 if (CC_ALGO(tp)->conn_init != NULL) in cc_conn_init()
451 CC_ALGO(tp)->conn_init(tp->ccv); in cc_conn_init()
455 cc_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type) in cc_cong_signal() argument
457 INP_WLOCK_ASSERT(tp->t_inpcb); in cc_cong_signal()
460 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type); in cc_cong_signal()
465 if (!IN_FASTRECOVERY(tp->t_flags)) { in cc_cong_signal()
466 tp->snd_recover = tp->snd_max; in cc_cong_signal()
467 if (tp->t_flags2 & TF2_ECN_PERMIT) in cc_cong_signal()
468 tp->t_flags2 |= TF2_ECN_SND_CWR; in cc_cong_signal()
472 if (!IN_CONGRECOVERY(tp->t_flags) || in cc_cong_signal()
477 SEQ_GEQ(th->th_ack, tp->snd_recover)) { in cc_cong_signal()
478 EXIT_CONGRECOVERY(tp->t_flags); in cc_cong_signal()
480 tp->snd_recover = tp->snd_max + 1; in cc_cong_signal()
481 if (tp->t_flags2 & TF2_ECN_PERMIT) in cc_cong_signal()
482 tp->t_flags2 |= TF2_ECN_SND_CWR; in cc_cong_signal()
486 tp->t_dupacks = 0; in cc_cong_signal()
487 tp->t_bytes_acked = 0; in cc_cong_signal()
488 EXIT_RECOVERY(tp->t_flags); in cc_cong_signal()
489 if (tp->t_flags2 & TF2_ECN_PERMIT) in cc_cong_signal()
490 tp->t_flags2 |= TF2_ECN_SND_CWR; in cc_cong_signal()
495 tp->snd_cwnd = tp->snd_cwnd_prev; in cc_cong_signal()
496 tp->snd_ssthresh = tp->snd_ssthresh_prev; in cc_cong_signal()
497 tp->snd_recover = tp->snd_recover_prev; in cc_cong_signal()
498 if (tp->t_flags & TF_WASFRECOVERY) in cc_cong_signal()
499 ENTER_FASTRECOVERY(tp->t_flags); in cc_cong_signal()
500 if (tp->t_flags & TF_WASCRECOVERY) in cc_cong_signal()
501 ENTER_CONGRECOVERY(tp->t_flags); in cc_cong_signal()
502 tp->snd_nxt = tp->snd_max; in cc_cong_signal()
503 tp->t_flags &= ~TF_PREVVALID; in cc_cong_signal()
504 tp->t_badrxtwin = 0; in cc_cong_signal()
508 if (CC_ALGO(tp)->cong_signal != NULL) { in cc_cong_signal()
510 tp->ccv->curack = th->th_ack; in cc_cong_signal()
511 CC_ALGO(tp)->cong_signal(tp->ccv, type); in cc_cong_signal()
516 cc_post_recovery(struct tcpcb *tp, struct tcphdr *th) in cc_post_recovery() argument
518 INP_WLOCK_ASSERT(tp->t_inpcb); in cc_post_recovery()
522 if (CC_ALGO(tp)->post_recovery != NULL) { in cc_post_recovery()
523 tp->ccv->curack = th->th_ack; in cc_post_recovery()
524 CC_ALGO(tp)->post_recovery(tp->ccv); in cc_post_recovery()
527 tp->t_bytes_acked = 0; in cc_post_recovery()
528 tp->sackhint.delivered_data = 0; in cc_post_recovery()
529 tp->sackhint.prr_out = 0; in cc_post_recovery()
541 #define DELAY_ACK(tp, tlen) \ argument
542 ((!tcp_timer_active(tp, TT_DELACK) && \
543 (tp->t_flags & TF_RXWIN0SENT) == 0) && \
544 (tlen <= tp->t_maxseg) && \
545 (V_tcp_delack_enabled || (tp->t_flags & TF_NEEDSYN)))
548 cc_ecnpkt_handler_flags(struct tcpcb *tp, uint16_t flags, uint8_t iptos) in cc_ecnpkt_handler_flags() argument
550 INP_WLOCK_ASSERT(tp->t_inpcb); in cc_ecnpkt_handler_flags()
552 if (CC_ALGO(tp)->ecnpkt_handler != NULL) { in cc_ecnpkt_handler_flags()
555 tp->ccv->flags |= CCF_IPHDR_CE; in cc_ecnpkt_handler_flags()
562 tp->ccv->flags &= ~CCF_IPHDR_CE; in cc_ecnpkt_handler_flags()
567 tp->ccv->flags |= CCF_TCPHDR_CWR; in cc_ecnpkt_handler_flags()
569 tp->ccv->flags &= ~CCF_TCPHDR_CWR; in cc_ecnpkt_handler_flags()
571 CC_ALGO(tp)->ecnpkt_handler(tp->ccv); in cc_ecnpkt_handler_flags()
573 if (tp->ccv->flags & CCF_ACKNOW) { in cc_ecnpkt_handler_flags()
574 tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); in cc_ecnpkt_handler_flags()
575 tp->t_flags |= TF_ACKNOW; in cc_ecnpkt_handler_flags()
581 cc_ecnpkt_handler(struct tcpcb *tp, struct tcphdr *th, uint8_t iptos) in cc_ecnpkt_handler() argument
583 cc_ecnpkt_handler_flags(tp, th->th_flags, iptos); in cc_ecnpkt_handler()
645 struct tcpcb *tp = NULL; in tcp_input_with_port() local
1034 tp = intotcpcb(inp); in tcp_input_with_port()
1035 if (tp == NULL || tp->t_state == TCPS_CLOSED) { in tcp_input_with_port()
1040 if ((tp->t_port != port) && (tp->t_state > TCPS_LISTEN)) { in tcp_input_with_port()
1046 if (tp->t_flags & TF_TOE) { in tcp_input_with_port()
1047 tcp_offload_input(tp, m); in tcp_input_with_port()
1062 ostate = tp->t_state; in tcp_input_with_port()
1077 KASSERT(tp->t_state == TCPS_LISTEN || !SOLISTENING(so), in tcp_input_with_port()
1078 ("%s: so accepting but tp %p not listening", __func__, tp)); in tcp_input_with_port()
1079 if (tp->t_state == TCPS_LISTEN && SOLISTENING(so)) { in tcp_input_with_port()
1172 tp = intotcpcb(inp); in tcp_input_with_port()
1173 KASSERT(tp->t_state == TCPS_SYN_RECEIVED, in tcp_input_with_port()
1180 TCP_PROBE5(receive, NULL, tp, m, tp, th); in tcp_input_with_port()
1181 tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, in tcp_input_with_port()
1370 tcp_trace(TA_INPUT, ostate, tp, in tcp_input_with_port()
1373 TCP_PROBE3(debug__input, tp, th, m); in tcp_input_with_port()
1385 } else if (tp->t_state == TCPS_LISTEN) { in tcp_input_with_port()
1396 if (tp->t_flags & TF_SIGNATURE) { in tcp_input_with_port()
1407 TCP_PROBE5(receive, NULL, tp, m, tp, th); in tcp_input_with_port()
1414 tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, iptos); in tcp_input_with_port()
1418 TCP_PROBE5(receive, NULL, tp, m, tp, th); in tcp_input_with_port()
1421 tcp_dropwithreset(m, th, tp, tlen, rstreason); in tcp_input_with_port()
1430 TCP_PROBE5(receive, NULL, tp, m, tp, th); in tcp_input_with_port()
1484 struct tcpcb *tp, int tlen) in tcp_autorcvbuf() argument
1489 tp->t_srtt != 0 && tp->rfbuf_ts != 0 && in tcp_autorcvbuf()
1490 TCP_TS_TO_TICKS(tcp_ts_getticks() - tp->rfbuf_ts) > in tcp_autorcvbuf()
1491 ((tp->t_srtt >> TCP_RTT_SHIFT)/2)) { in tcp_autorcvbuf()
1492 if (tp->rfbuf_cnt > ((so->so_rcv.sb_hiwat / 2)/ 4 * 3) && in tcp_autorcvbuf()
1496 TCP_PROBE6(receive__autoresize, NULL, tp, m, tp, th, newsize); in tcp_autorcvbuf()
1499 tp->rfbuf_ts = 0; in tcp_autorcvbuf()
1500 tp->rfbuf_cnt = 0; in tcp_autorcvbuf()
1502 tp->rfbuf_cnt += tlen; /* add up */ in tcp_autorcvbuf()
1514 tcp_handle_wakeup(struct tcpcb *tp, struct socket *so) in tcp_handle_wakeup() argument
1521 if (tp == NULL) { in tcp_handle_wakeup()
1527 INP_LOCK_ASSERT(tp->t_inpcb); in tcp_handle_wakeup()
1528 if (tp->t_flags & TF_WAKESOR) { in tcp_handle_wakeup()
1529 tp->t_flags &= ~TF_WAKESOR; in tcp_handle_wakeup()
1537 struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos) in tcp_do_segment() argument
1560 inc = &tp->t_inpcb->inp_inc; in tcp_do_segment()
1561 tp->sackhint.last_sack_ack = 0; in tcp_do_segment()
1566 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_segment()
1567 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", in tcp_do_segment()
1569 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", in tcp_do_segment()
1574 tcp_pcap_add(th, m, &(tp->t_inpkts)); in tcp_do_segment()
1576 TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0, in tcp_do_segment()
1593 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && in tcp_do_segment()
1594 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { in tcp_do_segment()
1605 tp->t_rcvtime = ticks; in tcp_do_segment()
1611 tiwin = th->th_win << tp->snd_scale; in tcp_do_segment()
1613 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin); in tcp_do_segment()
1619 if (tp->t_flags2 & TF2_ECN_PERMIT) { in tcp_do_segment()
1621 tp->t_flags2 &= ~TF2_ECN_SND_ECE; in tcp_do_segment()
1622 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
1626 tp->t_flags2 |= TF2_ECN_SND_ECE; in tcp_do_segment()
1638 cc_ecnpkt_handler(tp, th, iptos); in tcp_do_segment()
1642 cc_cong_signal(tp, th, CC_ECN); in tcp_do_segment()
1654 if ((tp->t_flags & TF_SIGNATURE) != 0 && in tcp_do_segment()
1667 to.to_tsecr -= tp->ts_offset; in tcp_do_segment()
1670 else if (tp->t_rxtshift == 1 && in tcp_do_segment()
1671 tp->t_flags & TF_PREVVALID && in tcp_do_segment()
1672 tp->t_badrxtwin != 0 && in tcp_do_segment()
1673 TSTMP_LT(to.to_tsecr, tp->t_badrxtwin)) in tcp_do_segment()
1674 cc_cong_signal(tp, th, CC_RTO_ERR); in tcp_do_segment()
1683 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { in tcp_do_segment()
1688 tp->t_flags2 |= TF2_ECN_PERMIT; in tcp_do_segment()
1689 tp->t_flags2 |= TF2_ECN_SND_ECE; in tcp_do_segment()
1693 (tp->t_flags & TF_REQ_SCALE) && in tcp_do_segment()
1694 !(tp->t_flags & TF_NOOPT)) { in tcp_do_segment()
1695 tp->t_flags |= TF_RCVD_SCALE; in tcp_do_segment()
1696 tp->snd_scale = to.to_wscale; in tcp_do_segment()
1698 tp->t_flags &= ~TF_REQ_SCALE; in tcp_do_segment()
1703 tp->snd_wnd = th->th_win; in tcp_do_segment()
1705 (tp->t_flags & TF_REQ_TSTMP) && in tcp_do_segment()
1706 !(tp->t_flags & TF_NOOPT)) { in tcp_do_segment()
1707 tp->t_flags |= TF_RCVD_TSTMP; in tcp_do_segment()
1708 tp->ts_recent = to.to_tsval; in tcp_do_segment()
1709 tp->ts_recent_age = tcp_ts_getticks(); in tcp_do_segment()
1711 tp->t_flags &= ~TF_REQ_TSTMP; in tcp_do_segment()
1713 tcp_mss(tp, to.to_mss); in tcp_do_segment()
1714 if ((tp->t_flags & TF_SACK_PERMIT) && in tcp_do_segment()
1716 (tp->t_flags & TF_NOOPT))) in tcp_do_segment()
1717 tp->t_flags &= ~TF_SACK_PERMIT; in tcp_do_segment()
1718 if (IS_FASTOPEN(tp->t_flags)) { in tcp_do_segment()
1720 !(tp->t_flags & TF_NOOPT)) { in tcp_do_segment()
1726 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0) in tcp_do_segment()
1730 tcp_fastopen_update_cache(tp, mss, in tcp_do_segment()
1733 tcp_fastopen_disable_path(tp); in tcp_do_segment()
1744 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) { in tcp_do_segment()
1767 if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) { in tcp_do_segment()
1792 if (tp->t_state == TCPS_ESTABLISHED && in tcp_do_segment()
1793 th->th_seq == tp->rcv_nxt && in tcp_do_segment()
1795 tp->snd_nxt == tp->snd_max && in tcp_do_segment()
1796 tiwin && tiwin == tp->snd_wnd && in tcp_do_segment()
1797 ((tp->t_flags & (TF_NEEDSYN|TF_NEEDFIN)) == 0) && in tcp_do_segment()
1798 SEGQ_EMPTY(tp) && in tcp_do_segment()
1800 TSTMP_GEQ(to.to_tsval, tp->ts_recent)) ) { in tcp_do_segment()
1808 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { in tcp_do_segment()
1809 tp->ts_recent_age = tcp_ts_getticks(); in tcp_do_segment()
1810 tp->ts_recent = to.to_tsval; in tcp_do_segment()
1814 if (SEQ_GT(th->th_ack, tp->snd_una) && in tcp_do_segment()
1815 SEQ_LEQ(th->th_ack, tp->snd_max) && in tcp_do_segment()
1816 !IN_RECOVERY(tp->t_flags) && in tcp_do_segment()
1818 TAILQ_EMPTY(&tp->snd_holes)) { in tcp_do_segment()
1828 tp->t_rxtshift == 1 && in tcp_do_segment()
1829 tp->t_flags & TF_PREVVALID && in tcp_do_segment()
1830 tp->t_badrxtwin != 0 && in tcp_do_segment()
1831 TSTMP_LT(ticks, tp->t_badrxtwin)) { in tcp_do_segment()
1832 cc_cong_signal(tp, th, CC_RTO_ERR); in tcp_do_segment()
1848 if (!tp->t_rttlow || tp->t_rttlow > t) in tcp_do_segment()
1849 tp->t_rttlow = t; in tcp_do_segment()
1850 tcp_xmit_timer(tp, in tcp_do_segment()
1852 } else if (tp->t_rtttime && in tcp_do_segment()
1853 SEQ_GT(th->th_ack, tp->t_rtseq)) { in tcp_do_segment()
1854 if (!tp->t_rttlow || in tcp_do_segment()
1855 tp->t_rttlow > ticks - tp->t_rtttime) in tcp_do_segment()
1856 tp->t_rttlow = ticks - tp->t_rtttime; in tcp_do_segment()
1857 tcp_xmit_timer(tp, in tcp_do_segment()
1858 ticks - tp->t_rtttime); in tcp_do_segment()
1860 acked = BYTES_THIS_ACK(tp, th); in tcp_do_segment()
1864 hhook_run_tcp_est_in(tp, th, &to); in tcp_do_segment()
1870 if (SEQ_GT(tp->snd_una, tp->snd_recover) && in tcp_do_segment()
1871 SEQ_LEQ(th->th_ack, tp->snd_recover)) in tcp_do_segment()
1872 tp->snd_recover = th->th_ack - 1; in tcp_do_segment()
1880 cc_ack_received(tp, th, nsegs, CC_ACK); in tcp_do_segment()
1882 tp->snd_una = th->th_ack; in tcp_do_segment()
1887 tp->snd_wl2 = th->th_ack; in tcp_do_segment()
1888 tp->t_dupacks = 0; in tcp_do_segment()
1902 tcp_trace(TA_INPUT, ostate, tp, in tcp_do_segment()
1906 TCP_PROBE3(debug__input, tp, th, m); in tcp_do_segment()
1907 if (tp->snd_una == tp->snd_max) in tcp_do_segment()
1908 tcp_timer_activate(tp, TT_REXMT, 0); in tcp_do_segment()
1909 else if (!tcp_timer_active(tp, TT_PERSIST)) in tcp_do_segment()
1910 tcp_timer_activate(tp, TT_REXMT, in tcp_do_segment()
1911 tp->t_rxtcur); in tcp_do_segment()
1914 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
1917 } else if (th->th_ack == tp->snd_una && in tcp_do_segment()
1927 if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks) in tcp_do_segment()
1928 tcp_clean_sackreport(tp); in tcp_do_segment()
1930 tp->rcv_nxt += tlen; in tcp_do_segment()
1932 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && in tcp_do_segment()
1933 (tp->t_fbyte_in == 0)) { in tcp_do_segment()
1934 tp->t_fbyte_in = ticks; in tcp_do_segment()
1935 if (tp->t_fbyte_in == 0) in tcp_do_segment()
1936 tp->t_fbyte_in = 1; in tcp_do_segment()
1937 if (tp->t_fbyte_out && tp->t_fbyte_in) in tcp_do_segment()
1938 tp->t_flags2 |= TF2_FBYTES_COMPLETE; in tcp_do_segment()
1944 tp->snd_wl1 = th->th_seq; in tcp_do_segment()
1949 tp->rcv_up = tp->rcv_nxt; in tcp_do_segment()
1954 tcp_trace(TA_INPUT, ostate, tp, in tcp_do_segment()
1957 TCP_PROBE3(debug__input, tp, th, m); in tcp_do_segment()
1959 newsize = tcp_autorcvbuf(m, th, so, tp, tlen); in tcp_do_segment()
1979 if (DELAY_ACK(tp, tlen)) { in tcp_do_segment()
1980 tp->t_flags |= TF_DELACK; in tcp_do_segment()
1982 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
1983 tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
1998 tp->rcv_wnd = imax(win, (int)(tp->rcv_adv - tp->rcv_nxt)); in tcp_do_segment()
2000 switch (tp->t_state) { in tcp_do_segment()
2007 (SEQ_LEQ(th->th_ack, tp->snd_una) || in tcp_do_segment()
2008 SEQ_GT(th->th_ack, tp->snd_max))) { in tcp_do_segment()
2012 if (IS_FASTOPEN(tp->t_flags)) { in tcp_do_segment()
2025 if ((tcp_timer_active(tp, TT_DELACK) || in tcp_do_segment()
2026 tcp_timer_active(tp, TT_REXMT))) in tcp_do_segment()
2051 TCP_PROBE5(connect__refused, NULL, tp, in tcp_do_segment()
2052 m, tp, th); in tcp_do_segment()
2053 tp = tcp_drop(tp, ECONNREFUSED); in tcp_do_segment()
2060 tp->irs = th->th_seq; in tcp_do_segment()
2061 tcp_rcvseqinit(tp); in tcp_do_segment()
2071 if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == in tcp_do_segment()
2073 tp->rcv_scale = tp->request_r_scale; in tcp_do_segment()
2075 tp->rcv_adv += min(tp->rcv_wnd, in tcp_do_segment()
2076 TCP_MAXWIN << tp->rcv_scale); in tcp_do_segment()
2077 tp->snd_una++; /* SYN is acked */ in tcp_do_segment()
2082 if (IS_FASTOPEN(tp->t_flags) && in tcp_do_segment()
2083 (tp->snd_una != tp->snd_max)) { in tcp_do_segment()
2084 tp->snd_nxt = th->th_ack; in tcp_do_segment()
2091 if (DELAY_ACK(tp, tlen) && tlen != 0 && !tfo_partial_ack) in tcp_do_segment()
2092 tcp_timer_activate(tp, TT_DELACK, in tcp_do_segment()
2095 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
2099 tp->t_flags2 |= TF2_ECN_PERMIT; in tcp_do_segment()
2109 tp->t_starttime = ticks; in tcp_do_segment()
2110 if (tp->t_flags & TF_NEEDFIN) { in tcp_do_segment()
2111 tcp_state_change(tp, TCPS_FIN_WAIT_1); in tcp_do_segment()
2112 tp->t_flags &= ~TF_NEEDFIN; in tcp_do_segment()
2115 tcp_state_change(tp, TCPS_ESTABLISHED); in tcp_do_segment()
2116 TCP_PROBE5(connect__established, NULL, tp, in tcp_do_segment()
2117 m, tp, th); in tcp_do_segment()
2118 cc_conn_init(tp); in tcp_do_segment()
2119 tcp_timer_activate(tp, TT_KEEP, in tcp_do_segment()
2120 TP_KEEPIDLE(tp)); in tcp_do_segment()
2131 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN); in tcp_do_segment()
2132 tcp_timer_activate(tp, TT_REXMT, 0); in tcp_do_segment()
2133 tcp_state_change(tp, TCPS_SYN_RECEIVED); in tcp_do_segment()
2136 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_segment()
2144 if (tlen > tp->rcv_wnd) { in tcp_do_segment()
2145 todrop = tlen - tp->rcv_wnd; in tcp_do_segment()
2147 tlen = tp->rcv_wnd; in tcp_do_segment()
2152 tp->snd_wl1 = th->th_seq - 1; in tcp_do_segment()
2153 tp->rcv_up = th->th_seq; in tcp_do_segment()
2204 if ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) && in tcp_do_segment()
2205 SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) || in tcp_do_segment()
2206 (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) { in tcp_do_segment()
2207 KASSERT(tp->t_state != TCPS_SYN_SENT, in tcp_do_segment()
2209 __func__, th, tp)); in tcp_do_segment()
2212 tp->last_ack_sent == th->th_seq) { in tcp_do_segment()
2215 switch (tp->t_state) { in tcp_do_segment()
2229 tp = tcp_close(tp); in tcp_do_segment()
2234 tcp_respond(tp, mtod(m, void *), th, m, in tcp_do_segment()
2235 tp->rcv_nxt, tp->snd_nxt, TH_ACK); in tcp_do_segment()
2236 tp->last_ack_sent = tp->rcv_nxt; in tcp_do_segment()
2247 if ((thflags & TH_SYN) && tp->t_state != TCPS_SYN_SENT && in tcp_do_segment()
2248 tp->t_state != TCPS_SYN_RECEIVED) { in tcp_do_segment()
2251 SEQ_GEQ(th->th_seq, tp->last_ack_sent) && in tcp_do_segment()
2252 SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) { in tcp_do_segment()
2253 tp = tcp_drop(tp, ECONNRESET); in tcp_do_segment()
2257 tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt, in tcp_do_segment()
2258 tp->snd_nxt, TH_ACK); in tcp_do_segment()
2259 tp->last_ack_sent = tp->rcv_nxt; in tcp_do_segment()
2269 if ((to.to_flags & TOF_TS) != 0 && tp->ts_recent && in tcp_do_segment()
2270 TSTMP_LT(to.to_tsval, tp->ts_recent)) { in tcp_do_segment()
2272 if (tcp_ts_getticks() - tp->ts_recent_age > TCP_PAWS_IDLE) { in tcp_do_segment()
2284 tp->ts_recent = 0; in tcp_do_segment()
2302 if (tp->t_state == TCPS_SYN_RECEIVED && SEQ_LT(th->th_seq, tp->irs)) { in tcp_do_segment()
2307 todrop = tp->rcv_nxt - th->th_seq; in tcp_do_segment()
2334 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
2345 if ((todrop > 0) && (tp->t_flags & TF_SACK_PERMIT)) { in tcp_do_segment()
2346 tcp_update_sack_list(tp, th->th_seq, in tcp_do_segment()
2352 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
2370 tp->t_state > TCPS_CLOSE_WAIT && tlen) { in tcp_do_segment()
2375 s, __func__, tcpstates[tp->t_state], tlen); in tcp_do_segment()
2378 tp = tcp_close(tp); in tcp_do_segment()
2388 todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd); in tcp_do_segment()
2400 if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) { in tcp_do_segment()
2401 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
2431 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in tcp_do_segment()
2432 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in tcp_do_segment()
2434 tp->ts_recent_age = tcp_ts_getticks(); in tcp_do_segment()
2435 tp->ts_recent = to.to_tsval; in tcp_do_segment()
2444 if (tp->t_state == TCPS_SYN_RECEIVED || in tcp_do_segment()
2445 (tp->t_flags & TF_NEEDSYN)) { in tcp_do_segment()
2446 if (tp->t_state == TCPS_SYN_RECEIVED && in tcp_do_segment()
2447 IS_FASTOPEN(tp->t_flags)) { in tcp_do_segment()
2448 tp->snd_wnd = tiwin; in tcp_do_segment()
2449 cc_conn_init(tp); in tcp_do_segment()
2452 } else if (tp->t_flags & TF_ACKNOW) in tcp_do_segment()
2461 switch (tp->t_state) { in tcp_do_segment()
2472 if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == in tcp_do_segment()
2474 tp->rcv_scale = tp->request_r_scale; in tcp_do_segment()
2476 tp->snd_wnd = tiwin; in tcp_do_segment()
2482 tp->t_starttime = ticks; in tcp_do_segment()
2483 if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) { in tcp_do_segment()
2484 tcp_fastopen_decrement_counter(tp->t_tfo_pending); in tcp_do_segment()
2485 tp->t_tfo_pending = NULL; in tcp_do_segment()
2487 if (tp->t_flags & TF_NEEDFIN) { in tcp_do_segment()
2488 tcp_state_change(tp, TCPS_FIN_WAIT_1); in tcp_do_segment()
2489 tp->t_flags &= ~TF_NEEDFIN; in tcp_do_segment()
2491 tcp_state_change(tp, TCPS_ESTABLISHED); in tcp_do_segment()
2492 TCP_PROBE5(accept__established, NULL, tp, in tcp_do_segment()
2493 m, tp, th); in tcp_do_segment()
2501 if (!IS_FASTOPEN(tp->t_flags)) in tcp_do_segment()
2502 cc_conn_init(tp); in tcp_do_segment()
2503 tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp)); in tcp_do_segment()
2510 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN)) in tcp_do_segment()
2517 (void) tcp_reass(tp, (struct tcphdr *)0, NULL, 0, in tcp_do_segment()
2519 tcp_handle_wakeup(tp, so); in tcp_do_segment()
2521 tp->snd_wl1 = th->th_seq - 1; in tcp_do_segment()
2538 if (SEQ_GT(th->th_ack, tp->snd_max)) { in tcp_do_segment()
2542 if (tcp_is_sack_recovery(tp, &to)) in tcp_do_segment()
2543 sack_changed = tcp_sack_doack(tp, &to, th->th_ack); in tcp_do_segment()
2549 tp->sackhint.sacked_bytes = 0; in tcp_do_segment()
2553 hhook_run_tcp_est_in(tp, th, &to); in tcp_do_segment()
2556 if (SEQ_LEQ(th->th_ack, tp->snd_una)) { in tcp_do_segment()
2557 maxseg = tcp_maxseg(tp); in tcp_do_segment()
2559 (tiwin == tp->snd_wnd || in tcp_do_segment()
2560 (tp->t_flags & TF_SACK_PERMIT))) { in tcp_do_segment()
2569 (TCPS_HAVERCVDFIN(tp->t_state) == 0)) { in tcp_do_segment()
2570 tp->t_dupacks = 0; in tcp_do_segment()
2611 if (th->th_ack != tp->snd_una || in tcp_do_segment()
2612 (tcp_is_sack_recovery(tp, &to) && in tcp_do_segment()
2615 else if (!tcp_timer_active(tp, TT_REXMT)) in tcp_do_segment()
2616 tp->t_dupacks = 0; in tcp_do_segment()
2617 else if (++tp->t_dupacks > tcprexmtthresh || in tcp_do_segment()
2618 IN_FASTRECOVERY(tp->t_flags)) { in tcp_do_segment()
2619 cc_ack_received(tp, th, nsegs, in tcp_do_segment()
2622 IN_FASTRECOVERY(tp->t_flags)) { in tcp_do_segment()
2623 tcp_do_prr_ack(tp, th, &to); in tcp_do_segment()
2624 } else if (tcp_is_sack_recovery(tp, &to) && in tcp_do_segment()
2625 IN_FASTRECOVERY(tp->t_flags)) { in tcp_do_segment()
2635 awnd = tcp_compute_pipe(tp); in tcp_do_segment()
2637 awnd = (tp->snd_nxt - tp->snd_fack) + in tcp_do_segment()
2638 tp->sackhint.sack_bytes_rexmit; in tcp_do_segment()
2640 if (awnd < tp->snd_ssthresh) { in tcp_do_segment()
2641 tp->snd_cwnd += maxseg; in tcp_do_segment()
2642 if (tp->snd_cwnd > tp->snd_ssthresh) in tcp_do_segment()
2643 tp->snd_cwnd = tp->snd_ssthresh; in tcp_do_segment()
2646 tp->snd_cwnd += maxseg; in tcp_do_segment()
2647 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
2649 } else if (tp->t_dupacks == tcprexmtthresh || in tcp_do_segment()
2650 (tp->t_flags & TF_SACK_PERMIT && in tcp_do_segment()
2652 tp->sackhint.sacked_bytes > in tcp_do_segment()
2663 tp->t_dupacks = tcprexmtthresh; in tcp_do_segment()
2664 tcp_seq onxt = tp->snd_nxt; in tcp_do_segment()
2674 (tp->t_flags & TF_SACK_PERMIT)) { in tcp_do_segment()
2675 if (IN_FASTRECOVERY(tp->t_flags)) { in tcp_do_segment()
2676 tp->t_dupacks = 0; in tcp_do_segment()
2681 tp->snd_recover)) { in tcp_do_segment()
2682 tp->t_dupacks = 0; in tcp_do_segment()
2687 cc_cong_signal(tp, th, CC_NDUPACK); in tcp_do_segment()
2688 cc_ack_received(tp, th, nsegs, in tcp_do_segment()
2690 tcp_timer_activate(tp, TT_REXMT, 0); in tcp_do_segment()
2691 tp->t_rtttime = 0; in tcp_do_segment()
2697 if (tcp_is_sack_recovery(tp, &to)) { in tcp_do_segment()
2698 tp->sackhint.prr_delivered = in tcp_do_segment()
2699 tp->sackhint.sacked_bytes; in tcp_do_segment()
2701 tp->sackhint.prr_delivered = in tcp_do_segment()
2702 imin(tp->snd_max - tp->snd_una, in tcp_do_segment()
2704 tp->t_dupacks) * maxseg); in tcp_do_segment()
2706 tp->sackhint.recover_fs = max(1, in tcp_do_segment()
2707 tp->snd_nxt - tp->snd_una); in tcp_do_segment()
2709 if (tcp_is_sack_recovery(tp, &to)) { in tcp_do_segment()
2712 tp->snd_recover = tp->snd_nxt; in tcp_do_segment()
2713 tp->snd_cwnd = maxseg; in tcp_do_segment()
2714 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
2715 if (SEQ_GT(th->th_ack, tp->snd_una)) in tcp_do_segment()
2719 tp->snd_nxt = th->th_ack; in tcp_do_segment()
2720 tp->snd_cwnd = maxseg; in tcp_do_segment()
2721 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
2722 KASSERT(tp->snd_limited <= 2, in tcp_do_segment()
2725 tp->snd_cwnd = tp->snd_ssthresh + in tcp_do_segment()
2727 (tp->t_dupacks - tp->snd_limited); in tcp_do_segment()
2728 if (SEQ_GT(onxt, tp->snd_nxt)) in tcp_do_segment()
2729 tp->snd_nxt = onxt; in tcp_do_segment()
2742 cc_ack_received(tp, th, nsegs, in tcp_do_segment()
2744 uint32_t oldcwnd = tp->snd_cwnd; in tcp_do_segment()
2745 tcp_seq oldsndmax = tp->snd_max; in tcp_do_segment()
2749 KASSERT(tp->t_dupacks == 1 || in tcp_do_segment()
2750 tp->t_dupacks == 2, in tcp_do_segment()
2753 if (tp->t_dupacks == 1) in tcp_do_segment()
2754 tp->snd_limited = 0; in tcp_do_segment()
2755 tp->snd_cwnd = in tcp_do_segment()
2756 (tp->snd_nxt - tp->snd_una) + in tcp_do_segment()
2757 (tp->t_dupacks - tp->snd_limited) * in tcp_do_segment()
2766 (tp->snd_nxt - tp->snd_una); in tcp_do_segment()
2769 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
2770 sent = tp->snd_max - oldsndmax; in tcp_do_segment()
2772 KASSERT((tp->t_dupacks == 2 && in tcp_do_segment()
2773 tp->snd_limited == 0) || in tcp_do_segment()
2775 tp->t_flags & TF_SENTFIN), in tcp_do_segment()
2778 tp->snd_limited = 2; in tcp_do_segment()
2780 ++tp->snd_limited; in tcp_do_segment()
2781 tp->snd_cwnd = oldcwnd; in tcp_do_segment()
2791 tp->t_dupacks = 0; in tcp_do_segment()
2801 if (tcp_is_sack_recovery(tp, &to) && in tcp_do_segment()
2803 tp->t_dupacks++; in tcp_do_segment()
2805 if (!IN_FASTRECOVERY(tp->t_flags) && in tcp_do_segment()
2806 (tp->sackhint.sacked_bytes > in tcp_do_segment()
2808 (maxseg = tcp_maxseg(tp))))) { in tcp_do_segment()
2815 KASSERT(SEQ_GT(th->th_ack, tp->snd_una), in tcp_do_segment()
2822 if (IN_FASTRECOVERY(tp->t_flags)) { in tcp_do_segment()
2823 if (SEQ_LT(th->th_ack, tp->snd_recover)) { in tcp_do_segment()
2824 if (tp->t_flags & TF_SACK_PERMIT) in tcp_do_segment()
2826 tcp_timer_activate(tp, TT_REXMT, 0); in tcp_do_segment()
2827 tp->t_rtttime = 0; in tcp_do_segment()
2828 tcp_do_prr_ack(tp, th, &to); in tcp_do_segment()
2829 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
2830 (void) tcp_output(tp); in tcp_do_segment()
2832 tcp_sack_partialack(tp, th); in tcp_do_segment()
2834 tcp_newreno_partial_ack(tp, th); in tcp_do_segment()
2836 cc_post_recovery(tp, th); in tcp_do_segment()
2837 } else if (IN_CONGRECOVERY(tp->t_flags)) { in tcp_do_segment()
2838 if (SEQ_LT(th->th_ack, tp->snd_recover)) { in tcp_do_segment()
2840 tp->sackhint.delivered_data = BYTES_THIS_ACK(tp, th); in tcp_do_segment()
2841 tp->snd_fack = th->th_ack; in tcp_do_segment()
2842 tcp_do_prr_ack(tp, th, &to); in tcp_do_segment()
2843 (void) tcp_output(tp); in tcp_do_segment()
2846 cc_post_recovery(tp, th); in tcp_do_segment()
2852 if (tp->t_flags & TF_NEEDSYN) { in tcp_do_segment()
2860 tp->t_flags &= ~TF_NEEDSYN; in tcp_do_segment()
2861 tp->snd_una++; in tcp_do_segment()
2863 if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == in tcp_do_segment()
2865 tp->rcv_scale = tp->request_r_scale; in tcp_do_segment()
2871 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_segment()
2881 tp->snd_una++; in tcp_do_segment()
2882 acked = BYTES_THIS_ACK(tp, th); in tcp_do_segment()
2885 tp->snd_una, th->th_ack, tp, m)); in tcp_do_segment()
2896 if (tp->t_rxtshift == 1 && in tcp_do_segment()
2897 tp->t_flags & TF_PREVVALID && in tcp_do_segment()
2898 tp->t_badrxtwin != 0 && in tcp_do_segment()
2901 TSTMP_LT(to.to_tsecr, tp->t_badrxtwin)) in tcp_do_segment()
2902 cc_cong_signal(tp, th, CC_RTO_ERR); in tcp_do_segment()
2922 if (!tp->t_rttlow || tp->t_rttlow > t) in tcp_do_segment()
2923 tp->t_rttlow = t; in tcp_do_segment()
2924 tcp_xmit_timer(tp, TCP_TS_TO_TICKS(t) + 1); in tcp_do_segment()
2925 } else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) { in tcp_do_segment()
2926 if (!tp->t_rttlow || tp->t_rttlow > ticks - tp->t_rtttime) in tcp_do_segment()
2927 tp->t_rttlow = ticks - tp->t_rtttime; in tcp_do_segment()
2928 tcp_xmit_timer(tp, ticks - tp->t_rtttime); in tcp_do_segment()
2937 if (th->th_ack == tp->snd_max) { in tcp_do_segment()
2938 tcp_timer_activate(tp, TT_REXMT, 0); in tcp_do_segment()
2940 } else if (!tcp_timer_active(tp, TT_PERSIST)) in tcp_do_segment()
2941 tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); in tcp_do_segment()
2955 cc_ack_received(tp, th, nsegs, CC_ACK); in tcp_do_segment()
2959 if (tp->snd_wnd >= sbavail(&so->so_snd)) in tcp_do_segment()
2960 tp->snd_wnd -= sbavail(&so->so_snd); in tcp_do_segment()
2962 tp->snd_wnd = 0; in tcp_do_segment()
2968 if (tp->snd_wnd >= (uint32_t) acked) in tcp_do_segment()
2969 tp->snd_wnd -= acked; in tcp_do_segment()
2971 tp->snd_wnd = 0; in tcp_do_segment()
2978 if (!IN_RECOVERY(tp->t_flags) && in tcp_do_segment()
2979 SEQ_GT(tp->snd_una, tp->snd_recover) && in tcp_do_segment()
2980 SEQ_LEQ(th->th_ack, tp->snd_recover)) in tcp_do_segment()
2981 tp->snd_recover = th->th_ack - 1; in tcp_do_segment()
2983 if (IN_RECOVERY(tp->t_flags) && in tcp_do_segment()
2984 SEQ_GEQ(th->th_ack, tp->snd_recover)) { in tcp_do_segment()
2985 EXIT_RECOVERY(tp->t_flags); in tcp_do_segment()
2987 tp->snd_una = th->th_ack; in tcp_do_segment()
2988 if (tp->t_flags & TF_SACK_PERMIT) { in tcp_do_segment()
2989 if (SEQ_GT(tp->snd_una, tp->snd_recover)) in tcp_do_segment()
2990 tp->snd_recover = tp->snd_una; in tcp_do_segment()
2992 if (SEQ_LT(tp->snd_nxt, tp->snd_una)) in tcp_do_segment()
2993 tp->snd_nxt = tp->snd_una; in tcp_do_segment()
2995 switch (tp->t_state) { in tcp_do_segment()
3016 tcp_timer_activate(tp, TT_2MSL, in tcp_do_segment()
3019 TP_MAXIDLE(tp))); in tcp_do_segment()
3021 tcp_state_change(tp, TCPS_FIN_WAIT_2); in tcp_do_segment()
3033 tcp_twstart(tp); in tcp_do_segment()
3047 tp = tcp_close(tp); in tcp_do_segment()
3055 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_segment()
3062 (SEQ_LT(tp->snd_wl1, th->th_seq) || in tcp_do_segment()
3063 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || in tcp_do_segment()
3064 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { in tcp_do_segment()
3067 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) in tcp_do_segment()
3069 tp->snd_wnd = tiwin; in tcp_do_segment()
3070 tp->snd_wl1 = th->th_seq; in tcp_do_segment()
3071 tp->snd_wl2 = th->th_ack; in tcp_do_segment()
3072 if (tp->snd_wnd > tp->max_sndwnd) in tcp_do_segment()
3073 tp->max_sndwnd = tp->snd_wnd; in tcp_do_segment()
3081 TCPS_HAVERCVDFIN(tp->t_state) == 0) { in tcp_do_segment()
3109 if (SEQ_GT(th->th_seq+th->th_urp, tp->rcv_up)) { in tcp_do_segment()
3110 tp->rcv_up = th->th_seq + th->th_urp; in tcp_do_segment()
3112 (tp->rcv_up - tp->rcv_nxt) - 1; in tcp_do_segment()
3116 tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA); in tcp_do_segment()
3136 if (SEQ_GT(tp->rcv_nxt, tp->rcv_up)) in tcp_do_segment()
3137 tp->rcv_up = tp->rcv_nxt; in tcp_do_segment()
3140 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_segment()
3150 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && in tcp_do_segment()
3151 IS_FASTOPEN(tp->t_flags)); in tcp_do_segment()
3153 TCPS_HAVERCVDFIN(tp->t_state) == 0) { in tcp_do_segment()
3155 tcp_seq save_rnxt = tp->rcv_nxt; in tcp_do_segment()
3170 if (th->th_seq == tp->rcv_nxt && in tcp_do_segment()
3171 SEGQ_EMPTY(tp) && in tcp_do_segment()
3172 (TCPS_HAVEESTABLISHED(tp->t_state) || in tcp_do_segment()
3174 if (DELAY_ACK(tp, tlen) || tfo_syn) in tcp_do_segment()
3175 tp->t_flags |= TF_DELACK; in tcp_do_segment()
3177 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
3178 tp->rcv_nxt += tlen; in tcp_do_segment()
3180 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && in tcp_do_segment()
3181 (tp->t_fbyte_in == 0)) { in tcp_do_segment()
3182 tp->t_fbyte_in = ticks; in tcp_do_segment()
3183 if (tp->t_fbyte_in == 0) in tcp_do_segment()
3184 tp->t_fbyte_in = 1; in tcp_do_segment()
3185 if (tp->t_fbyte_out && tp->t_fbyte_in) in tcp_do_segment()
3186 tp->t_flags2 |= TF2_FBYTES_COMPLETE; in tcp_do_segment()
3196 tp->t_flags |= TF_WAKESOR; in tcp_do_segment()
3206 thflags = tcp_reass(tp, th, &temp, &tlen, m); in tcp_do_segment()
3207 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
3209 if ((tp->t_flags & TF_SACK_PERMIT) && in tcp_do_segment()
3211 TCPS_HAVEESTABLISHED(tp->t_state)) { in tcp_do_segment()
3217 tcp_update_sack_list(tp, save_start, in tcp_do_segment()
3219 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) { in tcp_do_segment()
3220 if ((tp->rcv_numsacks >= 1) && in tcp_do_segment()
3221 (tp->sackblks[0].end == save_start)) { in tcp_do_segment()
3226 tcp_update_sack_list(tp, in tcp_do_segment()
3227 tp->sackblks[0].start, in tcp_do_segment()
3228 tp->sackblks[0].end); in tcp_do_segment()
3230 tcp_update_dsack_list(tp, save_start, in tcp_do_segment()
3235 tcp_update_dsack_list(tp, save_start, in tcp_do_segment()
3238 tcp_update_dsack_list(tp, save_start, in tcp_do_segment()
3242 tcp_handle_wakeup(tp, so); in tcp_do_segment()
3250 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) in tcp_do_segment()
3251 len = so->so_rcv.sb_hiwat - (tp->rcv_adv - tp->rcv_nxt); in tcp_do_segment()
3265 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { in tcp_do_segment()
3275 if (tp->t_flags & TF_NEEDSYN) in tcp_do_segment()
3276 tp->t_flags |= TF_DELACK; in tcp_do_segment()
3278 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
3279 tp->rcv_nxt++; in tcp_do_segment()
3281 switch (tp->t_state) { in tcp_do_segment()
3287 tp->t_starttime = ticks; in tcp_do_segment()
3290 tcp_state_change(tp, TCPS_CLOSE_WAIT); in tcp_do_segment()
3298 tcp_state_change(tp, TCPS_CLOSING); in tcp_do_segment()
3307 tcp_twstart(tp); in tcp_do_segment()
3313 tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, in tcp_do_segment()
3316 TCP_PROBE3(debug__input, tp, th, m); in tcp_do_segment()
3321 if (needoutput || (tp->t_flags & TF_ACKNOW)) in tcp_do_segment()
3322 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
3325 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_segment()
3327 if (tp->t_flags & TF_DELACK) { in tcp_do_segment()
3328 tp->t_flags &= ~TF_DELACK; in tcp_do_segment()
3329 tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); in tcp_do_segment()
3331 INP_WUNLOCK(tp->t_inpcb); in tcp_do_segment()
3350 if (tp->t_state == TCPS_SYN_RECEIVED && (thflags & TH_ACK) && in tcp_do_segment()
3351 (SEQ_GT(tp->snd_una, th->th_ack) || in tcp_do_segment()
3352 SEQ_GT(th->th_ack, tp->snd_max)) ) { in tcp_do_segment()
3358 tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen, in tcp_do_segment()
3361 TCP_PROBE3(debug__input, tp, th, m); in tcp_do_segment()
3362 tp->t_flags |= TF_ACKNOW; in tcp_do_segment()
3363 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_do_segment()
3364 INP_WUNLOCK(tp->t_inpcb); in tcp_do_segment()
3369 if (tp != NULL) { in tcp_do_segment()
3370 tcp_dropwithreset(m, th, tp, tlen, rstreason); in tcp_do_segment()
3371 INP_WUNLOCK(tp->t_inpcb); in tcp_do_segment()
3381 if (tp == NULL || (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) in tcp_do_segment()
3382 tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen, in tcp_do_segment()
3385 TCP_PROBE3(debug__input, tp, th, m); in tcp_do_segment()
3386 if (tp != NULL) { in tcp_do_segment()
3387 INP_WUNLOCK(tp->t_inpcb); in tcp_do_segment()
3398 tcp_dropwithreset(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, in tcp_dropwithreset() argument
3408 if (tp != NULL) { in tcp_dropwithreset()
3409 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_dropwithreset()
3444 tcp_respond(tp, mtod(m, void *), th, m, (tcp_seq)0, in tcp_dropwithreset()
3451 tcp_respond(tp, mtod(m, void *), th, m, th->th_seq+tlen, in tcp_dropwithreset()
3579 struct tcpcb *tp = sototcpcb(so); in tcp_pulloutofband() local
3581 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_pulloutofband()
3583 tp->t_iobc = *cp; in tcp_pulloutofband()
3584 tp->t_oobflags |= TCPOOB_HAVEDATA; in tcp_pulloutofband()
3604 tcp_xmit_timer(struct tcpcb *tp, int rtt) in tcp_xmit_timer() argument
3608 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_xmit_timer()
3611 tp->t_rttupdated++; in tcp_xmit_timer()
3613 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, in tcp_xmit_timer()
3616 if ((tp->t_srtt != 0) && (tp->t_rxtshift <= TCP_RTT_INVALIDATE)) { in tcp_xmit_timer()
3625 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT)); in tcp_xmit_timer()
3627 if ((tp->t_srtt += delta) <= 0) in tcp_xmit_timer()
3628 tp->t_srtt = 1; in tcp_xmit_timer()
3642 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT); in tcp_xmit_timer()
3643 if ((tp->t_rttvar += delta) <= 0) in tcp_xmit_timer()
3644 tp->t_rttvar = 1; in tcp_xmit_timer()
3645 if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar) in tcp_xmit_timer()
3646 tp->t_rttbest = tp->t_srtt + tp->t_rttvar; in tcp_xmit_timer()
3653 tp->t_srtt = rtt << TCP_RTT_SHIFT; in tcp_xmit_timer()
3654 tp->t_rttvar = rtt << (TCP_RTTVAR_SHIFT - 1); in tcp_xmit_timer()
3655 tp->t_rttbest = tp->t_srtt + tp->t_rttvar; in tcp_xmit_timer()
3657 tp->t_rtttime = 0; in tcp_xmit_timer()
3658 tp->t_rxtshift = 0; in tcp_xmit_timer()
3671 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp), in tcp_xmit_timer()
3672 max(tp->t_rttmin, rtt + 2), TCPTV_REXMTMAX); in tcp_xmit_timer()
3681 tp->t_softerror = 0; in tcp_xmit_timer()
3706 tcp_mss_update(struct tcpcb *tp, int offer, int mtuoffer, in tcp_mss_update() argument
3711 struct inpcb *inp = tp->t_inpcb; in tcp_mss_update()
3722 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_mss_update()
3724 if (tp->t_port) in tcp_mss_update()
3735 tp->t_maxseg = V_tcp_v6mssdflt; in tcp_mss_update()
3744 tp->t_maxseg = V_tcp_mssdflt; in tcp_mss_update()
3770 offer = tp->t_maxseg; in tcp_mss_update()
3851 tp->t_maxseg = mss; in tcp_mss_update()
3855 tcp_mss(struct tcpcb *tp, int offer) in tcp_mss() argument
3864 KASSERT(tp != NULL, ("%s: tp == NULL", __func__)); in tcp_mss()
3867 tcp_mss_update(tp, offer, -1, &metrics, &cap); in tcp_mss()
3869 mss = tp->t_maxseg; in tcp_mss()
3870 inp = tp->t_inpcb; in tcp_mss()
3903 tp->t_maxseg = max(mss, 64); in tcp_mss()
3921 tp->t_flags |= TF_TSO; in tcp_mss()
3922 tp->t_tsomax = cap.tsomax; in tcp_mss()
3923 tp->t_tsomaxsegcount = cap.tsomaxsegcount; in tcp_mss()
3924 tp->t_tsomaxsegsize = cap.tsomaxsegsize; in tcp_mss()
3971 tcp_do_prr_ack(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to) in tcp_do_prr_ack() argument
3974 int maxseg = tcp_maxseg(tp); in tcp_do_prr_ack()
3976 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_do_prr_ack()
3983 if (tcp_is_sack_recovery(tp, to) || in tcp_do_prr_ack()
3984 (IN_CONGRECOVERY(tp->t_flags) && in tcp_do_prr_ack()
3985 !IN_FASTRECOVERY(tp->t_flags))) { in tcp_do_prr_ack()
3986 del_data = tp->sackhint.delivered_data; in tcp_do_prr_ack()
3988 pipe = tcp_compute_pipe(tp); in tcp_do_prr_ack()
3990 pipe = (tp->snd_nxt - tp->snd_fack) + in tcp_do_prr_ack()
3991 tp->sackhint.sack_bytes_rexmit; in tcp_do_prr_ack()
3993 if (tp->sackhint.prr_delivered < (tcprexmtthresh * maxseg + in tcp_do_prr_ack()
3994 tp->snd_recover - tp->snd_una)) in tcp_do_prr_ack()
3996 pipe = imax(0, tp->snd_max - tp->snd_una - in tcp_do_prr_ack()
3997 imin(INT_MAX / 65536, tp->t_dupacks) * maxseg); in tcp_do_prr_ack()
3999 tp->sackhint.prr_delivered += del_data; in tcp_do_prr_ack()
4003 if (pipe >= tp->snd_ssthresh) { in tcp_do_prr_ack()
4004 if (tp->sackhint.recover_fs == 0) in tcp_do_prr_ack()
4005 tp->sackhint.recover_fs = in tcp_do_prr_ack()
4006 imax(1, tp->snd_nxt - tp->snd_una); in tcp_do_prr_ack()
4007 snd_cnt = howmany((long)tp->sackhint.prr_delivered * in tcp_do_prr_ack()
4008 tp->snd_ssthresh, tp->sackhint.recover_fs) - in tcp_do_prr_ack()
4009 tp->sackhint.prr_out; in tcp_do_prr_ack()
4012 limit = tp->sackhint.prr_delivered - in tcp_do_prr_ack()
4013 tp->sackhint.prr_out; in tcp_do_prr_ack()
4015 limit = imax(tp->sackhint.prr_delivered - in tcp_do_prr_ack()
4016 tp->sackhint.prr_out, del_data) + in tcp_do_prr_ack()
4018 snd_cnt = imin((tp->snd_ssthresh - pipe), limit); in tcp_do_prr_ack()
4026 if (IN_FASTRECOVERY(tp->t_flags)) { in tcp_do_prr_ack()
4027 if (tcp_is_sack_recovery(tp, to)) { in tcp_do_prr_ack()
4028 tp->snd_cwnd = tp->snd_nxt - tp->snd_recover + in tcp_do_prr_ack()
4029 tp->sackhint.sack_bytes_rexmit + in tcp_do_prr_ack()
4032 tp->snd_cwnd = (tp->snd_max - tp->snd_una) + in tcp_do_prr_ack()
4035 } else if (IN_CONGRECOVERY(tp->t_flags)) in tcp_do_prr_ack()
4036 tp->snd_cwnd = pipe - del_data + (snd_cnt * maxseg); in tcp_do_prr_ack()
4037 tp->snd_cwnd = imax(maxseg, tp->snd_cwnd); in tcp_do_prr_ack()
4047 tcp_newreno_partial_ack(struct tcpcb *tp, struct tcphdr *th) in tcp_newreno_partial_ack() argument
4049 tcp_seq onxt = tp->snd_nxt; in tcp_newreno_partial_ack()
4050 uint32_t ocwnd = tp->snd_cwnd; in tcp_newreno_partial_ack()
4051 u_int maxseg = tcp_maxseg(tp); in tcp_newreno_partial_ack()
4053 INP_WLOCK_ASSERT(tp->t_inpcb); in tcp_newreno_partial_ack()
4055 tcp_timer_activate(tp, TT_REXMT, 0); in tcp_newreno_partial_ack()
4056 tp->t_rtttime = 0; in tcp_newreno_partial_ack()
4057 tp->snd_nxt = th->th_ack; in tcp_newreno_partial_ack()
4062 tp->snd_cwnd = maxseg + BYTES_THIS_ACK(tp, th); in tcp_newreno_partial_ack()
4063 tp->t_flags |= TF_ACKNOW; in tcp_newreno_partial_ack()
4064 (void) tp->t_fb->tfb_tcp_output(tp); in tcp_newreno_partial_ack()
4065 tp->snd_cwnd = ocwnd; in tcp_newreno_partial_ack()
4066 if (SEQ_GT(onxt, tp->snd_nxt)) in tcp_newreno_partial_ack()
4067 tp->snd_nxt = onxt; in tcp_newreno_partial_ack()
4072 if (tp->snd_cwnd > BYTES_THIS_ACK(tp, th)) in tcp_newreno_partial_ack()
4073 tp->snd_cwnd -= BYTES_THIS_ACK(tp, th); in tcp_newreno_partial_ack()
4075 tp->snd_cwnd = 0; in tcp_newreno_partial_ack()
4076 tp->snd_cwnd += maxseg; in tcp_newreno_partial_ack()
4080 tcp_compute_pipe(struct tcpcb *tp) in tcp_compute_pipe() argument
4082 return (tp->snd_max - tp->snd_una + in tcp_compute_pipe()
4083 tp->sackhint.sack_bytes_rexmit - in tcp_compute_pipe()
4084 tp->sackhint.sacked_bytes); in tcp_compute_pipe()