273 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,  in __mptcp_move_skb()  argument
277 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skb()
282 	__skb_unlink(skb, &ssk->sk_receive_queue);  in __mptcp_move_skb()
422 	const struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_timeout_from_subflow()  local
424 	return inet_csk(ssk)->icsk_pending && !subflow->stale_count ?  in mptcp_timeout_from_subflow()
425 	       icsk_timeout(inet_csk(ssk)) - jiffies : 0;  in mptcp_timeout_from_subflow()
438 static inline bool tcp_can_send_ack(const struct sock *ssk)  in tcp_can_send_ack()  argument
440 	return !((1 << inet_sk_state_load(ssk)) &  in tcp_can_send_ack()
444 void __mptcp_subflow_send_ack(struct sock *ssk)  in __mptcp_subflow_send_ack()  argument
446 	if (tcp_can_send_ack(ssk))  in __mptcp_subflow_send_ack()
447 		tcp_send_ack(ssk);  in __mptcp_subflow_send_ack()
450 static void mptcp_subflow_send_ack(struct sock *ssk)  in mptcp_subflow_send_ack()  argument
454 	slow = lock_sock_fast(ssk);  in mptcp_subflow_send_ack()
455 	__mptcp_subflow_send_ack(ssk);  in mptcp_subflow_send_ack()
456 	unlock_sock_fast(ssk, slow);  in mptcp_subflow_send_ack()
467 static void mptcp_subflow_cleanup_rbuf(struct sock *ssk, int copied)  in mptcp_subflow_cleanup_rbuf()  argument
471 	slow = lock_sock_fast(ssk);  in mptcp_subflow_cleanup_rbuf()
472 	if (tcp_can_send_ack(ssk))  in mptcp_subflow_cleanup_rbuf()
473 		tcp_cleanup_rbuf(ssk, copied);  in mptcp_subflow_cleanup_rbuf()
474 	unlock_sock_fast(ssk, slow);  in mptcp_subflow_cleanup_rbuf()
477 static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty)  in mptcp_subflow_could_cleanup()  argument
479 	const struct inet_connection_sock *icsk = inet_csk(ssk);  in mptcp_subflow_could_cleanup()
481 	const struct tcp_sock *tp = tcp_sk(ssk);  in mptcp_subflow_could_cleanup()
502 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_cleanup_rbuf()  local
504 		if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty))  in mptcp_cleanup_rbuf()
505 			mptcp_subflow_cleanup_rbuf(ssk, copied);  in mptcp_cleanup_rbuf()
559 static void mptcp_dss_corruption(struct mptcp_sock *msk, struct sock *ssk)  in mptcp_dss_corruption()  argument
562 		MPTCP_INC_STATS(sock_net(ssk),  in mptcp_dss_corruption()
564 		mptcp_do_fallback(ssk);  in mptcp_dss_corruption()
566 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSCORRUPTIONRESET);  in mptcp_dss_corruption()
567 		mptcp_subflow_reset(ssk);  in mptcp_dss_corruption()
572 					   struct sock *ssk)  in __mptcp_move_skbs_from_subflow()  argument
574 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skbs_from_subflow()
580 	pr_debug("msk=%p ssk=%p\n", msk, ssk);  in __mptcp_move_skbs_from_subflow()
581 	tp = tcp_sk(ssk);  in __mptcp_move_skbs_from_subflow()
595 		skb = skb_peek(&ssk->sk_receive_queue);  in __mptcp_move_skbs_from_subflow()
616 			ret = __mptcp_move_skb(msk, ssk, skb, offset, len) || ret;  in __mptcp_move_skbs_from_subflow()
621 				mptcp_dss_corruption(msk, ssk);  in __mptcp_move_skbs_from_subflow()
626 				mptcp_dss_corruption(msk, ssk);  in __mptcp_move_skbs_from_subflow()
629 			sk_eat_skb(ssk, skb);  in __mptcp_move_skbs_from_subflow()
633 		more_data_avail = mptcp_subflow_data_available(ssk);  in __mptcp_move_skbs_from_subflow()
687 static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk)  in __mptcp_subflow_error_report()  argument
689 	int err = sock_error(ssk);  in __mptcp_subflow_error_report()
706 	ssk_state = inet_sk_state_load(ssk);  in __mptcp_subflow_error_report()
730 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)  in move_skbs_to_msk()  argument
735 	moved = __mptcp_move_skbs_from_subflow(msk, ssk);  in move_skbs_to_msk()
737 	if (unlikely(ssk->sk_err)) {  in move_skbs_to_msk()
754 static void __mptcp_rcvbuf_update(struct sock *sk, struct sock *ssk)  in __mptcp_rcvbuf_update()  argument
756 	if (unlikely(ssk->sk_rcvbuf > sk->sk_rcvbuf))  in __mptcp_rcvbuf_update()
757 		WRITE_ONCE(sk->sk_rcvbuf, ssk->sk_rcvbuf);  in __mptcp_rcvbuf_update()
760 static void __mptcp_data_ready(struct sock *sk, struct sock *ssk)  in __mptcp_data_ready()  argument
764 	__mptcp_rcvbuf_update(sk, ssk);  in __mptcp_data_ready()
767 	if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk))  in __mptcp_data_ready()
771 void mptcp_data_ready(struct sock *sk, struct sock *ssk)  in mptcp_data_ready()  argument
773 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_data_ready()
784 		__mptcp_data_ready(sk, ssk);  in mptcp_data_ready()
790 static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk)  in mptcp_subflow_joined()  argument
792 	mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq);  in mptcp_subflow_joined()
794 	mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC);  in mptcp_subflow_joined()
797 static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk)  in __mptcp_finish_join()  argument
807 	if (sk->sk_socket && !ssk->sk_socket)  in __mptcp_finish_join()
808 		mptcp_sock_graft(ssk, sk->sk_socket);  in __mptcp_finish_join()
810 	mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++;  in __mptcp_finish_join()
811 	mptcp_sockopt_sync_locked(msk, ssk);  in __mptcp_finish_join()
812 	mptcp_subflow_joined(msk, ssk);  in __mptcp_finish_join()
814 	__mptcp_propagate_sndbuf(sk, ssk);  in __mptcp_finish_join()
824 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_flush_join_list()  local
825 		bool slow = lock_sock_fast(ssk);  in __mptcp_flush_join_list()
828 		if (!__mptcp_finish_join(msk, ssk))  in __mptcp_flush_join_list()
829 			mptcp_subflow_reset(ssk);  in __mptcp_flush_join_list()
830 		unlock_sock_fast(ssk, slow);  in __mptcp_flush_join_list()
991 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_enter_memory_pressure()  local
994 			tcp_enter_memory_pressure(ssk);  in mptcp_enter_memory_pressure()
995 		sk_stream_moderate_sndbuf(ssk);  in mptcp_enter_memory_pressure()
1042 static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk,  in mptcp_check_allowed_size()  argument
1054 	if (unlikely(tcp_sk(ssk)->snd_wnd < mptcp_snd_wnd)) {  in mptcp_check_allowed_size()
1055 		tcp_sk(ssk)->snd_wnd = min_t(u64, U32_MAX, mptcp_snd_wnd);  in mptcp_check_allowed_size()
1056 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_SNDWNDSHARED);  in mptcp_check_allowed_size()
1091 static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp)  in __mptcp_alloc_tx_skb()  argument
1099 	if (likely(sk_wmem_schedule(ssk, skb->truesize))) {  in __mptcp_alloc_tx_skb()
1100 		tcp_skb_entail(ssk, skb);  in __mptcp_alloc_tx_skb()
1108 static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held)  in mptcp_alloc_tx_skb()  argument
1112 	return __mptcp_alloc_tx_skb(sk, ssk, gfp);  in mptcp_alloc_tx_skb()
1128 				      struct sock *ssk,  in mptcp_update_infinite_map()  argument
1137 	MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPTX);  in mptcp_update_infinite_map()
1138 	mptcp_subflow_ctx(ssk)->send_infinite_map = 0;  in mptcp_update_infinite_map()
1140 	mptcp_do_fallback(ssk);  in mptcp_update_infinite_map()
1145 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,  in mptcp_sendmsg_frag()  argument
1161 		 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);  in mptcp_sendmsg_frag()
1167 	if (unlikely(!__tcp_can_send(ssk)))  in mptcp_sendmsg_frag()
1171 	if (unlikely(ssk->sk_gso_max_size > MPTCP_MAX_GSO_SIZE))  in mptcp_sendmsg_frag()
1172 		ssk->sk_gso_max_size = MPTCP_MAX_GSO_SIZE;  in mptcp_sendmsg_frag()
1173 	info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags);  in mptcp_sendmsg_frag()
1176 	skb = tcp_write_queue_tail(ssk);  in mptcp_sendmsg_frag()
1187 			tcp_mark_push(tcp_sk(ssk), skb);  in mptcp_sendmsg_frag()
1194 			tcp_mark_push(tcp_sk(ssk), skb);  in mptcp_sendmsg_frag()
1201 		skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held);  in mptcp_sendmsg_frag()
1211 	copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy);  in mptcp_sendmsg_frag()
1215 		if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) {  in mptcp_sendmsg_frag()
1216 			tcp_remove_empty_skb(ssk);  in mptcp_sendmsg_frag()
1226 	if (!sk_wmem_schedule(ssk, copy)) {  in mptcp_sendmsg_frag()
1227 		tcp_remove_empty_skb(ssk);  in mptcp_sendmsg_frag()
1241 	sk_wmem_queued_add(ssk, copy);  in mptcp_sendmsg_frag()
1242 	sk_mem_charge(ssk, copy);  in mptcp_sendmsg_frag()
1243 	WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy);  in mptcp_sendmsg_frag()
1256 	mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;  in mptcp_sendmsg_frag()
1266 		mptcp_subflow_ctx(ssk)->rel_write_seq += copy;  in mptcp_sendmsg_frag()
1270 		tcp_push_pending_frames(ssk);  in mptcp_sendmsg_frag()
1276 	if (mptcp_subflow_ctx(ssk)->send_infinite_map)  in mptcp_sendmsg_frag()
1277 		mptcp_update_infinite_map(msk, ssk, mpext);  in mptcp_sendmsg_frag()
1279 	mptcp_subflow_ctx(ssk)->rel_write_seq += copy;  in mptcp_sendmsg_frag()
1290 	struct sock *ssk;  member
1331 	struct sock *ssk;  in mptcp_subflow_get_send()  local
1337 		send_info[i].ssk = NULL;  in mptcp_subflow_get_send()
1345 		ssk =  mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_send()
1354 			subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate);  in mptcp_subflow_get_send()
1360 		linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace);  in mptcp_subflow_get_send()
1362 			send_info[backup].ssk = ssk;  in mptcp_subflow_get_send()
1370 		send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk;  in mptcp_subflow_get_send()
1383 	ssk = send_info[SSK_MODE_ACTIVE].ssk;  in mptcp_subflow_get_send()
1384 	if (!ssk || !sk_stream_memory_free(ssk))  in mptcp_subflow_get_send()
1388 	wmem = READ_ONCE(ssk->sk_wmem_queued);  in mptcp_subflow_get_send()
1390 		return ssk;  in mptcp_subflow_get_send()
1392 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_get_send()
1394 					   READ_ONCE(ssk->sk_pacing_rate) * burst,  in mptcp_subflow_get_send()
1397 	return ssk;  in mptcp_subflow_get_send()
1400 static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info)  in mptcp_push_release()  argument
1402 	tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal);  in mptcp_push_release()
1403 	release_sock(ssk);  in mptcp_push_release()
1442 static int __subflow_push_pending(struct sock *sk, struct sock *ssk,  in __subflow_push_pending()  argument
1456 			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info);  in __subflow_push_pending()
1471 		    !sk_stream_memory_free(ssk) ||  in __subflow_push_pending()
1472 		    !mptcp_subflow_active(mptcp_subflow_ctx(ssk))) {  in __subflow_push_pending()
1488 	struct sock *prev_ssk = NULL, *ssk = NULL;  in __mptcp_push_pending()  local
1509 				prev_ssk = ssk;  in __mptcp_push_pending()
1510 				ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_push_pending()
1511 				if (ssk != prev_ssk) {  in __mptcp_push_pending()
1522 					lock_sock(ssk);  in __mptcp_push_pending()
1527 				ret = __subflow_push_pending(sk, ssk, &info);  in __mptcp_push_pending()
1530 					    (1 << ssk->sk_state) &  in __mptcp_push_pending()
1541 	if (ssk)  in __mptcp_push_pending()
1542 		mptcp_push_release(ssk, &info);  in __mptcp_push_pending()
1551 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first)  in __mptcp_subflow_push_pending()  argument
1563 		struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_subflow_push_pending()
1571 			ret = __subflow_push_pending(sk, ssk, &info);  in __mptcp_subflow_push_pending()
1584 			ret = __subflow_push_pending(sk, ssk, &info);  in __mptcp_subflow_push_pending()
1593 				if (xmit_ssk != ssk) {  in __mptcp_subflow_push_pending()
1607 		tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,  in __mptcp_subflow_push_pending()
1625 	struct sock *ssk;  in mptcp_sendmsg_fastopen()  local
1635 		ssk = __mptcp_nmpc_sk(msk);  in mptcp_sendmsg_fastopen()
1636 		if (IS_ERR(ssk))  in mptcp_sendmsg_fastopen()
1637 			return PTR_ERR(ssk);  in mptcp_sendmsg_fastopen()
1642 	ssk = msk->first;  in mptcp_sendmsg_fastopen()
1644 	lock_sock(ssk);  in mptcp_sendmsg_fastopen()
1647 	ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL);  in mptcp_sendmsg_fastopen()
1650 	release_sock(ssk);  in mptcp_sendmsg_fastopen()
1984 				struct sock *ssk;  in mptcp_rcv_space_adjust()  local
1987 				ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_rcv_space_adjust()
1988 				slow = lock_sock_fast(ssk);  in mptcp_rcv_space_adjust()
1989 				WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf);  in mptcp_rcv_space_adjust()
1990 				WRITE_ONCE(tcp_sk(ssk)->window_clamp, window_clamp);  in mptcp_rcv_space_adjust()
1991 				if (tcp_can_send_ack(ssk))  in mptcp_rcv_space_adjust()
1992 					tcp_cleanup_rbuf(ssk, 1);  in mptcp_rcv_space_adjust()
1993 				unlock_sock_fast(ssk, slow);  in mptcp_rcv_space_adjust()
2035 		struct sock *ssk;  in __mptcp_move_skbs()  local
2049 		ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_move_skbs()
2050 		slowpath = lock_sock_fast(ssk);  in __mptcp_move_skbs()
2051 		ret = __mptcp_move_skbs_from_subflow(msk, ssk) || ret;  in __mptcp_move_skbs()
2052 		if (unlikely(ssk->sk_err))  in __mptcp_move_skbs()
2054 		unlock_sock_fast(ssk, slowpath);  in __mptcp_move_skbs()
2242 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_retrans()  local
2248 		if (!tcp_rtx_and_write_queues_empty(ssk)) {  in mptcp_subflow_get_retrans()
2249 			mptcp_pm_subflow_chk_stale(msk, ssk);  in mptcp_subflow_get_retrans()
2256 				backup = ssk;  in mptcp_subflow_get_retrans()
2261 			pick = ssk;  in mptcp_subflow_get_retrans()
2316 static void __mptcp_subflow_disconnect(struct sock *ssk,  in __mptcp_subflow_disconnect()  argument
2320 	if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) ||  in __mptcp_subflow_disconnect()
2325 		WARN_ON_ONCE(tcp_disconnect(ssk, 0));  in __mptcp_subflow_disconnect()
2328 		tcp_shutdown(ssk, SEND_SHUTDOWN);  in __mptcp_subflow_disconnect()
2340 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,  in __mptcp_close_ssk()  argument
2352 	if (msk->in_accept_queue && msk->first == ssk &&  in __mptcp_close_ssk()
2353 	    (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) {  in __mptcp_close_ssk()
2357 		lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);  in __mptcp_close_ssk()
2358 		mptcp_subflow_drop_ctx(ssk);  in __mptcp_close_ssk()
2362 	dispose_it = msk->free_first || ssk != msk->first;  in __mptcp_close_ssk()
2366 	lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);  in __mptcp_close_ssk()
2372 		ssk->sk_lingertime = 0;  in __mptcp_close_ssk()
2373 		sock_set_flag(ssk, SOCK_LINGER);  in __mptcp_close_ssk()
2379 		__mptcp_subflow_disconnect(ssk, subflow, flags);  in __mptcp_close_ssk()
2380 		release_sock(ssk);  in __mptcp_close_ssk()
2391 	if (!inet_csk(ssk)->icsk_ulp_ops) {  in __mptcp_close_ssk()
2392 		WARN_ON_ONCE(!sock_flag(ssk, SOCK_DEAD));  in __mptcp_close_ssk()
2396 		__tcp_close(ssk, 0);  in __mptcp_close_ssk()
2399 		__sock_put(ssk);  in __mptcp_close_ssk()
2403 	__mptcp_subflow_error_report(sk, ssk);  in __mptcp_close_ssk()
2404 	release_sock(ssk);  in __mptcp_close_ssk()
2406 	sock_put(ssk);  in __mptcp_close_ssk()
2408 	if (ssk == msk->first)  in __mptcp_close_ssk()
2433 void mptcp_close_ssk(struct sock *sk, struct sock *ssk,  in mptcp_close_ssk()  argument
2443 		mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL);  in mptcp_close_ssk()
2450 	__mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH);  in mptcp_close_ssk()
2466 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close_subflow()  local
2467 		int ssk_state = inet_sk_state_load(ssk);  in __mptcp_close_subflow()
2475 		if (!skb_queue_empty_lockless(&ssk->sk_receive_queue))  in __mptcp_close_subflow()
2478 		mptcp_close_ssk(sk, ssk, subflow);  in __mptcp_close_subflow()
2548 	struct sock *ssk;  in __mptcp_retrans()  local
2583 			ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_retrans()
2585 			lock_sock(ssk);  in __mptcp_retrans()
2592 				ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_retrans()
2602 				tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,  in __mptcp_retrans()
2607 			release_sock(ssk);  in __mptcp_retrans()
2645 	struct sock *ssk = msk->first;  in mptcp_mp_fail_no_response()  local
2648 	if (!ssk)  in mptcp_mp_fail_no_response()
2653 	slow = lock_sock_fast(ssk);  in mptcp_mp_fail_no_response()
2654 	mptcp_subflow_reset(ssk);  in mptcp_mp_fail_no_response()
2655 	WRITE_ONCE(mptcp_subflow_ctx(ssk)->fail_tout, 0);  in mptcp_mp_fail_no_response()
2656 	unlock_sock_fast(ssk, slow);  in mptcp_mp_fail_no_response()
2810 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)  in mptcp_subflow_shutdown()  argument
2812 	lock_sock(ssk);  in mptcp_subflow_shutdown()
2814 	switch (ssk->sk_state) {  in mptcp_subflow_shutdown()
2820 		WARN_ON_ONCE(tcp_disconnect(ssk, O_NONBLOCK));  in mptcp_subflow_shutdown()
2825 			ssk->sk_shutdown |= how;  in mptcp_subflow_shutdown()
2826 			tcp_shutdown(ssk, how);  in mptcp_subflow_shutdown()
2834 			pr_debug("Sending DATA_FIN on subflow %p\n", ssk);  in mptcp_subflow_shutdown()
2835 			tcp_send_ack(ssk);  in mptcp_subflow_shutdown()
2842 	release_sock(ssk);  in mptcp_subflow_shutdown()
2970 	struct sock *ssk;  in mptcp_check_listen_stop()  local
2976 	ssk = mptcp_sk(sk)->first;  in mptcp_check_listen_stop()
2977 	if (WARN_ON_ONCE(!ssk || inet_sk_state_load(ssk) != TCP_LISTEN))  in mptcp_check_listen_stop()
2980 	lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);  in mptcp_check_listen_stop()
2981 	tcp_set_state(ssk, TCP_CLOSE);  in mptcp_check_listen_stop()
2982 	mptcp_subflow_queue_clean(sk, ssk);  in mptcp_check_listen_stop()
2983 	inet_csk_listen_stop(ssk);  in mptcp_check_listen_stop()
2984 	mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED);  in mptcp_check_listen_stop()
2985 	release_sock(ssk);  in mptcp_check_listen_stop()
3018 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close()  local
3019 		bool slow = lock_sock_fast_nested(ssk);  in __mptcp_close()
3021 		subflows_alive += ssk->sk_state != TCP_CLOSE;  in __mptcp_close()
3026 		if (ssk == msk->first)  in __mptcp_close()
3032 		ssk->sk_socket = NULL;  in __mptcp_close()
3033 		ssk->sk_wq = NULL;  in __mptcp_close()
3034 		unlock_sock_fast(ssk, slow);  in __mptcp_close()
3072 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)  in mptcp_copy_inaddrs()  argument
3075 	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);  in mptcp_copy_inaddrs()
3078 	msk->sk_v6_daddr = ssk->sk_v6_daddr;  in mptcp_copy_inaddrs()
3079 	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;  in mptcp_copy_inaddrs()
3087 	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;  in mptcp_copy_inaddrs()
3088 	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;  in mptcp_copy_inaddrs()
3089 	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;  in mptcp_copy_inaddrs()
3090 	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;  in mptcp_copy_inaddrs()
3091 	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;  in mptcp_copy_inaddrs()
3092 	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;  in mptcp_copy_inaddrs()
3192 				 struct sock *ssk,  in mptcp_sk_clone_init()  argument
3228 	WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd);  in mptcp_sk_clone_init()
3244 	WRITE_ONCE(msk->first, ssk);  in mptcp_sk_clone_init()
3245 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_sk_clone_init()
3247 	sock_hold(ssk);  in mptcp_sk_clone_init()
3257 	mptcp_copy_inaddrs(nsk, ssk);  in mptcp_sk_clone_init()
3258 	__mptcp_propagate_sndbuf(nsk, ssk);  in mptcp_sk_clone_init()
3260 	mptcp_rcv_space_init(msk, ssk);  in mptcp_sk_clone_init()
3270 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)  in mptcp_rcv_space_init()  argument
3272 	const struct tcp_sock *tp = tcp_sk(ssk);  in mptcp_rcv_space_init()
3326 void __mptcp_check_push(struct sock *sk, struct sock *ssk)  in __mptcp_check_push()  argument
3332 		__mptcp_subflow_push_pending(sk, ssk, false);  in __mptcp_check_push()
3404 static void schedule_3rdack_retransmission(struct sock *ssk)  in schedule_3rdack_retransmission()  argument
3406 	struct inet_connection_sock *icsk = inet_csk(ssk);  in schedule_3rdack_retransmission()
3407 	struct tcp_sock *tp = tcp_sk(ssk);  in schedule_3rdack_retransmission()
3410 	if (READ_ONCE(mptcp_subflow_ctx(ssk)->fully_established))  in schedule_3rdack_retransmission()
3423 	sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout);  in schedule_3rdack_retransmission()
3426 void mptcp_subflow_process_delegated(struct sock *ssk, long status)  in mptcp_subflow_process_delegated()  argument
3428 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_process_delegated()
3434 			__mptcp_subflow_push_pending(sk, ssk, true);  in mptcp_subflow_process_delegated()
3448 		schedule_3rdack_retransmission(ssk);  in mptcp_subflow_process_delegated()
3476 void mptcp_finish_connect(struct sock *ssk)  in mptcp_finish_connect()  argument
3482 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_connect()
3496 	mptcp_pm_new_connection(msk, ssk, 0);  in mptcp_finish_connect()
3508 bool mptcp_finish_join(struct sock *ssk)  in mptcp_finish_join()  argument
3510 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_join()
3525 		mptcp_subflow_joined(msk, ssk);  in mptcp_finish_join()
3526 		mptcp_propagate_sndbuf(parent, ssk);  in mptcp_finish_join()
3538 		ret = __mptcp_finish_join(msk, ssk);  in mptcp_finish_join()
3540 			sock_hold(ssk);  in mptcp_finish_join()
3544 		sock_hold(ssk);  in mptcp_finish_join()
3634 	struct sock *ssk;  in mptcp_connect()  local
3636 	ssk = __mptcp_nmpc_sk(msk);  in mptcp_connect()
3637 	if (IS_ERR(ssk))  in mptcp_connect()
3638 		return PTR_ERR(ssk);  in mptcp_connect()
3641 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_connect()
3646 	if (rcu_access_pointer(tcp_sk(ssk)->md5sig_info))  in mptcp_connect()
3651 			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEACTIVEDISABLED);  in mptcp_connect()
3653 		} else if (mptcp_token_new_connect(ssk) < 0) {  in mptcp_connect()
3654 			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_TOKENFALLBACKINIT);  in mptcp_connect()
3669 		lock_sock(ssk);  in mptcp_connect()
3674 	if (ssk->sk_state != TCP_CLOSE)  in mptcp_connect()
3677 	if (BPF_CGROUP_PRE_CONNECT_ENABLED(ssk)) {  in mptcp_connect()
3678 		err = ssk->sk_prot->pre_connect(ssk, uaddr, addr_len);  in mptcp_connect()
3683 	err = ssk->sk_prot->connect(ssk, uaddr, addr_len);  in mptcp_connect()
3687 	inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk));  in mptcp_connect()
3691 		release_sock(ssk);  in mptcp_connect()
3703 	mptcp_copy_inaddrs(sk, ssk);  in mptcp_connect()
3743 	struct sock *ssk, *sk = sock->sk;  in mptcp_bind()  local
3747 	ssk = __mptcp_nmpc_sk(msk);  in mptcp_bind()
3748 	if (IS_ERR(ssk)) {  in mptcp_bind()
3749 		err = PTR_ERR(ssk);  in mptcp_bind()
3754 		err = inet_bind_sk(ssk, uaddr, addr_len);  in mptcp_bind()
3757 		err = inet6_bind_sk(ssk, uaddr, addr_len);  in mptcp_bind()
3760 		mptcp_copy_inaddrs(sk, ssk);  in mptcp_bind()
3771 	struct sock *ssk;  in mptcp_listen()  local
3782 	ssk = __mptcp_nmpc_sk(msk);  in mptcp_listen()
3783 	if (IS_ERR(ssk)) {  in mptcp_listen()
3784 		err = PTR_ERR(ssk);  in mptcp_listen()
3791 	lock_sock(ssk);  in mptcp_listen()
3792 	err = __inet_listen_sk(ssk, backlog);  in mptcp_listen()
3793 	release_sock(ssk);  in mptcp_listen()
3794 	mptcp_set_state(sk, inet_sk_state_load(ssk));  in mptcp_listen()
3798 		mptcp_copy_inaddrs(sk, ssk);  in mptcp_listen()
3799 		mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED);  in mptcp_listen()
3811 	struct sock *ssk, *newsk;  in mptcp_stream_accept()  local
3818 	ssk = READ_ONCE(msk->first);  in mptcp_stream_accept()
3819 	if (!ssk)  in mptcp_stream_accept()
3822 	pr_debug("ssk=%p, listener=%p\n", ssk, mptcp_subflow_ctx(ssk));  in mptcp_stream_accept()
3823 	newsk = inet_csk_accept(ssk, arg);  in mptcp_stream_accept()
3844 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK);  in mptcp_stream_accept()
3858 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_stream_accept()  local
3860 			if (!ssk->sk_socket)  in mptcp_stream_accept()
3861 				mptcp_sock_graft(ssk, newsock);  in mptcp_stream_accept()
3922 		struct sock *ssk = READ_ONCE(msk->first);  in mptcp_poll()  local
3924 		if (WARN_ON_ONCE(!ssk))  in mptcp_poll()
3927 		return inet_csk_listen_poll(ssk);  in mptcp_poll()
3994 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_napi_poll()  local
3996 		bh_lock_sock_nested(ssk);  in mptcp_napi_poll()
3997 		if (!sock_owned_by_user(ssk)) {  in mptcp_napi_poll()
3998 			mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0));  in mptcp_napi_poll()
4008 		bh_unlock_sock(ssk);  in mptcp_napi_poll()
4009 		sock_put(ssk);  in mptcp_napi_poll()