413 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,  in sk_msg_recvmsg()  argument
421 	msg_rx = sk_psock_peek_msg(psock);  in sk_msg_recvmsg()
474 			msg_rx = sk_psock_next_msg(psock, msg_rx);  in sk_msg_recvmsg()
482 			msg_rx = sk_psock_dequeue_msg(psock);  in sk_msg_recvmsg()
485 		msg_rx = sk_psock_peek_msg(psock);  in sk_msg_recvmsg()
494 	struct sk_psock *psock;  in sk_msg_is_readable()  local
498 	psock = sk_psock(sk);  in sk_msg_is_readable()
499 	if (likely(psock))  in sk_msg_is_readable()
500 		empty = list_empty(&psock->ingress_msg);  in sk_msg_is_readable()
531 					struct sk_psock *psock,  in sk_psock_skb_ingress_enqueue()  argument
553 	psock->ingress_bytes += len;  in sk_psock_skb_ingress_enqueue()
561 	sk_psock_queue_msg(psock, msg);  in sk_psock_skb_ingress_enqueue()
562 	sk_psock_data_ready(sk, psock);  in sk_psock_skb_ingress_enqueue()
566 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
569 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_skb_ingress()  argument
572 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress()
581 		return sk_psock_skb_ingress_self(psock, skb, off, len);  in sk_psock_skb_ingress()
593 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress()
603 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_skb_ingress_self()  argument
607 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress_self()
613 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress_self()
619 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_handle_skb()  argument
625 		if (!sock_writeable(psock->sk))  in sk_psock_handle_skb()
627 		return skb_send_sock(psock->sk, skb, off, len);  in sk_psock_handle_skb()
630 	err = sk_psock_skb_ingress(psock, skb, off, len);  in sk_psock_handle_skb()
636 static void sk_psock_skb_state(struct sk_psock *psock,  in sk_psock_skb_state()  argument
640 	spin_lock_bh(&psock->ingress_lock);  in sk_psock_skb_state()
641 	if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_skb_state()
645 	spin_unlock_bh(&psock->ingress_lock);  in sk_psock_skb_state()
651 	struct sk_psock *psock = container_of(dwork, struct sk_psock, work);  in sk_psock_backlog()  local
652 	struct sk_psock_work_state *state = &psock->work_state;  in sk_psock_backlog()
658 	mutex_lock(&psock->work_mutex);  in sk_psock_backlog()
664 	while ((skb = skb_peek(&psock->ingress_skb))) {  in sk_psock_backlog()
677 			if (!sock_flag(psock->sk, SOCK_DEAD))  in sk_psock_backlog()
678 				ret = sk_psock_handle_skb(psock, skb, off,  in sk_psock_backlog()
682 					sk_psock_skb_state(psock, state, len, off);  in sk_psock_backlog()
687 					if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_backlog()
688 						schedule_delayed_work(&psock->work, 1);  in sk_psock_backlog()
692 				sk_psock_report_error(psock, ret ? -ret : EPIPE);  in sk_psock_backlog()
693 				sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_backlog()
700 		skb = skb_dequeue(&psock->ingress_skb);  in sk_psock_backlog()
704 	mutex_unlock(&psock->work_mutex);  in sk_psock_backlog()
709 	struct sk_psock *psock;  in sk_psock_init()  local
715 		psock = ERR_PTR(-EINVAL);  in sk_psock_init()
720 		psock = ERR_PTR(-EBUSY);  in sk_psock_init()
724 	psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);  in sk_psock_init()
725 	if (!psock) {  in sk_psock_init()
726 		psock = ERR_PTR(-ENOMEM);  in sk_psock_init()
731 	psock->sk = sk;  in sk_psock_init()
732 	psock->eval = __SK_NONE;  in sk_psock_init()
733 	psock->sk_proto = prot;  in sk_psock_init()
734 	psock->saved_unhash = prot->unhash;  in sk_psock_init()
735 	psock->saved_destroy = prot->destroy;  in sk_psock_init()
736 	psock->saved_close = prot->close;  in sk_psock_init()
737 	psock->saved_write_space = sk->sk_write_space;  in sk_psock_init()
739 	INIT_LIST_HEAD(&psock->link);  in sk_psock_init()
740 	spin_lock_init(&psock->link_lock);  in sk_psock_init()
742 	INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);  in sk_psock_init()
743 	mutex_init(&psock->work_mutex);  in sk_psock_init()
744 	INIT_LIST_HEAD(&psock->ingress_msg);  in sk_psock_init()
745 	spin_lock_init(&psock->ingress_lock);  in sk_psock_init()
746 	skb_queue_head_init(&psock->ingress_skb);  in sk_psock_init()
748 	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_init()
749 	refcount_set(&psock->refcnt, 1);  in sk_psock_init()
751 	__rcu_assign_sk_user_data_with_flags(sk, psock,  in sk_psock_init()
758 	return psock;  in sk_psock_init()
762 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)  in sk_psock_link_pop()  argument
766 	spin_lock_bh(&psock->link_lock);  in sk_psock_link_pop()
767 	link = list_first_entry_or_null(&psock->link, struct sk_psock_link,  in sk_psock_link_pop()
771 	spin_unlock_bh(&psock->link_lock);  in sk_psock_link_pop()
775 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)  in __sk_psock_purge_ingress_msg()  argument
779 	list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {  in __sk_psock_purge_ingress_msg()
782 			atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc);  in __sk_psock_purge_ingress_msg()
783 		sk_msg_free(psock->sk, msg);  in __sk_psock_purge_ingress_msg()
788 static void __sk_psock_zap_ingress(struct sk_psock *psock)  in __sk_psock_zap_ingress()  argument
792 	while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {  in __sk_psock_zap_ingress()
794 		sock_drop(psock->sk, skb);  in __sk_psock_zap_ingress()
796 	__sk_psock_purge_ingress_msg(psock);  in __sk_psock_zap_ingress()
799 static void sk_psock_link_destroy(struct sk_psock *psock)  in sk_psock_link_destroy()  argument
803 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sk_psock_link_destroy()
809 void sk_psock_stop(struct sk_psock *psock)  in sk_psock_stop()  argument
811 	spin_lock_bh(&psock->ingress_lock);  in sk_psock_stop()
812 	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_stop()
813 	sk_psock_cork_free(psock);  in sk_psock_stop()
814 	spin_unlock_bh(&psock->ingress_lock);  in sk_psock_stop()
817 static void sk_psock_done_strp(struct sk_psock *psock);
821 	struct sk_psock *psock = container_of(to_rcu_work(work),  in sk_psock_destroy()  local
825 	sk_psock_done_strp(psock);  in sk_psock_destroy()
827 	cancel_delayed_work_sync(&psock->work);  in sk_psock_destroy()
828 	__sk_psock_zap_ingress(psock);  in sk_psock_destroy()
829 	mutex_destroy(&psock->work_mutex);  in sk_psock_destroy()
831 	psock_progs_drop(&psock->progs);  in sk_psock_destroy()
833 	sk_psock_link_destroy(psock);  in sk_psock_destroy()
834 	sk_psock_cork_free(psock);  in sk_psock_destroy()
836 	if (psock->sk_redir)  in sk_psock_destroy()
837 		sock_put(psock->sk_redir);  in sk_psock_destroy()
838 	if (psock->sk_pair)  in sk_psock_destroy()
839 		sock_put(psock->sk_pair);  in sk_psock_destroy()
840 	sock_put(psock->sk);  in sk_psock_destroy()
841 	kfree(psock);  in sk_psock_destroy()
844 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)  in sk_psock_drop()  argument
847 	sk_psock_restore_proto(sk, psock);  in sk_psock_drop()
849 	if (psock->progs.stream_parser)  in sk_psock_drop()
850 		sk_psock_stop_strp(sk, psock);  in sk_psock_drop()
851 	else if (psock->progs.stream_verdict || psock->progs.skb_verdict)  in sk_psock_drop()
852 		sk_psock_stop_verdict(sk, psock);  in sk_psock_drop()
855 	sk_psock_stop(psock);  in sk_psock_drop()
857 	INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);  in sk_psock_drop()
858 	queue_rcu_work(system_wq, &psock->rwork);  in sk_psock_drop()
875 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,  in sk_psock_msg_verdict()  argument
882 	prog = READ_ONCE(psock->progs.msg_parser);  in sk_psock_msg_verdict()
892 	psock->apply_bytes = msg->apply_bytes;  in sk_psock_msg_verdict()
894 		if (psock->sk_redir) {  in sk_psock_msg_verdict()
895 			sock_put(psock->sk_redir);  in sk_psock_msg_verdict()
896 			psock->sk_redir = NULL;  in sk_psock_msg_verdict()
902 		psock->redir_ingress = sk_msg_to_ingress(msg);  in sk_psock_msg_verdict()
903 		psock->sk_redir = msg->sk_redir;  in sk_psock_msg_verdict()
904 		sock_hold(psock->sk_redir);  in sk_psock_msg_verdict()
964 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)  in sk_psock_tls_strp_read()  argument
970 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_tls_strp_read()
972 		skb->sk = psock->sk;  in sk_psock_tls_strp_read()
979 	sk_psock_tls_verdict_apply(skb, psock, ret);  in sk_psock_tls_strp_read()
985 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_verdict_apply()  argument
995 		sk_other = psock->sk;  in sk_psock_verdict_apply()
997 		    !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_verdict_apply()
1008 		if (skb_queue_empty(&psock->ingress_skb)) {  in sk_psock_verdict_apply()
1017 			err = sk_psock_skb_ingress_self(psock, skb, off, len);  in sk_psock_verdict_apply()
1020 			spin_lock_bh(&psock->ingress_lock);  in sk_psock_verdict_apply()
1021 			if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_verdict_apply()
1022 				skb_queue_tail(&psock->ingress_skb, skb);  in sk_psock_verdict_apply()
1023 				schedule_delayed_work(&psock->work, 0);  in sk_psock_verdict_apply()
1026 			spin_unlock_bh(&psock->ingress_lock);  in sk_psock_verdict_apply()
1032 		tcp_eat_skb(psock->sk, skb);  in sk_psock_verdict_apply()
1033 		err = sk_psock_skb_redirect(psock, skb);  in sk_psock_verdict_apply()
1039 		tcp_eat_skb(psock->sk, skb);  in sk_psock_verdict_apply()
1040 		sock_drop(psock->sk, skb);  in sk_psock_verdict_apply()
1048 	struct sk_psock *psock;  in sk_psock_write_space()  local
1052 	psock = sk_psock(sk);  in sk_psock_write_space()
1053 	if (likely(psock)) {  in sk_psock_write_space()
1054 		if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_write_space()
1055 			schedule_delayed_work(&psock->work, 0);  in sk_psock_write_space()
1056 		write_space = psock->saved_write_space;  in sk_psock_write_space()
1066 	struct sk_psock *psock;  in sk_psock_strp_read()  local
1073 	psock = sk_psock(sk);  in sk_psock_strp_read()
1074 	if (unlikely(!psock)) {  in sk_psock_strp_read()
1078 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_strp_read()
1088 	sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_strp_read()
1100 	struct sk_psock *psock = container_of(strp, struct sk_psock, strp);  in sk_psock_strp_parse()  local
1105 	prog = READ_ONCE(psock->progs.stream_parser);  in sk_psock_strp_parse()
1107 		skb->sk = psock->sk;  in sk_psock_strp_parse()
1118 	struct sk_psock *psock;  in sk_psock_strp_data_ready()  local
1123 	psock = sk_psock(sk);  in sk_psock_strp_data_ready()
1124 	if (likely(psock)) {  in sk_psock_strp_data_ready()
1126 			psock->saved_data_ready(sk);  in sk_psock_strp_data_ready()
1129 			strp_data_ready(&psock->strp);  in sk_psock_strp_data_ready()
1136 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_init_strp()  argument
1146 	ret = strp_init(&psock->strp, sk, &cb);  in sk_psock_init_strp()
1148 		sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);  in sk_psock_init_strp()
1151 		psock->strp.cb.read_sock = tcp_bpf_strp_read_sock;  in sk_psock_init_strp()
1152 		psock->copied_seq = tcp_sk(sk)->copied_seq;  in sk_psock_init_strp()
1157 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_strp()  argument
1159 	if (psock->saved_data_ready)  in sk_psock_start_strp()
1162 	psock->saved_data_ready = sk->sk_data_ready;  in sk_psock_start_strp()
1167 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_strp()  argument
1169 	psock_set_prog(&psock->progs.stream_parser, NULL);  in sk_psock_stop_strp()
1171 	if (!psock->saved_data_ready)  in sk_psock_stop_strp()
1174 	sk->sk_data_ready = psock->saved_data_ready;  in sk_psock_stop_strp()
1175 	psock->saved_data_ready = NULL;  in sk_psock_stop_strp()
1176 	strp_stop(&psock->strp);  in sk_psock_stop_strp()
1179 static void sk_psock_done_strp(struct sk_psock *psock)  in sk_psock_done_strp()  argument
1182 	if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))  in sk_psock_done_strp()
1183 		strp_done(&psock->strp);  in sk_psock_done_strp()
1186 static void sk_psock_done_strp(struct sk_psock *psock)  in sk_psock_done_strp()  argument
1193 	struct sk_psock *psock;  in sk_psock_verdict_recv()  local
1199 	psock = sk_psock(sk);  in sk_psock_verdict_recv()
1200 	if (unlikely(!psock)) {  in sk_psock_verdict_recv()
1206 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_verdict_recv()
1208 		prog = READ_ONCE(psock->progs.skb_verdict);  in sk_psock_verdict_recv()
1215 	ret = sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_verdict_recv()
1238 		struct sk_psock *psock;  in sk_psock_verdict_data_ready()  local
1241 		psock = sk_psock(sk);  in sk_psock_verdict_data_ready()
1242 		if (psock)  in sk_psock_verdict_data_ready()
1243 			sk_psock_data_ready(sk, psock);  in sk_psock_verdict_data_ready()
1248 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_verdict()  argument
1250 	if (psock->saved_data_ready)  in sk_psock_start_verdict()
1253 	psock->saved_data_ready = sk->sk_data_ready;  in sk_psock_start_verdict()
1258 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_verdict()  argument
1260 	psock_set_prog(&psock->progs.stream_verdict, NULL);  in sk_psock_stop_verdict()
1261 	psock_set_prog(&psock->progs.skb_verdict, NULL);  in sk_psock_stop_verdict()
1263 	if (!psock->saved_data_ready)  in sk_psock_stop_verdict()
1266 	sk->sk_data_ready = psock->saved_data_ready;  in sk_psock_stop_verdict()
1267 	psock->saved_data_ready = NULL;  in sk_psock_stop_verdict()