1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ 3 4 #ifndef _LINUX_SKMSG_H 5 #define _LINUX_SKMSG_H 6 7 #include <linux/bpf.h> 8 #include <linux/filter.h> 9 #include <linux/scatterlist.h> 10 #include <linux/skbuff.h> 11 12 #include <net/sock.h> 13 #include <net/tcp.h> 14 #include <net/strparser.h> 15 16 #define MAX_MSG_FRAGS MAX_SKB_FRAGS 17 18 enum __sk_action { 19 __SK_DROP = 0, 20 __SK_PASS, 21 __SK_REDIRECT, 22 __SK_NONE, 23 }; 24 25 struct sk_msg_sg { 26 u32 start; 27 u32 curr; 28 u32 end; 29 u32 size; 30 u32 copybreak; 31 unsigned long copy; 32 /* The extra element is used for chaining the front and sections when 33 * the list becomes partitioned (e.g. end < start). The crypto APIs 34 * require the chaining. 35 */ 36 struct scatterlist data[MAX_MSG_FRAGS + 1]; 37 }; 38 static_assert(BITS_PER_LONG >= MAX_MSG_FRAGS); 39 40 /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ 41 struct sk_msg { 42 struct sk_msg_sg sg; 43 void *data; 44 void *data_end; 45 u32 apply_bytes; 46 u32 cork_bytes; 47 u32 flags; 48 struct sk_buff *skb; 49 struct sock *sk_redir; 50 struct sock *sk; 51 struct list_head list; 52 }; 53 54 struct sk_psock_progs { 55 struct bpf_prog *msg_parser; 56 struct bpf_prog *skb_parser; 57 struct bpf_prog *skb_verdict; 58 }; 59 60 enum sk_psock_state_bits { 61 SK_PSOCK_TX_ENABLED, 62 }; 63 64 struct sk_psock_link { 65 struct list_head list; 66 struct bpf_map *map; 67 void *link_raw; 68 }; 69 70 struct sk_psock_parser { 71 struct strparser strp; 72 bool enabled; 73 void (*saved_data_ready)(struct sock *sk); 74 }; 75 76 struct sk_psock_work_state { 77 struct sk_buff *skb; 78 u32 len; 79 u32 off; 80 }; 81 82 struct sk_psock { 83 struct sock *sk; 84 struct sock *sk_redir; 85 u32 apply_bytes; 86 u32 cork_bytes; 87 u32 eval; 88 struct sk_msg *cork; 89 struct sk_psock_progs progs; 90 struct sk_psock_parser parser; 91 struct sk_buff_head ingress_skb; 92 struct list_head ingress_msg; 93 unsigned long state; 94 struct list_head link; 95 spinlock_t link_lock; 96 refcount_t refcnt; 97 void (*saved_unhash)(struct sock *sk); 98 void (*saved_close)(struct sock *sk, long timeout); 99 void (*saved_write_space)(struct sock *sk); 100 struct proto *sk_proto; 101 struct sk_psock_work_state work_state; 102 struct work_struct work; 103 union { 104 struct rcu_head rcu; 105 struct work_struct gc; 106 }; 107 }; 108 109 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, 110 int elem_first_coalesce); 111 int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, 112 u32 off, u32 len); 113 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len); 114 int sk_msg_free(struct sock *sk, struct sk_msg *msg); 115 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg); 116 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes); 117 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg, 118 u32 bytes); 119 120 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes); 121 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes); 122 123 int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 124 struct sk_msg *msg, u32 bytes); 125 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, 126 struct sk_msg *msg, u32 bytes); 127 128 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) 129 { 130 WARN_ON(i == msg->sg.end && bytes); 131 } 132 133 static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes) 134 { 135 if (psock->apply_bytes) { 136 if (psock->apply_bytes < bytes) 137 psock->apply_bytes = 0; 138 else 139 psock->apply_bytes -= bytes; 140 } 141 } 142 143 #define sk_msg_iter_var_prev(var) \ 144 do { \ 145 if (var == 0) \ 146 var = MAX_MSG_FRAGS - 1; \ 147 else \ 148 var--; \ 149 } while (0) 150 151 #define sk_msg_iter_var_next(var) \ 152 do { \ 153 var++; \ 154 if (var == MAX_MSG_FRAGS) \ 155 var = 0; \ 156 } while (0) 157 158 #define sk_msg_iter_prev(msg, which) \ 159 sk_msg_iter_var_prev(msg->sg.which) 160 161 #define sk_msg_iter_next(msg, which) \ 162 sk_msg_iter_var_next(msg->sg.which) 163 164 static inline void sk_msg_clear_meta(struct sk_msg *msg) 165 { 166 memset(&msg->sg, 0, offsetofend(struct sk_msg_sg, copy)); 167 } 168 169 static inline void sk_msg_init(struct sk_msg *msg) 170 { 171 BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != MAX_MSG_FRAGS); 172 memset(msg, 0, sizeof(*msg)); 173 sg_init_marker(msg->sg.data, MAX_MSG_FRAGS); 174 } 175 176 static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src, 177 int which, u32 size) 178 { 179 dst->sg.data[which] = src->sg.data[which]; 180 dst->sg.data[which].length = size; 181 dst->sg.size += size; 182 src->sg.data[which].length -= size; 183 src->sg.data[which].offset += size; 184 } 185 186 static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src) 187 { 188 memcpy(dst, src, sizeof(*src)); 189 sk_msg_init(src); 190 } 191 192 static inline bool sk_msg_full(const struct sk_msg *msg) 193 { 194 return (msg->sg.end == msg->sg.start) && msg->sg.size; 195 } 196 197 static inline u32 sk_msg_elem_used(const struct sk_msg *msg) 198 { 199 if (sk_msg_full(msg)) 200 return MAX_MSG_FRAGS; 201 202 return msg->sg.end >= msg->sg.start ? 203 msg->sg.end - msg->sg.start : 204 msg->sg.end + (MAX_MSG_FRAGS - msg->sg.start); 205 } 206 207 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which) 208 { 209 return &msg->sg.data[which]; 210 } 211 212 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which) 213 { 214 return msg->sg.data[which]; 215 } 216 217 static inline struct page *sk_msg_page(struct sk_msg *msg, int which) 218 { 219 return sg_page(sk_msg_elem(msg, which)); 220 } 221 222 static inline bool sk_msg_to_ingress(const struct sk_msg *msg) 223 { 224 return msg->flags & BPF_F_INGRESS; 225 } 226 227 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) 228 { 229 struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start); 230 231 if (test_bit(msg->sg.start, &msg->sg.copy)) { 232 msg->data = NULL; 233 msg->data_end = NULL; 234 } else { 235 msg->data = sg_virt(sge); 236 msg->data_end = msg->data + sge->length; 237 } 238 } 239 240 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, 241 u32 len, u32 offset) 242 { 243 struct scatterlist *sge; 244 245 get_page(page); 246 sge = sk_msg_elem(msg, msg->sg.end); 247 sg_set_page(sge, page, len, offset); 248 sg_unmark_end(sge); 249 250 __set_bit(msg->sg.end, &msg->sg.copy); 251 msg->sg.size += len; 252 sk_msg_iter_next(msg, end); 253 } 254 255 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) 256 { 257 do { 258 if (copy_state) 259 __set_bit(i, &msg->sg.copy); 260 else 261 __clear_bit(i, &msg->sg.copy); 262 sk_msg_iter_var_next(i); 263 if (i == msg->sg.end) 264 break; 265 } while (1); 266 } 267 268 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start) 269 { 270 sk_msg_sg_copy(msg, start, true); 271 } 272 273 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start) 274 { 275 sk_msg_sg_copy(msg, start, false); 276 } 277 278 static inline struct sk_psock *sk_psock(const struct sock *sk) 279 { 280 return rcu_dereference_sk_user_data(sk); 281 } 282 283 static inline void sk_psock_queue_msg(struct sk_psock *psock, 284 struct sk_msg *msg) 285 { 286 list_add_tail(&msg->list, &psock->ingress_msg); 287 } 288 289 static inline bool sk_psock_queue_empty(const struct sk_psock *psock) 290 { 291 return psock ? list_empty(&psock->ingress_msg) : true; 292 } 293 294 static inline void sk_psock_report_error(struct sk_psock *psock, int err) 295 { 296 struct sock *sk = psock->sk; 297 298 sk->sk_err = err; 299 sk->sk_error_report(sk); 300 } 301 302 struct sk_psock *sk_psock_init(struct sock *sk, int node); 303 304 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock); 305 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock); 306 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock); 307 308 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, 309 struct sk_msg *msg); 310 311 static inline struct sk_psock_link *sk_psock_init_link(void) 312 { 313 return kzalloc(sizeof(struct sk_psock_link), 314 GFP_ATOMIC | __GFP_NOWARN); 315 } 316 317 static inline void sk_psock_free_link(struct sk_psock_link *link) 318 { 319 kfree(link); 320 } 321 322 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock); 323 #if defined(CONFIG_BPF_STREAM_PARSER) 324 void sk_psock_unlink(struct sock *sk, struct sk_psock_link *link); 325 #else 326 static inline void sk_psock_unlink(struct sock *sk, 327 struct sk_psock_link *link) 328 { 329 } 330 #endif 331 332 void __sk_psock_purge_ingress_msg(struct sk_psock *psock); 333 334 static inline void sk_psock_cork_free(struct sk_psock *psock) 335 { 336 if (psock->cork) { 337 sk_msg_free(psock->sk, psock->cork); 338 kfree(psock->cork); 339 psock->cork = NULL; 340 } 341 } 342 343 static inline void sk_psock_update_proto(struct sock *sk, 344 struct sk_psock *psock, 345 struct proto *ops) 346 { 347 psock->saved_unhash = sk->sk_prot->unhash; 348 psock->saved_close = sk->sk_prot->close; 349 psock->saved_write_space = sk->sk_write_space; 350 351 psock->sk_proto = sk->sk_prot; 352 sk->sk_prot = ops; 353 } 354 355 static inline void sk_psock_restore_proto(struct sock *sk, 356 struct sk_psock *psock) 357 { 358 sk->sk_write_space = psock->saved_write_space; 359 360 if (psock->sk_proto) { 361 struct inet_connection_sock *icsk = inet_csk(sk); 362 bool has_ulp = !!icsk->icsk_ulp_data; 363 364 if (has_ulp) 365 tcp_update_ulp(sk, psock->sk_proto); 366 else 367 sk->sk_prot = psock->sk_proto; 368 psock->sk_proto = NULL; 369 } 370 } 371 372 static inline void sk_psock_set_state(struct sk_psock *psock, 373 enum sk_psock_state_bits bit) 374 { 375 set_bit(bit, &psock->state); 376 } 377 378 static inline void sk_psock_clear_state(struct sk_psock *psock, 379 enum sk_psock_state_bits bit) 380 { 381 clear_bit(bit, &psock->state); 382 } 383 384 static inline bool sk_psock_test_state(const struct sk_psock *psock, 385 enum sk_psock_state_bits bit) 386 { 387 return test_bit(bit, &psock->state); 388 } 389 390 static inline struct sk_psock *sk_psock_get_checked(struct sock *sk) 391 { 392 struct sk_psock *psock; 393 394 rcu_read_lock(); 395 psock = sk_psock(sk); 396 if (psock) { 397 if (sk->sk_prot->recvmsg != tcp_bpf_recvmsg) { 398 psock = ERR_PTR(-EBUSY); 399 goto out; 400 } 401 402 if (!refcount_inc_not_zero(&psock->refcnt)) 403 psock = ERR_PTR(-EBUSY); 404 } 405 out: 406 rcu_read_unlock(); 407 return psock; 408 } 409 410 static inline struct sk_psock *sk_psock_get(struct sock *sk) 411 { 412 struct sk_psock *psock; 413 414 rcu_read_lock(); 415 psock = sk_psock(sk); 416 if (psock && !refcount_inc_not_zero(&psock->refcnt)) 417 psock = NULL; 418 rcu_read_unlock(); 419 return psock; 420 } 421 422 void sk_psock_stop(struct sock *sk, struct sk_psock *psock); 423 void sk_psock_destroy(struct rcu_head *rcu); 424 void sk_psock_drop(struct sock *sk, struct sk_psock *psock); 425 426 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock) 427 { 428 if (refcount_dec_and_test(&psock->refcnt)) 429 sk_psock_drop(sk, psock); 430 } 431 432 static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock) 433 { 434 if (psock->parser.enabled) 435 psock->parser.saved_data_ready(sk); 436 else 437 sk->sk_data_ready(sk); 438 } 439 440 static inline void psock_set_prog(struct bpf_prog **pprog, 441 struct bpf_prog *prog) 442 { 443 prog = xchg(pprog, prog); 444 if (prog) 445 bpf_prog_put(prog); 446 } 447 448 static inline void psock_progs_drop(struct sk_psock_progs *progs) 449 { 450 psock_set_prog(&progs->msg_parser, NULL); 451 psock_set_prog(&progs->skb_parser, NULL); 452 psock_set_prog(&progs->skb_verdict, NULL); 453 } 454 455 #endif /* _LINUX_SKMSG_H */ 456