1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ 3 4 #ifndef _LINUX_SKMSG_H 5 #define _LINUX_SKMSG_H 6 7 #include <linux/bpf.h> 8 #include <linux/filter.h> 9 #include <linux/scatterlist.h> 10 #include <linux/skbuff.h> 11 12 #include <net/sock.h> 13 #include <net/tcp.h> 14 #include <net/strparser.h> 15 16 #define MAX_MSG_FRAGS MAX_SKB_FRAGS 17 18 enum __sk_action { 19 __SK_DROP = 0, 20 __SK_PASS, 21 __SK_REDIRECT, 22 __SK_NONE, 23 }; 24 25 struct sk_msg_sg { 26 u32 start; 27 u32 curr; 28 u32 end; 29 u32 size; 30 u32 copybreak; 31 bool copy[MAX_MSG_FRAGS]; 32 /* The extra element is used for chaining the front and sections when 33 * the list becomes partitioned (e.g. end < start). The crypto APIs 34 * require the chaining. 35 */ 36 struct scatterlist data[MAX_MSG_FRAGS + 1]; 37 }; 38 39 struct sk_msg { 40 struct sk_msg_sg sg; 41 void *data; 42 void *data_end; 43 u32 apply_bytes; 44 u32 cork_bytes; 45 u32 flags; 46 struct sk_buff *skb; 47 struct sock *sk_redir; 48 struct sock *sk; 49 struct list_head list; 50 }; 51 52 struct sk_psock_progs { 53 struct bpf_prog *msg_parser; 54 struct bpf_prog *skb_parser; 55 struct bpf_prog *skb_verdict; 56 }; 57 58 enum sk_psock_state_bits { 59 SK_PSOCK_TX_ENABLED, 60 }; 61 62 struct sk_psock_link { 63 struct list_head list; 64 struct bpf_map *map; 65 void *link_raw; 66 }; 67 68 struct sk_psock_parser { 69 struct strparser strp; 70 bool enabled; 71 void (*saved_data_ready)(struct sock *sk); 72 }; 73 74 struct sk_psock_work_state { 75 struct sk_buff *skb; 76 u32 len; 77 u32 off; 78 }; 79 80 struct sk_psock { 81 struct sock *sk; 82 struct sock *sk_redir; 83 u32 apply_bytes; 84 u32 cork_bytes; 85 u32 eval; 86 struct sk_msg *cork; 87 struct sk_psock_progs progs; 88 struct sk_psock_parser parser; 89 struct sk_buff_head ingress_skb; 90 struct list_head ingress_msg; 91 unsigned long state; 92 struct list_head link; 93 spinlock_t link_lock; 94 refcount_t refcnt; 95 void (*saved_unhash)(struct sock *sk); 96 void (*saved_close)(struct sock *sk, long timeout); 97 void (*saved_write_space)(struct sock *sk); 98 struct proto *sk_proto; 99 struct sk_psock_work_state work_state; 100 struct work_struct work; 101 union { 102 struct rcu_head rcu; 103 struct work_struct gc; 104 }; 105 }; 106 107 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, 108 int elem_first_coalesce); 109 int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, 110 u32 off, u32 len); 111 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len); 112 int sk_msg_free(struct sock *sk, struct sk_msg *msg); 113 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg); 114 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes); 115 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg, 116 u32 bytes); 117 118 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes); 119 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes); 120 121 int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 122 struct sk_msg *msg, u32 bytes); 123 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, 124 struct sk_msg *msg, u32 bytes); 125 126 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) 127 { 128 WARN_ON(i == msg->sg.end && bytes); 129 } 130 131 static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes) 132 { 133 if (psock->apply_bytes) { 134 if (psock->apply_bytes < bytes) 135 psock->apply_bytes = 0; 136 else 137 psock->apply_bytes -= bytes; 138 } 139 } 140 141 #define sk_msg_iter_var_prev(var) \ 142 do { \ 143 if (var == 0) \ 144 var = MAX_MSG_FRAGS - 1; \ 145 else \ 146 var--; \ 147 } while (0) 148 149 #define sk_msg_iter_var_next(var) \ 150 do { \ 151 var++; \ 152 if (var == MAX_MSG_FRAGS) \ 153 var = 0; \ 154 } while (0) 155 156 #define sk_msg_iter_prev(msg, which) \ 157 sk_msg_iter_var_prev(msg->sg.which) 158 159 #define sk_msg_iter_next(msg, which) \ 160 sk_msg_iter_var_next(msg->sg.which) 161 162 static inline void sk_msg_clear_meta(struct sk_msg *msg) 163 { 164 memset(&msg->sg, 0, offsetofend(struct sk_msg_sg, copy)); 165 } 166 167 static inline void sk_msg_init(struct sk_msg *msg) 168 { 169 BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != MAX_MSG_FRAGS); 170 memset(msg, 0, sizeof(*msg)); 171 sg_init_marker(msg->sg.data, MAX_MSG_FRAGS); 172 } 173 174 static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src, 175 int which, u32 size) 176 { 177 dst->sg.data[which] = src->sg.data[which]; 178 dst->sg.data[which].length = size; 179 dst->sg.size += size; 180 src->sg.data[which].length -= size; 181 src->sg.data[which].offset += size; 182 } 183 184 static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src) 185 { 186 memcpy(dst, src, sizeof(*src)); 187 sk_msg_init(src); 188 } 189 190 static inline bool sk_msg_full(const struct sk_msg *msg) 191 { 192 return (msg->sg.end == msg->sg.start) && msg->sg.size; 193 } 194 195 static inline u32 sk_msg_elem_used(const struct sk_msg *msg) 196 { 197 if (sk_msg_full(msg)) 198 return MAX_MSG_FRAGS; 199 200 return msg->sg.end >= msg->sg.start ? 201 msg->sg.end - msg->sg.start : 202 msg->sg.end + (MAX_MSG_FRAGS - msg->sg.start); 203 } 204 205 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which) 206 { 207 return &msg->sg.data[which]; 208 } 209 210 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which) 211 { 212 return msg->sg.data[which]; 213 } 214 215 static inline struct page *sk_msg_page(struct sk_msg *msg, int which) 216 { 217 return sg_page(sk_msg_elem(msg, which)); 218 } 219 220 static inline bool sk_msg_to_ingress(const struct sk_msg *msg) 221 { 222 return msg->flags & BPF_F_INGRESS; 223 } 224 225 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) 226 { 227 struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start); 228 229 if (msg->sg.copy[msg->sg.start]) { 230 msg->data = NULL; 231 msg->data_end = NULL; 232 } else { 233 msg->data = sg_virt(sge); 234 msg->data_end = msg->data + sge->length; 235 } 236 } 237 238 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, 239 u32 len, u32 offset) 240 { 241 struct scatterlist *sge; 242 243 get_page(page); 244 sge = sk_msg_elem(msg, msg->sg.end); 245 sg_set_page(sge, page, len, offset); 246 sg_unmark_end(sge); 247 248 msg->sg.copy[msg->sg.end] = true; 249 msg->sg.size += len; 250 sk_msg_iter_next(msg, end); 251 } 252 253 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) 254 { 255 do { 256 msg->sg.copy[i] = copy_state; 257 sk_msg_iter_var_next(i); 258 if (i == msg->sg.end) 259 break; 260 } while (1); 261 } 262 263 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start) 264 { 265 sk_msg_sg_copy(msg, start, true); 266 } 267 268 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start) 269 { 270 sk_msg_sg_copy(msg, start, false); 271 } 272 273 static inline struct sk_psock *sk_psock(const struct sock *sk) 274 { 275 return rcu_dereference_sk_user_data(sk); 276 } 277 278 static inline void sk_psock_queue_msg(struct sk_psock *psock, 279 struct sk_msg *msg) 280 { 281 list_add_tail(&msg->list, &psock->ingress_msg); 282 } 283 284 static inline bool sk_psock_queue_empty(const struct sk_psock *psock) 285 { 286 return psock ? list_empty(&psock->ingress_msg) : true; 287 } 288 289 static inline void sk_psock_report_error(struct sk_psock *psock, int err) 290 { 291 struct sock *sk = psock->sk; 292 293 sk->sk_err = err; 294 sk->sk_error_report(sk); 295 } 296 297 struct sk_psock *sk_psock_init(struct sock *sk, int node); 298 299 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock); 300 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock); 301 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock); 302 303 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, 304 struct sk_msg *msg); 305 306 static inline struct sk_psock_link *sk_psock_init_link(void) 307 { 308 return kzalloc(sizeof(struct sk_psock_link), 309 GFP_ATOMIC | __GFP_NOWARN); 310 } 311 312 static inline void sk_psock_free_link(struct sk_psock_link *link) 313 { 314 kfree(link); 315 } 316 317 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock); 318 #if defined(CONFIG_BPF_STREAM_PARSER) 319 void sk_psock_unlink(struct sock *sk, struct sk_psock_link *link); 320 #else 321 static inline void sk_psock_unlink(struct sock *sk, 322 struct sk_psock_link *link) 323 { 324 } 325 #endif 326 327 void __sk_psock_purge_ingress_msg(struct sk_psock *psock); 328 329 static inline void sk_psock_cork_free(struct sk_psock *psock) 330 { 331 if (psock->cork) { 332 sk_msg_free(psock->sk, psock->cork); 333 kfree(psock->cork); 334 psock->cork = NULL; 335 } 336 } 337 338 static inline void sk_psock_update_proto(struct sock *sk, 339 struct sk_psock *psock, 340 struct proto *ops) 341 { 342 psock->saved_unhash = sk->sk_prot->unhash; 343 psock->saved_close = sk->sk_prot->close; 344 psock->saved_write_space = sk->sk_write_space; 345 346 psock->sk_proto = sk->sk_prot; 347 sk->sk_prot = ops; 348 } 349 350 static inline void sk_psock_restore_proto(struct sock *sk, 351 struct sk_psock *psock) 352 { 353 if (psock->sk_proto) { 354 sk->sk_prot = psock->sk_proto; 355 psock->sk_proto = NULL; 356 } 357 } 358 359 static inline void sk_psock_set_state(struct sk_psock *psock, 360 enum sk_psock_state_bits bit) 361 { 362 set_bit(bit, &psock->state); 363 } 364 365 static inline void sk_psock_clear_state(struct sk_psock *psock, 366 enum sk_psock_state_bits bit) 367 { 368 clear_bit(bit, &psock->state); 369 } 370 371 static inline bool sk_psock_test_state(const struct sk_psock *psock, 372 enum sk_psock_state_bits bit) 373 { 374 return test_bit(bit, &psock->state); 375 } 376 377 static inline struct sk_psock *sk_psock_get_checked(struct sock *sk) 378 { 379 struct sk_psock *psock; 380 381 rcu_read_lock(); 382 psock = sk_psock(sk); 383 if (psock) { 384 if (sk->sk_prot->recvmsg != tcp_bpf_recvmsg) { 385 psock = ERR_PTR(-EBUSY); 386 goto out; 387 } 388 389 if (!refcount_inc_not_zero(&psock->refcnt)) 390 psock = ERR_PTR(-EBUSY); 391 } 392 out: 393 rcu_read_unlock(); 394 return psock; 395 } 396 397 static inline struct sk_psock *sk_psock_get(struct sock *sk) 398 { 399 struct sk_psock *psock; 400 401 rcu_read_lock(); 402 psock = sk_psock(sk); 403 if (psock && !refcount_inc_not_zero(&psock->refcnt)) 404 psock = NULL; 405 rcu_read_unlock(); 406 return psock; 407 } 408 409 void sk_psock_stop(struct sock *sk, struct sk_psock *psock); 410 void sk_psock_destroy(struct rcu_head *rcu); 411 void sk_psock_drop(struct sock *sk, struct sk_psock *psock); 412 413 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock) 414 { 415 if (refcount_dec_and_test(&psock->refcnt)) 416 sk_psock_drop(sk, psock); 417 } 418 419 static inline void psock_set_prog(struct bpf_prog **pprog, 420 struct bpf_prog *prog) 421 { 422 prog = xchg(pprog, prog); 423 if (prog) 424 bpf_prog_put(prog); 425 } 426 427 static inline void psock_progs_drop(struct sk_psock_progs *progs) 428 { 429 psock_set_prog(&progs->msg_parser, NULL); 430 psock_set_prog(&progs->skb_parser, NULL); 431 psock_set_prog(&progs->skb_verdict, NULL); 432 } 433 434 #endif /* _LINUX_SKMSG_H */ 435