1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Definitions for the 'struct sk_buff' memory handlers. 4 * 5 * Authors: 6 * Alan Cox, <[email protected]> 7 * Florian La Roche, <[email protected]> 8 */ 9 10 #ifndef _LINUX_SKBUFF_H 11 #define _LINUX_SKBUFF_H 12 13 #include <linux/kernel.h> 14 #include <linux/compiler.h> 15 #include <linux/time.h> 16 #include <linux/bug.h> 17 #include <linux/bvec.h> 18 #include <linux/cache.h> 19 #include <linux/rbtree.h> 20 #include <linux/socket.h> 21 #include <linux/refcount.h> 22 23 #include <linux/atomic.h> 24 #include <asm/types.h> 25 #include <linux/spinlock.h> 26 #include <linux/net.h> 27 #include <linux/textsearch.h> 28 #include <net/checksum.h> 29 #include <linux/rcupdate.h> 30 #include <linux/hrtimer.h> 31 #include <linux/dma-mapping.h> 32 #include <linux/netdev_features.h> 33 #include <linux/sched.h> 34 #include <linux/sched/clock.h> 35 #include <net/flow_dissector.h> 36 #include <linux/splice.h> 37 #include <linux/in6.h> 38 #include <linux/if_packet.h> 39 #include <linux/llist.h> 40 #include <net/flow.h> 41 #include <net/page_pool.h> 42 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 43 #include <linux/netfilter/nf_conntrack_common.h> 44 #endif 45 #include <net/net_debug.h> 46 #include <net/dropreason.h> 47 48 /** 49 * DOC: skb checksums 50 * 51 * The interface for checksum offload between the stack and networking drivers 52 * is as follows... 53 * 54 * IP checksum related features 55 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 56 * 57 * Drivers advertise checksum offload capabilities in the features of a device. 58 * From the stack's point of view these are capabilities offered by the driver. 59 * A driver typically only advertises features that it is capable of offloading 60 * to its device. 61 * 62 * .. flat-table:: Checksum related device features 63 * :widths: 1 10 64 * 65 * * - %NETIF_F_HW_CSUM 66 * - The driver (or its device) is able to compute one 67 * IP (one's complement) checksum for any combination 68 * of protocols or protocol layering. The checksum is 69 * computed and set in a packet per the CHECKSUM_PARTIAL 70 * interface (see below). 71 * 72 * * - %NETIF_F_IP_CSUM 73 * - Driver (device) is only able to checksum plain 74 * TCP or UDP packets over IPv4. These are specifically 75 * unencapsulated packets of the form IPv4|TCP or 76 * IPv4|UDP where the Protocol field in the IPv4 header 77 * is TCP or UDP. The IPv4 header may contain IP options. 78 * This feature cannot be set in features for a device 79 * with NETIF_F_HW_CSUM also set. This feature is being 80 * DEPRECATED (see below). 81 * 82 * * - %NETIF_F_IPV6_CSUM 83 * - Driver (device) is only able to checksum plain 84 * TCP or UDP packets over IPv6. These are specifically 85 * unencapsulated packets of the form IPv6|TCP or 86 * IPv6|UDP where the Next Header field in the IPv6 87 * header is either TCP or UDP. IPv6 extension headers 88 * are not supported with this feature. This feature 89 * cannot be set in features for a device with 90 * NETIF_F_HW_CSUM also set. This feature is being 91 * DEPRECATED (see below). 92 * 93 * * - %NETIF_F_RXCSUM 94 * - Driver (device) performs receive checksum offload. 95 * This flag is only used to disable the RX checksum 96 * feature for a device. The stack will accept receive 97 * checksum indication in packets received on a device 98 * regardless of whether NETIF_F_RXCSUM is set. 99 * 100 * Checksumming of received packets by device 101 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 102 * 103 * Indication of checksum verification is set in &sk_buff.ip_summed. 104 * Possible values are: 105 * 106 * - %CHECKSUM_NONE 107 * 108 * Device did not checksum this packet e.g. due to lack of capabilities. 109 * The packet contains full (though not verified) checksum in packet but 110 * not in skb->csum. Thus, skb->csum is undefined in this case. 111 * 112 * - %CHECKSUM_UNNECESSARY 113 * 114 * The hardware you're dealing with doesn't calculate the full checksum 115 * (as in %CHECKSUM_COMPLETE), but it does parse headers and verify checksums 116 * for specific protocols. For such packets it will set %CHECKSUM_UNNECESSARY 117 * if their checksums are okay. &sk_buff.csum is still undefined in this case 118 * though. A driver or device must never modify the checksum field in the 119 * packet even if checksum is verified. 120 * 121 * %CHECKSUM_UNNECESSARY is applicable to following protocols: 122 * 123 * - TCP: IPv6 and IPv4. 124 * - UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a 125 * zero UDP checksum for either IPv4 or IPv6, the networking stack 126 * may perform further validation in this case. 127 * - GRE: only if the checksum is present in the header. 128 * - SCTP: indicates the CRC in SCTP header has been validated. 129 * - FCOE: indicates the CRC in FC frame has been validated. 130 * 131 * &sk_buff.csum_level indicates the number of consecutive checksums found in 132 * the packet minus one that have been verified as %CHECKSUM_UNNECESSARY. 133 * For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet 134 * and a device is able to verify the checksums for UDP (possibly zero), 135 * GRE (checksum flag is set) and TCP, &sk_buff.csum_level would be set to 136 * two. If the device were only able to verify the UDP checksum and not 137 * GRE, either because it doesn't support GRE checksum or because GRE 138 * checksum is bad, skb->csum_level would be set to zero (TCP checksum is 139 * not considered in this case). 140 * 141 * - %CHECKSUM_COMPLETE 142 * 143 * This is the most generic way. The device supplied checksum of the _whole_ 144 * packet as seen by netif_rx() and fills in &sk_buff.csum. This means the 145 * hardware doesn't need to parse L3/L4 headers to implement this. 146 * 147 * Notes: 148 * 149 * - Even if device supports only some protocols, but is able to produce 150 * skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY. 151 * - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols. 152 * 153 * - %CHECKSUM_PARTIAL 154 * 155 * A checksum is set up to be offloaded to a device as described in the 156 * output description for CHECKSUM_PARTIAL. This may occur on a packet 157 * received directly from another Linux OS, e.g., a virtualized Linux kernel 158 * on the same host, or it may be set in the input path in GRO or remote 159 * checksum offload. For the purposes of checksum verification, the checksum 160 * referred to by skb->csum_start + skb->csum_offset and any preceding 161 * checksums in the packet are considered verified. Any checksums in the 162 * packet that are after the checksum being offloaded are not considered to 163 * be verified. 164 * 165 * Checksumming on transmit for non-GSO 166 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 167 * 168 * The stack requests checksum offload in the &sk_buff.ip_summed for a packet. 169 * Values are: 170 * 171 * - %CHECKSUM_PARTIAL 172 * 173 * The driver is required to checksum the packet as seen by hard_start_xmit() 174 * from &sk_buff.csum_start up to the end, and to record/write the checksum at 175 * offset &sk_buff.csum_start + &sk_buff.csum_offset. 176 * A driver may verify that the 177 * csum_start and csum_offset values are valid values given the length and 178 * offset of the packet, but it should not attempt to validate that the 179 * checksum refers to a legitimate transport layer checksum -- it is the 180 * purview of the stack to validate that csum_start and csum_offset are set 181 * correctly. 182 * 183 * When the stack requests checksum offload for a packet, the driver MUST 184 * ensure that the checksum is set correctly. A driver can either offload the 185 * checksum calculation to the device, or call skb_checksum_help (in the case 186 * that the device does not support offload for a particular checksum). 187 * 188 * %NETIF_F_IP_CSUM and %NETIF_F_IPV6_CSUM are being deprecated in favor of 189 * %NETIF_F_HW_CSUM. New devices should use %NETIF_F_HW_CSUM to indicate 190 * checksum offload capability. 191 * skb_csum_hwoffload_help() can be called to resolve %CHECKSUM_PARTIAL based 192 * on network device checksumming capabilities: if a packet does not match 193 * them, skb_checksum_help() or skb_crc32c_help() (depending on the value of 194 * &sk_buff.csum_not_inet, see :ref:`crc`) 195 * is called to resolve the checksum. 196 * 197 * - %CHECKSUM_NONE 198 * 199 * The skb was already checksummed by the protocol, or a checksum is not 200 * required. 201 * 202 * - %CHECKSUM_UNNECESSARY 203 * 204 * This has the same meaning as CHECKSUM_NONE for checksum offload on 205 * output. 206 * 207 * - %CHECKSUM_COMPLETE 208 * 209 * Not used in checksum output. If a driver observes a packet with this value 210 * set in skbuff, it should treat the packet as if %CHECKSUM_NONE were set. 211 * 212 * .. _crc: 213 * 214 * Non-IP checksum (CRC) offloads 215 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 216 * 217 * .. flat-table:: 218 * :widths: 1 10 219 * 220 * * - %NETIF_F_SCTP_CRC 221 * - This feature indicates that a device is capable of 222 * offloading the SCTP CRC in a packet. To perform this offload the stack 223 * will set csum_start and csum_offset accordingly, set ip_summed to 224 * %CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication 225 * in the skbuff that the %CHECKSUM_PARTIAL refers to CRC32c. 226 * A driver that supports both IP checksum offload and SCTP CRC32c offload 227 * must verify which offload is configured for a packet by testing the 228 * value of &sk_buff.csum_not_inet; skb_crc32c_csum_help() is provided to 229 * resolve %CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1. 230 * 231 * * - %NETIF_F_FCOE_CRC 232 * - This feature indicates that a device is capable of offloading the FCOE 233 * CRC in a packet. To perform this offload the stack will set ip_summed 234 * to %CHECKSUM_PARTIAL and set csum_start and csum_offset 235 * accordingly. Note that there is no indication in the skbuff that the 236 * %CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports 237 * both IP checksum offload and FCOE CRC offload must verify which offload 238 * is configured for a packet, presumably by inspecting packet headers. 239 * 240 * Checksumming on output with GSO 241 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 242 * 243 * In the case of a GSO packet (skb_is_gso() is true), checksum offload 244 * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the 245 * gso_type is %SKB_GSO_TCPV4 or %SKB_GSO_TCPV6, TCP checksum offload as 246 * part of the GSO operation is implied. If a checksum is being offloaded 247 * with GSO then ip_summed is %CHECKSUM_PARTIAL, and both csum_start and 248 * csum_offset are set to refer to the outermost checksum being offloaded 249 * (two offloaded checksums are possible with UDP encapsulation). 250 */ 251 252 /* Don't change this without changing skb_csum_unnecessary! */ 253 #define CHECKSUM_NONE 0 254 #define CHECKSUM_UNNECESSARY 1 255 #define CHECKSUM_COMPLETE 2 256 #define CHECKSUM_PARTIAL 3 257 258 /* Maximum value in skb->csum_level */ 259 #define SKB_MAX_CSUM_LEVEL 3 260 261 #define SKB_DATA_ALIGN(X) ALIGN(X, SMP_CACHE_BYTES) 262 #define SKB_WITH_OVERHEAD(X) \ 263 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 264 #define SKB_MAX_ORDER(X, ORDER) \ 265 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X)) 266 #define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0)) 267 #define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2)) 268 269 /* return minimum truesize of one skb containing X bytes of data */ 270 #define SKB_TRUESIZE(X) ((X) + \ 271 SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \ 272 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 273 274 struct ahash_request; 275 struct net_device; 276 struct scatterlist; 277 struct pipe_inode_info; 278 struct iov_iter; 279 struct napi_struct; 280 struct bpf_prog; 281 union bpf_attr; 282 struct skb_ext; 283 284 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 285 struct nf_bridge_info { 286 enum { 287 BRNF_PROTO_UNCHANGED, 288 BRNF_PROTO_8021Q, 289 BRNF_PROTO_PPPOE 290 } orig_proto:8; 291 u8 pkt_otherhost:1; 292 u8 in_prerouting:1; 293 u8 bridged_dnat:1; 294 __u16 frag_max_size; 295 struct net_device *physindev; 296 297 /* always valid & non-NULL from FORWARD on, for physdev match */ 298 struct net_device *physoutdev; 299 union { 300 /* prerouting: detect dnat in orig/reply direction */ 301 __be32 ipv4_daddr; 302 struct in6_addr ipv6_daddr; 303 304 /* after prerouting + nat detected: store original source 305 * mac since neigh resolution overwrites it, only used while 306 * skb is out in neigh layer. 307 */ 308 char neigh_header[8]; 309 }; 310 }; 311 #endif 312 313 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 314 /* Chain in tc_skb_ext will be used to share the tc chain with 315 * ovs recirc_id. It will be set to the current chain by tc 316 * and read by ovs to recirc_id. 317 */ 318 struct tc_skb_ext { 319 __u32 chain; 320 __u16 mru; 321 __u16 zone; 322 u8 post_ct:1; 323 u8 post_ct_snat:1; 324 u8 post_ct_dnat:1; 325 }; 326 #endif 327 328 struct sk_buff_head { 329 /* These two members must be first to match sk_buff. */ 330 struct_group_tagged(sk_buff_list, list, 331 struct sk_buff *next; 332 struct sk_buff *prev; 333 ); 334 335 __u32 qlen; 336 spinlock_t lock; 337 }; 338 339 struct sk_buff; 340 341 /* To allow 64K frame to be packed as single skb without frag_list we 342 * require 64K/PAGE_SIZE pages plus 1 additional page to allow for 343 * buffers which do not start on a page boundary. 344 * 345 * Since GRO uses frags we allocate at least 16 regardless of page 346 * size. 347 */ 348 #if (65536/PAGE_SIZE + 1) < 16 349 #define MAX_SKB_FRAGS 16UL 350 #else 351 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1) 352 #endif 353 extern int sysctl_max_skb_frags; 354 355 /* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to 356 * segment using its current segmentation instead. 357 */ 358 #define GSO_BY_FRAGS 0xFFFF 359 360 typedef struct bio_vec skb_frag_t; 361 362 /** 363 * skb_frag_size() - Returns the size of a skb fragment 364 * @frag: skb fragment 365 */ 366 static inline unsigned int skb_frag_size(const skb_frag_t *frag) 367 { 368 return frag->bv_len; 369 } 370 371 /** 372 * skb_frag_size_set() - Sets the size of a skb fragment 373 * @frag: skb fragment 374 * @size: size of fragment 375 */ 376 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size) 377 { 378 frag->bv_len = size; 379 } 380 381 /** 382 * skb_frag_size_add() - Increments the size of a skb fragment by @delta 383 * @frag: skb fragment 384 * @delta: value to add 385 */ 386 static inline void skb_frag_size_add(skb_frag_t *frag, int delta) 387 { 388 frag->bv_len += delta; 389 } 390 391 /** 392 * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta 393 * @frag: skb fragment 394 * @delta: value to subtract 395 */ 396 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta) 397 { 398 frag->bv_len -= delta; 399 } 400 401 /** 402 * skb_frag_must_loop - Test if %p is a high memory page 403 * @p: fragment's page 404 */ 405 static inline bool skb_frag_must_loop(struct page *p) 406 { 407 #if defined(CONFIG_HIGHMEM) 408 if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) || PageHighMem(p)) 409 return true; 410 #endif 411 return false; 412 } 413 414 /** 415 * skb_frag_foreach_page - loop over pages in a fragment 416 * 417 * @f: skb frag to operate on 418 * @f_off: offset from start of f->bv_page 419 * @f_len: length from f_off to loop over 420 * @p: (temp var) current page 421 * @p_off: (temp var) offset from start of current page, 422 * non-zero only on first page. 423 * @p_len: (temp var) length in current page, 424 * < PAGE_SIZE only on first and last page. 425 * @copied: (temp var) length so far, excluding current p_len. 426 * 427 * A fragment can hold a compound page, in which case per-page 428 * operations, notably kmap_atomic, must be called for each 429 * regular page. 430 */ 431 #define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied) \ 432 for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT), \ 433 p_off = (f_off) & (PAGE_SIZE - 1), \ 434 p_len = skb_frag_must_loop(p) ? \ 435 min_t(u32, f_len, PAGE_SIZE - p_off) : f_len, \ 436 copied = 0; \ 437 copied < f_len; \ 438 copied += p_len, p++, p_off = 0, \ 439 p_len = min_t(u32, f_len - copied, PAGE_SIZE)) \ 440 441 #define HAVE_HW_TIME_STAMP 442 443 /** 444 * struct skb_shared_hwtstamps - hardware time stamps 445 * @hwtstamp: hardware time stamp transformed into duration 446 * since arbitrary point in time 447 * @netdev_data: address/cookie of network device driver used as 448 * reference to actual hardware time stamp 449 * 450 * Software time stamps generated by ktime_get_real() are stored in 451 * skb->tstamp. 452 * 453 * hwtstamps can only be compared against other hwtstamps from 454 * the same device. 455 * 456 * This structure is attached to packets as part of the 457 * &skb_shared_info. Use skb_hwtstamps() to get a pointer. 458 */ 459 struct skb_shared_hwtstamps { 460 union { 461 ktime_t hwtstamp; 462 void *netdev_data; 463 }; 464 }; 465 466 /* Definitions for tx_flags in struct skb_shared_info */ 467 enum { 468 /* generate hardware time stamp */ 469 SKBTX_HW_TSTAMP = 1 << 0, 470 471 /* generate software time stamp when queueing packet to NIC */ 472 SKBTX_SW_TSTAMP = 1 << 1, 473 474 /* device driver is going to provide hardware time stamp */ 475 SKBTX_IN_PROGRESS = 1 << 2, 476 477 /* generate hardware time stamp based on cycles if supported */ 478 SKBTX_HW_TSTAMP_USE_CYCLES = 1 << 3, 479 480 /* generate wifi status information (where possible) */ 481 SKBTX_WIFI_STATUS = 1 << 4, 482 483 /* determine hardware time stamp based on time or cycles */ 484 SKBTX_HW_TSTAMP_NETDEV = 1 << 5, 485 486 /* generate software time stamp when entering packet scheduling */ 487 SKBTX_SCHED_TSTAMP = 1 << 6, 488 }; 489 490 #define SKBTX_ANY_SW_TSTAMP (SKBTX_SW_TSTAMP | \ 491 SKBTX_SCHED_TSTAMP) 492 #define SKBTX_ANY_TSTAMP (SKBTX_HW_TSTAMP | \ 493 SKBTX_HW_TSTAMP_USE_CYCLES | \ 494 SKBTX_ANY_SW_TSTAMP) 495 496 /* Definitions for flags in struct skb_shared_info */ 497 enum { 498 /* use zcopy routines */ 499 SKBFL_ZEROCOPY_ENABLE = BIT(0), 500 501 /* This indicates at least one fragment might be overwritten 502 * (as in vmsplice(), sendfile() ...) 503 * If we need to compute a TX checksum, we'll need to copy 504 * all frags to avoid possible bad checksum 505 */ 506 SKBFL_SHARED_FRAG = BIT(1), 507 508 /* segment contains only zerocopy data and should not be 509 * charged to the kernel memory. 510 */ 511 SKBFL_PURE_ZEROCOPY = BIT(2), 512 513 SKBFL_DONT_ORPHAN = BIT(3), 514 515 /* page references are managed by the ubuf_info, so it's safe to 516 * use frags only up until ubuf_info is released 517 */ 518 SKBFL_MANAGED_FRAG_REFS = BIT(4), 519 }; 520 521 #define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG) 522 #define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \ 523 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS) 524 525 /* 526 * The callback notifies userspace to release buffers when skb DMA is done in 527 * lower device, the skb last reference should be 0 when calling this. 528 * The zerocopy_success argument is true if zero copy transmit occurred, 529 * false on data copy or out of memory error caused by data copy attempt. 530 * The ctx field is used to track device context. 531 * The desc field is used to track userspace buffer index. 532 */ 533 struct ubuf_info { 534 void (*callback)(struct sk_buff *, struct ubuf_info *, 535 bool zerocopy_success); 536 union { 537 struct { 538 unsigned long desc; 539 void *ctx; 540 }; 541 struct { 542 u32 id; 543 u16 len; 544 u16 zerocopy:1; 545 u32 bytelen; 546 }; 547 }; 548 refcount_t refcnt; 549 u8 flags; 550 551 struct mmpin { 552 struct user_struct *user; 553 unsigned int num_pg; 554 } mmp; 555 }; 556 557 #define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg)) 558 559 int mm_account_pinned_pages(struct mmpin *mmp, size_t size); 560 void mm_unaccount_pinned_pages(struct mmpin *mmp); 561 562 /* This data is invariant across clones and lives at 563 * the end of the header data, ie. at skb->end. 564 */ 565 struct skb_shared_info { 566 __u8 flags; 567 __u8 meta_len; 568 __u8 nr_frags; 569 __u8 tx_flags; 570 unsigned short gso_size; 571 /* Warning: this field is not always filled in (UFO)! */ 572 unsigned short gso_segs; 573 struct sk_buff *frag_list; 574 struct skb_shared_hwtstamps hwtstamps; 575 unsigned int gso_type; 576 u32 tskey; 577 578 /* 579 * Warning : all fields before dataref are cleared in __alloc_skb() 580 */ 581 atomic_t dataref; 582 unsigned int xdp_frags_size; 583 584 /* Intermediate layers must ensure that destructor_arg 585 * remains valid until skb destructor */ 586 void * destructor_arg; 587 588 /* must be last field, see pskb_expand_head() */ 589 skb_frag_t frags[MAX_SKB_FRAGS]; 590 }; 591 592 /** 593 * DOC: dataref and headerless skbs 594 * 595 * Transport layers send out clones of payload skbs they hold for 596 * retransmissions. To allow lower layers of the stack to prepend their headers 597 * we split &skb_shared_info.dataref into two halves. 598 * The lower 16 bits count the overall number of references. 599 * The higher 16 bits indicate how many of the references are payload-only. 600 * skb_header_cloned() checks if skb is allowed to add / write the headers. 601 * 602 * The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr 603 * (via __skb_header_release()). Any clone created from marked skb will get 604 * &sk_buff.hdr_len populated with the available headroom. 605 * If there's the only clone in existence it's able to modify the headroom 606 * at will. The sequence of calls inside the transport layer is:: 607 * 608 * <alloc skb> 609 * skb_reserve() 610 * __skb_header_release() 611 * skb_clone() 612 * // send the clone down the stack 613 * 614 * This is not a very generic construct and it depends on the transport layers 615 * doing the right thing. In practice there's usually only one payload-only skb. 616 * Having multiple payload-only skbs with different lengths of hdr_len is not 617 * possible. The payload-only skbs should never leave their owner. 618 */ 619 #define SKB_DATAREF_SHIFT 16 620 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1) 621 622 623 enum { 624 SKB_FCLONE_UNAVAILABLE, /* skb has no fclone (from head_cache) */ 625 SKB_FCLONE_ORIG, /* orig skb (from fclone_cache) */ 626 SKB_FCLONE_CLONE, /* companion fclone skb (from fclone_cache) */ 627 }; 628 629 enum { 630 SKB_GSO_TCPV4 = 1 << 0, 631 632 /* This indicates the skb is from an untrusted source. */ 633 SKB_GSO_DODGY = 1 << 1, 634 635 /* This indicates the tcp segment has CWR set. */ 636 SKB_GSO_TCP_ECN = 1 << 2, 637 638 SKB_GSO_TCP_FIXEDID = 1 << 3, 639 640 SKB_GSO_TCPV6 = 1 << 4, 641 642 SKB_GSO_FCOE = 1 << 5, 643 644 SKB_GSO_GRE = 1 << 6, 645 646 SKB_GSO_GRE_CSUM = 1 << 7, 647 648 SKB_GSO_IPXIP4 = 1 << 8, 649 650 SKB_GSO_IPXIP6 = 1 << 9, 651 652 SKB_GSO_UDP_TUNNEL = 1 << 10, 653 654 SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11, 655 656 SKB_GSO_PARTIAL = 1 << 12, 657 658 SKB_GSO_TUNNEL_REMCSUM = 1 << 13, 659 660 SKB_GSO_SCTP = 1 << 14, 661 662 SKB_GSO_ESP = 1 << 15, 663 664 SKB_GSO_UDP = 1 << 16, 665 666 SKB_GSO_UDP_L4 = 1 << 17, 667 668 SKB_GSO_FRAGLIST = 1 << 18, 669 }; 670 671 #if BITS_PER_LONG > 32 672 #define NET_SKBUFF_DATA_USES_OFFSET 1 673 #endif 674 675 #ifdef NET_SKBUFF_DATA_USES_OFFSET 676 typedef unsigned int sk_buff_data_t; 677 #else 678 typedef unsigned char *sk_buff_data_t; 679 #endif 680 681 /** 682 * DOC: Basic sk_buff geometry 683 * 684 * struct sk_buff itself is a metadata structure and does not hold any packet 685 * data. All the data is held in associated buffers. 686 * 687 * &sk_buff.head points to the main "head" buffer. The head buffer is divided 688 * into two parts: 689 * 690 * - data buffer, containing headers and sometimes payload; 691 * this is the part of the skb operated on by the common helpers 692 * such as skb_put() or skb_pull(); 693 * - shared info (struct skb_shared_info) which holds an array of pointers 694 * to read-only data in the (page, offset, length) format. 695 * 696 * Optionally &skb_shared_info.frag_list may point to another skb. 697 * 698 * Basic diagram may look like this:: 699 * 700 * --------------- 701 * | sk_buff | 702 * --------------- 703 * ,--------------------------- + head 704 * / ,----------------- + data 705 * / / ,----------- + tail 706 * | | | , + end 707 * | | | | 708 * v v v v 709 * ----------------------------------------------- 710 * | headroom | data | tailroom | skb_shared_info | 711 * ----------------------------------------------- 712 * + [page frag] 713 * + [page frag] 714 * + [page frag] 715 * + [page frag] --------- 716 * + frag_list --> | sk_buff | 717 * --------- 718 * 719 */ 720 721 /** 722 * struct sk_buff - socket buffer 723 * @next: Next buffer in list 724 * @prev: Previous buffer in list 725 * @tstamp: Time we arrived/left 726 * @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point 727 * for retransmit timer 728 * @rbnode: RB tree node, alternative to next/prev for netem/tcp 729 * @list: queue head 730 * @ll_node: anchor in an llist (eg socket defer_list) 731 * @sk: Socket we are owned by 732 * @ip_defrag_offset: (aka @sk) alternate use of @sk, used in 733 * fragmentation management 734 * @dev: Device we arrived on/are leaving by 735 * @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL 736 * @cb: Control buffer. Free for use by every layer. Put private vars here 737 * @_skb_refdst: destination entry (with norefcount bit) 738 * @sp: the security path, used for xfrm 739 * @len: Length of actual data 740 * @data_len: Data length 741 * @mac_len: Length of link layer header 742 * @hdr_len: writable header length of cloned skb 743 * @csum: Checksum (must include start/offset pair) 744 * @csum_start: Offset from skb->head where checksumming should start 745 * @csum_offset: Offset from csum_start where checksum should be stored 746 * @priority: Packet queueing priority 747 * @ignore_df: allow local fragmentation 748 * @cloned: Head may be cloned (check refcnt to be sure) 749 * @ip_summed: Driver fed us an IP checksum 750 * @nohdr: Payload reference only, must not modify header 751 * @pkt_type: Packet class 752 * @fclone: skbuff clone status 753 * @ipvs_property: skbuff is owned by ipvs 754 * @inner_protocol_type: whether the inner protocol is 755 * ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO 756 * @remcsum_offload: remote checksum offload is enabled 757 * @offload_fwd_mark: Packet was L2-forwarded in hardware 758 * @offload_l3_fwd_mark: Packet was L3-forwarded in hardware 759 * @tc_skip_classify: do not classify packet. set by IFB device 760 * @tc_at_ingress: used within tc_classify to distinguish in/egress 761 * @redirected: packet was redirected by packet classifier 762 * @from_ingress: packet was redirected from the ingress path 763 * @nf_skip_egress: packet shall skip nf egress - see netfilter_netdev.h 764 * @peeked: this packet has been seen already, so stats have been 765 * done for it, don't do them again 766 * @nf_trace: netfilter packet trace flag 767 * @protocol: Packet protocol from driver 768 * @destructor: Destruct function 769 * @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue) 770 * @_sk_redir: socket redirection information for skmsg 771 * @_nfct: Associated connection, if any (with nfctinfo bits) 772 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c 773 * @skb_iif: ifindex of device we arrived on 774 * @tc_index: Traffic control index 775 * @hash: the packet hash 776 * @queue_mapping: Queue mapping for multiqueue devices 777 * @head_frag: skb was allocated from page fragments, 778 * not allocated by kmalloc() or vmalloc(). 779 * @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves 780 * @pp_recycle: mark the packet for recycling instead of freeing (implies 781 * page_pool support on driver) 782 * @active_extensions: active extensions (skb_ext_id types) 783 * @ndisc_nodetype: router type (from link layer) 784 * @ooo_okay: allow the mapping of a socket to a queue to be changed 785 * @l4_hash: indicate hash is a canonical 4-tuple hash over transport 786 * ports. 787 * @sw_hash: indicates hash was computed in software stack 788 * @wifi_acked_valid: wifi_acked was set 789 * @wifi_acked: whether frame was acked on wifi or not 790 * @no_fcs: Request NIC to treat last 4 bytes as Ethernet FCS 791 * @encapsulation: indicates the inner headers in the skbuff are valid 792 * @encap_hdr_csum: software checksum is needed 793 * @csum_valid: checksum is already valid 794 * @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL 795 * @csum_complete_sw: checksum was completed by software 796 * @csum_level: indicates the number of consecutive checksums found in 797 * the packet minus one that have been verified as 798 * CHECKSUM_UNNECESSARY (max 3) 799 * @dst_pending_confirm: need to confirm neighbour 800 * @decrypted: Decrypted SKB 801 * @slow_gro: state present at GRO time, slower prepare step required 802 * @mono_delivery_time: When set, skb->tstamp has the 803 * delivery_time in mono clock base (i.e. EDT). Otherwise, the 804 * skb->tstamp has the (rcv) timestamp at ingress and 805 * delivery_time at egress. 806 * @napi_id: id of the NAPI struct this skb came from 807 * @sender_cpu: (aka @napi_id) source CPU in XPS 808 * @alloc_cpu: CPU which did the skb allocation. 809 * @secmark: security marking 810 * @mark: Generic packet mark 811 * @reserved_tailroom: (aka @mark) number of bytes of free space available 812 * at the tail of an sk_buff 813 * @vlan_present: VLAN tag is present 814 * @vlan_proto: vlan encapsulation protocol 815 * @vlan_tci: vlan tag control information 816 * @inner_protocol: Protocol (encapsulation) 817 * @inner_ipproto: (aka @inner_protocol) stores ipproto when 818 * skb->inner_protocol_type == ENCAP_TYPE_IPPROTO; 819 * @inner_transport_header: Inner transport layer header (encapsulation) 820 * @inner_network_header: Network layer header (encapsulation) 821 * @inner_mac_header: Link layer header (encapsulation) 822 * @transport_header: Transport layer header 823 * @network_header: Network layer header 824 * @mac_header: Link layer header 825 * @kcov_handle: KCOV remote handle for remote coverage collection 826 * @tail: Tail pointer 827 * @end: End pointer 828 * @head: Head of buffer 829 * @data: Data head pointer 830 * @truesize: Buffer size 831 * @users: User count - see {datagram,tcp}.c 832 * @extensions: allocated extensions, valid if active_extensions is nonzero 833 */ 834 835 struct sk_buff { 836 union { 837 struct { 838 /* These two members must be first to match sk_buff_head. */ 839 struct sk_buff *next; 840 struct sk_buff *prev; 841 842 union { 843 struct net_device *dev; 844 /* Some protocols might use this space to store information, 845 * while device pointer would be NULL. 846 * UDP receive path is one user. 847 */ 848 unsigned long dev_scratch; 849 }; 850 }; 851 struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */ 852 struct list_head list; 853 struct llist_node ll_node; 854 }; 855 856 union { 857 struct sock *sk; 858 int ip_defrag_offset; 859 }; 860 861 union { 862 ktime_t tstamp; 863 u64 skb_mstamp_ns; /* earliest departure time */ 864 }; 865 /* 866 * This is the control buffer. It is free to use for every 867 * layer. Please put your private variables there. If you 868 * want to keep them across layers you have to do a skb_clone() 869 * first. This is owned by whoever has the skb queued ATM. 870 */ 871 char cb[48] __aligned(8); 872 873 union { 874 struct { 875 unsigned long _skb_refdst; 876 void (*destructor)(struct sk_buff *skb); 877 }; 878 struct list_head tcp_tsorted_anchor; 879 #ifdef CONFIG_NET_SOCK_MSG 880 unsigned long _sk_redir; 881 #endif 882 }; 883 884 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 885 unsigned long _nfct; 886 #endif 887 unsigned int len, 888 data_len; 889 __u16 mac_len, 890 hdr_len; 891 892 /* Following fields are _not_ copied in __copy_skb_header() 893 * Note that queue_mapping is here mostly to fill a hole. 894 */ 895 __u16 queue_mapping; 896 897 /* if you move cloned around you also must adapt those constants */ 898 #ifdef __BIG_ENDIAN_BITFIELD 899 #define CLONED_MASK (1 << 7) 900 #else 901 #define CLONED_MASK 1 902 #endif 903 #define CLONED_OFFSET offsetof(struct sk_buff, __cloned_offset) 904 905 /* private: */ 906 __u8 __cloned_offset[0]; 907 /* public: */ 908 __u8 cloned:1, 909 nohdr:1, 910 fclone:2, 911 peeked:1, 912 head_frag:1, 913 pfmemalloc:1, 914 pp_recycle:1; /* page_pool recycle indicator */ 915 #ifdef CONFIG_SKB_EXTENSIONS 916 __u8 active_extensions; 917 #endif 918 919 /* Fields enclosed in headers group are copied 920 * using a single memcpy() in __copy_skb_header() 921 */ 922 struct_group(headers, 923 924 /* private: */ 925 __u8 __pkt_type_offset[0]; 926 /* public: */ 927 __u8 pkt_type:3; /* see PKT_TYPE_MAX */ 928 __u8 ignore_df:1; 929 __u8 nf_trace:1; 930 __u8 ip_summed:2; 931 __u8 ooo_okay:1; 932 933 __u8 l4_hash:1; 934 __u8 sw_hash:1; 935 __u8 wifi_acked_valid:1; 936 __u8 wifi_acked:1; 937 __u8 no_fcs:1; 938 /* Indicates the inner headers are valid in the skbuff. */ 939 __u8 encapsulation:1; 940 __u8 encap_hdr_csum:1; 941 __u8 csum_valid:1; 942 943 /* private: */ 944 __u8 __pkt_vlan_present_offset[0]; 945 /* public: */ 946 __u8 vlan_present:1; /* See PKT_VLAN_PRESENT_BIT */ 947 __u8 csum_complete_sw:1; 948 __u8 csum_level:2; 949 __u8 dst_pending_confirm:1; 950 __u8 mono_delivery_time:1; /* See SKB_MONO_DELIVERY_TIME_MASK */ 951 #ifdef CONFIG_NET_CLS_ACT 952 __u8 tc_skip_classify:1; 953 __u8 tc_at_ingress:1; /* See TC_AT_INGRESS_MASK */ 954 #endif 955 #ifdef CONFIG_IPV6_NDISC_NODETYPE 956 __u8 ndisc_nodetype:2; 957 #endif 958 959 __u8 ipvs_property:1; 960 __u8 inner_protocol_type:1; 961 __u8 remcsum_offload:1; 962 #ifdef CONFIG_NET_SWITCHDEV 963 __u8 offload_fwd_mark:1; 964 __u8 offload_l3_fwd_mark:1; 965 #endif 966 __u8 redirected:1; 967 #ifdef CONFIG_NET_REDIRECT 968 __u8 from_ingress:1; 969 #endif 970 #ifdef CONFIG_NETFILTER_SKIP_EGRESS 971 __u8 nf_skip_egress:1; 972 #endif 973 #ifdef CONFIG_TLS_DEVICE 974 __u8 decrypted:1; 975 #endif 976 __u8 slow_gro:1; 977 __u8 csum_not_inet:1; 978 979 #ifdef CONFIG_NET_SCHED 980 __u16 tc_index; /* traffic control index */ 981 #endif 982 983 union { 984 __wsum csum; 985 struct { 986 __u16 csum_start; 987 __u16 csum_offset; 988 }; 989 }; 990 __u32 priority; 991 int skb_iif; 992 __u32 hash; 993 __be16 vlan_proto; 994 __u16 vlan_tci; 995 #if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS) 996 union { 997 unsigned int napi_id; 998 unsigned int sender_cpu; 999 }; 1000 #endif 1001 u16 alloc_cpu; 1002 #ifdef CONFIG_NETWORK_SECMARK 1003 __u32 secmark; 1004 #endif 1005 1006 union { 1007 __u32 mark; 1008 __u32 reserved_tailroom; 1009 }; 1010 1011 union { 1012 __be16 inner_protocol; 1013 __u8 inner_ipproto; 1014 }; 1015 1016 __u16 inner_transport_header; 1017 __u16 inner_network_header; 1018 __u16 inner_mac_header; 1019 1020 __be16 protocol; 1021 __u16 transport_header; 1022 __u16 network_header; 1023 __u16 mac_header; 1024 1025 #ifdef CONFIG_KCOV 1026 u64 kcov_handle; 1027 #endif 1028 1029 ); /* end headers group */ 1030 1031 /* These elements must be at the end, see alloc_skb() for details. */ 1032 sk_buff_data_t tail; 1033 sk_buff_data_t end; 1034 unsigned char *head, 1035 *data; 1036 unsigned int truesize; 1037 refcount_t users; 1038 1039 #ifdef CONFIG_SKB_EXTENSIONS 1040 /* only useable after checking ->active_extensions != 0 */ 1041 struct skb_ext *extensions; 1042 #endif 1043 }; 1044 1045 /* if you move pkt_type around you also must adapt those constants */ 1046 #ifdef __BIG_ENDIAN_BITFIELD 1047 #define PKT_TYPE_MAX (7 << 5) 1048 #else 1049 #define PKT_TYPE_MAX 7 1050 #endif 1051 #define PKT_TYPE_OFFSET offsetof(struct sk_buff, __pkt_type_offset) 1052 1053 /* if you move pkt_vlan_present, tc_at_ingress, or mono_delivery_time 1054 * around, you also must adapt these constants. 1055 */ 1056 #ifdef __BIG_ENDIAN_BITFIELD 1057 #define PKT_VLAN_PRESENT_BIT 7 1058 #define TC_AT_INGRESS_MASK (1 << 0) 1059 #define SKB_MONO_DELIVERY_TIME_MASK (1 << 2) 1060 #else 1061 #define PKT_VLAN_PRESENT_BIT 0 1062 #define TC_AT_INGRESS_MASK (1 << 7) 1063 #define SKB_MONO_DELIVERY_TIME_MASK (1 << 5) 1064 #endif 1065 #define PKT_VLAN_PRESENT_OFFSET offsetof(struct sk_buff, __pkt_vlan_present_offset) 1066 1067 #ifdef __KERNEL__ 1068 /* 1069 * Handling routines are only of interest to the kernel 1070 */ 1071 1072 #define SKB_ALLOC_FCLONE 0x01 1073 #define SKB_ALLOC_RX 0x02 1074 #define SKB_ALLOC_NAPI 0x04 1075 1076 /** 1077 * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves 1078 * @skb: buffer 1079 */ 1080 static inline bool skb_pfmemalloc(const struct sk_buff *skb) 1081 { 1082 return unlikely(skb->pfmemalloc); 1083 } 1084 1085 /* 1086 * skb might have a dst pointer attached, refcounted or not. 1087 * _skb_refdst low order bit is set if refcount was _not_ taken 1088 */ 1089 #define SKB_DST_NOREF 1UL 1090 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF) 1091 1092 /** 1093 * skb_dst - returns skb dst_entry 1094 * @skb: buffer 1095 * 1096 * Returns skb dst_entry, regardless of reference taken or not. 1097 */ 1098 static inline struct dst_entry *skb_dst(const struct sk_buff *skb) 1099 { 1100 /* If refdst was not refcounted, check we still are in a 1101 * rcu_read_lock section 1102 */ 1103 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) && 1104 !rcu_read_lock_held() && 1105 !rcu_read_lock_bh_held()); 1106 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK); 1107 } 1108 1109 /** 1110 * skb_dst_set - sets skb dst 1111 * @skb: buffer 1112 * @dst: dst entry 1113 * 1114 * Sets skb dst, assuming a reference was taken on dst and should 1115 * be released by skb_dst_drop() 1116 */ 1117 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst) 1118 { 1119 skb->slow_gro |= !!dst; 1120 skb->_skb_refdst = (unsigned long)dst; 1121 } 1122 1123 /** 1124 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference 1125 * @skb: buffer 1126 * @dst: dst entry 1127 * 1128 * Sets skb dst, assuming a reference was not taken on dst. 1129 * If dst entry is cached, we do not take reference and dst_release 1130 * will be avoided by refdst_drop. If dst entry is not cached, we take 1131 * reference, so that last dst_release can destroy the dst immediately. 1132 */ 1133 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst) 1134 { 1135 WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); 1136 skb->slow_gro |= !!dst; 1137 skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; 1138 } 1139 1140 /** 1141 * skb_dst_is_noref - Test if skb dst isn't refcounted 1142 * @skb: buffer 1143 */ 1144 static inline bool skb_dst_is_noref(const struct sk_buff *skb) 1145 { 1146 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb); 1147 } 1148 1149 /** 1150 * skb_rtable - Returns the skb &rtable 1151 * @skb: buffer 1152 */ 1153 static inline struct rtable *skb_rtable(const struct sk_buff *skb) 1154 { 1155 return (struct rtable *)skb_dst(skb); 1156 } 1157 1158 /* For mangling skb->pkt_type from user space side from applications 1159 * such as nft, tc, etc, we only allow a conservative subset of 1160 * possible pkt_types to be set. 1161 */ 1162 static inline bool skb_pkt_type_ok(u32 ptype) 1163 { 1164 return ptype <= PACKET_OTHERHOST; 1165 } 1166 1167 /** 1168 * skb_napi_id - Returns the skb's NAPI id 1169 * @skb: buffer 1170 */ 1171 static inline unsigned int skb_napi_id(const struct sk_buff *skb) 1172 { 1173 #ifdef CONFIG_NET_RX_BUSY_POLL 1174 return skb->napi_id; 1175 #else 1176 return 0; 1177 #endif 1178 } 1179 1180 /** 1181 * skb_unref - decrement the skb's reference count 1182 * @skb: buffer 1183 * 1184 * Returns true if we can free the skb. 1185 */ 1186 static inline bool skb_unref(struct sk_buff *skb) 1187 { 1188 if (unlikely(!skb)) 1189 return false; 1190 if (likely(refcount_read(&skb->users) == 1)) 1191 smp_rmb(); 1192 else if (likely(!refcount_dec_and_test(&skb->users))) 1193 return false; 1194 1195 return true; 1196 } 1197 1198 void __fix_address 1199 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason); 1200 1201 /** 1202 * kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason 1203 * @skb: buffer to free 1204 */ 1205 static inline void kfree_skb(struct sk_buff *skb) 1206 { 1207 kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED); 1208 } 1209 1210 void skb_release_head_state(struct sk_buff *skb); 1211 void kfree_skb_list_reason(struct sk_buff *segs, 1212 enum skb_drop_reason reason); 1213 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt); 1214 void skb_tx_error(struct sk_buff *skb); 1215 1216 static inline void kfree_skb_list(struct sk_buff *segs) 1217 { 1218 kfree_skb_list_reason(segs, SKB_DROP_REASON_NOT_SPECIFIED); 1219 } 1220 1221 #ifdef CONFIG_TRACEPOINTS 1222 void consume_skb(struct sk_buff *skb); 1223 #else 1224 static inline void consume_skb(struct sk_buff *skb) 1225 { 1226 return kfree_skb(skb); 1227 } 1228 #endif 1229 1230 void __consume_stateless_skb(struct sk_buff *skb); 1231 void __kfree_skb(struct sk_buff *skb); 1232 extern struct kmem_cache *skbuff_head_cache; 1233 1234 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen); 1235 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 1236 bool *fragstolen, int *delta_truesize); 1237 1238 struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags, 1239 int node); 1240 struct sk_buff *__build_skb(void *data, unsigned int frag_size); 1241 struct sk_buff *build_skb(void *data, unsigned int frag_size); 1242 struct sk_buff *build_skb_around(struct sk_buff *skb, 1243 void *data, unsigned int frag_size); 1244 void skb_attempt_defer_free(struct sk_buff *skb); 1245 1246 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size); 1247 1248 /** 1249 * alloc_skb - allocate a network buffer 1250 * @size: size to allocate 1251 * @priority: allocation mask 1252 * 1253 * This function is a convenient wrapper around __alloc_skb(). 1254 */ 1255 static inline struct sk_buff *alloc_skb(unsigned int size, 1256 gfp_t priority) 1257 { 1258 return __alloc_skb(size, priority, 0, NUMA_NO_NODE); 1259 } 1260 1261 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 1262 unsigned long data_len, 1263 int max_page_order, 1264 int *errcode, 1265 gfp_t gfp_mask); 1266 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first); 1267 1268 /* Layout of fast clones : [skb1][skb2][fclone_ref] */ 1269 struct sk_buff_fclones { 1270 struct sk_buff skb1; 1271 1272 struct sk_buff skb2; 1273 1274 refcount_t fclone_ref; 1275 }; 1276 1277 /** 1278 * skb_fclone_busy - check if fclone is busy 1279 * @sk: socket 1280 * @skb: buffer 1281 * 1282 * Returns true if skb is a fast clone, and its clone is not freed. 1283 * Some drivers call skb_orphan() in their ndo_start_xmit(), 1284 * so we also check that this didnt happen. 1285 */ 1286 static inline bool skb_fclone_busy(const struct sock *sk, 1287 const struct sk_buff *skb) 1288 { 1289 const struct sk_buff_fclones *fclones; 1290 1291 fclones = container_of(skb, struct sk_buff_fclones, skb1); 1292 1293 return skb->fclone == SKB_FCLONE_ORIG && 1294 refcount_read(&fclones->fclone_ref) > 1 && 1295 READ_ONCE(fclones->skb2.sk) == sk; 1296 } 1297 1298 /** 1299 * alloc_skb_fclone - allocate a network buffer from fclone cache 1300 * @size: size to allocate 1301 * @priority: allocation mask 1302 * 1303 * This function is a convenient wrapper around __alloc_skb(). 1304 */ 1305 static inline struct sk_buff *alloc_skb_fclone(unsigned int size, 1306 gfp_t priority) 1307 { 1308 return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE); 1309 } 1310 1311 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src); 1312 void skb_headers_offset_update(struct sk_buff *skb, int off); 1313 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask); 1314 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority); 1315 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old); 1316 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority); 1317 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1318 gfp_t gfp_mask, bool fclone); 1319 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, 1320 gfp_t gfp_mask) 1321 { 1322 return __pskb_copy_fclone(skb, headroom, gfp_mask, false); 1323 } 1324 1325 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask); 1326 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, 1327 unsigned int headroom); 1328 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom); 1329 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom, 1330 int newtailroom, gfp_t priority); 1331 int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 1332 int offset, int len); 1333 int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, 1334 int offset, int len); 1335 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer); 1336 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error); 1337 1338 /** 1339 * skb_pad - zero pad the tail of an skb 1340 * @skb: buffer to pad 1341 * @pad: space to pad 1342 * 1343 * Ensure that a buffer is followed by a padding area that is zero 1344 * filled. Used by network drivers which may DMA or transfer data 1345 * beyond the buffer end onto the wire. 1346 * 1347 * May return error in out of memory cases. The skb is freed on error. 1348 */ 1349 static inline int skb_pad(struct sk_buff *skb, int pad) 1350 { 1351 return __skb_pad(skb, pad, true); 1352 } 1353 #define dev_kfree_skb(a) consume_skb(a) 1354 1355 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 1356 int offset, size_t size); 1357 1358 struct skb_seq_state { 1359 __u32 lower_offset; 1360 __u32 upper_offset; 1361 __u32 frag_idx; 1362 __u32 stepped_offset; 1363 struct sk_buff *root_skb; 1364 struct sk_buff *cur_skb; 1365 __u8 *frag_data; 1366 __u32 frag_off; 1367 }; 1368 1369 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 1370 unsigned int to, struct skb_seq_state *st); 1371 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 1372 struct skb_seq_state *st); 1373 void skb_abort_seq_read(struct skb_seq_state *st); 1374 1375 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 1376 unsigned int to, struct ts_config *config); 1377 1378 /* 1379 * Packet hash types specify the type of hash in skb_set_hash. 1380 * 1381 * Hash types refer to the protocol layer addresses which are used to 1382 * construct a packet's hash. The hashes are used to differentiate or identify 1383 * flows of the protocol layer for the hash type. Hash types are either 1384 * layer-2 (L2), layer-3 (L3), or layer-4 (L4). 1385 * 1386 * Properties of hashes: 1387 * 1388 * 1) Two packets in different flows have different hash values 1389 * 2) Two packets in the same flow should have the same hash value 1390 * 1391 * A hash at a higher layer is considered to be more specific. A driver should 1392 * set the most specific hash possible. 1393 * 1394 * A driver cannot indicate a more specific hash than the layer at which a hash 1395 * was computed. For instance an L3 hash cannot be set as an L4 hash. 1396 * 1397 * A driver may indicate a hash level which is less specific than the 1398 * actual layer the hash was computed on. For instance, a hash computed 1399 * at L4 may be considered an L3 hash. This should only be done if the 1400 * driver can't unambiguously determine that the HW computed the hash at 1401 * the higher layer. Note that the "should" in the second property above 1402 * permits this. 1403 */ 1404 enum pkt_hash_types { 1405 PKT_HASH_TYPE_NONE, /* Undefined type */ 1406 PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */ 1407 PKT_HASH_TYPE_L3, /* Input: src_IP, dst_IP */ 1408 PKT_HASH_TYPE_L4, /* Input: src_IP, dst_IP, src_port, dst_port */ 1409 }; 1410 1411 static inline void skb_clear_hash(struct sk_buff *skb) 1412 { 1413 skb->hash = 0; 1414 skb->sw_hash = 0; 1415 skb->l4_hash = 0; 1416 } 1417 1418 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb) 1419 { 1420 if (!skb->l4_hash) 1421 skb_clear_hash(skb); 1422 } 1423 1424 static inline void 1425 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4) 1426 { 1427 skb->l4_hash = is_l4; 1428 skb->sw_hash = is_sw; 1429 skb->hash = hash; 1430 } 1431 1432 static inline void 1433 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type) 1434 { 1435 /* Used by drivers to set hash from HW */ 1436 __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4); 1437 } 1438 1439 static inline void 1440 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4) 1441 { 1442 __skb_set_hash(skb, hash, true, is_l4); 1443 } 1444 1445 void __skb_get_hash(struct sk_buff *skb); 1446 u32 __skb_get_hash_symmetric(const struct sk_buff *skb); 1447 u32 skb_get_poff(const struct sk_buff *skb); 1448 u32 __skb_get_poff(const struct sk_buff *skb, const void *data, 1449 const struct flow_keys_basic *keys, int hlen); 1450 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 1451 const void *data, int hlen_proto); 1452 1453 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb, 1454 int thoff, u8 ip_proto) 1455 { 1456 return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0); 1457 } 1458 1459 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 1460 const struct flow_dissector_key *key, 1461 unsigned int key_count); 1462 1463 struct bpf_flow_dissector; 1464 u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, 1465 __be16 proto, int nhoff, int hlen, unsigned int flags); 1466 1467 bool __skb_flow_dissect(const struct net *net, 1468 const struct sk_buff *skb, 1469 struct flow_dissector *flow_dissector, 1470 void *target_container, const void *data, 1471 __be16 proto, int nhoff, int hlen, unsigned int flags); 1472 1473 static inline bool skb_flow_dissect(const struct sk_buff *skb, 1474 struct flow_dissector *flow_dissector, 1475 void *target_container, unsigned int flags) 1476 { 1477 return __skb_flow_dissect(NULL, skb, flow_dissector, 1478 target_container, NULL, 0, 0, 0, flags); 1479 } 1480 1481 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb, 1482 struct flow_keys *flow, 1483 unsigned int flags) 1484 { 1485 memset(flow, 0, sizeof(*flow)); 1486 return __skb_flow_dissect(NULL, skb, &flow_keys_dissector, 1487 flow, NULL, 0, 0, 0, flags); 1488 } 1489 1490 static inline bool 1491 skb_flow_dissect_flow_keys_basic(const struct net *net, 1492 const struct sk_buff *skb, 1493 struct flow_keys_basic *flow, 1494 const void *data, __be16 proto, 1495 int nhoff, int hlen, unsigned int flags) 1496 { 1497 memset(flow, 0, sizeof(*flow)); 1498 return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow, 1499 data, proto, nhoff, hlen, flags); 1500 } 1501 1502 void skb_flow_dissect_meta(const struct sk_buff *skb, 1503 struct flow_dissector *flow_dissector, 1504 void *target_container); 1505 1506 /* Gets a skb connection tracking info, ctinfo map should be a 1507 * map of mapsize to translate enum ip_conntrack_info states 1508 * to user states. 1509 */ 1510 void 1511 skb_flow_dissect_ct(const struct sk_buff *skb, 1512 struct flow_dissector *flow_dissector, 1513 void *target_container, 1514 u16 *ctinfo_map, size_t mapsize, 1515 bool post_ct, u16 zone); 1516 void 1517 skb_flow_dissect_tunnel_info(const struct sk_buff *skb, 1518 struct flow_dissector *flow_dissector, 1519 void *target_container); 1520 1521 void skb_flow_dissect_hash(const struct sk_buff *skb, 1522 struct flow_dissector *flow_dissector, 1523 void *target_container); 1524 1525 static inline __u32 skb_get_hash(struct sk_buff *skb) 1526 { 1527 if (!skb->l4_hash && !skb->sw_hash) 1528 __skb_get_hash(skb); 1529 1530 return skb->hash; 1531 } 1532 1533 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) 1534 { 1535 if (!skb->l4_hash && !skb->sw_hash) { 1536 struct flow_keys keys; 1537 __u32 hash = __get_hash_from_flowi6(fl6, &keys); 1538 1539 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 1540 } 1541 1542 return skb->hash; 1543 } 1544 1545 __u32 skb_get_hash_perturb(const struct sk_buff *skb, 1546 const siphash_key_t *perturb); 1547 1548 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb) 1549 { 1550 return skb->hash; 1551 } 1552 1553 static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from) 1554 { 1555 to->hash = from->hash; 1556 to->sw_hash = from->sw_hash; 1557 to->l4_hash = from->l4_hash; 1558 }; 1559 1560 static inline void skb_copy_decrypted(struct sk_buff *to, 1561 const struct sk_buff *from) 1562 { 1563 #ifdef CONFIG_TLS_DEVICE 1564 to->decrypted = from->decrypted; 1565 #endif 1566 } 1567 1568 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1569 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) 1570 { 1571 return skb->head + skb->end; 1572 } 1573 1574 static inline unsigned int skb_end_offset(const struct sk_buff *skb) 1575 { 1576 return skb->end; 1577 } 1578 1579 static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) 1580 { 1581 skb->end = offset; 1582 } 1583 #else 1584 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) 1585 { 1586 return skb->end; 1587 } 1588 1589 static inline unsigned int skb_end_offset(const struct sk_buff *skb) 1590 { 1591 return skb->end - skb->head; 1592 } 1593 1594 static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) 1595 { 1596 skb->end = skb->head + offset; 1597 } 1598 #endif 1599 1600 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1601 struct ubuf_info *uarg); 1602 1603 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); 1604 1605 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1606 bool success); 1607 1608 int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, 1609 struct sk_buff *skb, struct iov_iter *from, 1610 size_t length); 1611 1612 static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb, 1613 struct msghdr *msg, int len) 1614 { 1615 return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len); 1616 } 1617 1618 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1619 struct msghdr *msg, int len, 1620 struct ubuf_info *uarg); 1621 1622 /* Internal */ 1623 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB))) 1624 1625 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) 1626 { 1627 return &skb_shinfo(skb)->hwtstamps; 1628 } 1629 1630 static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb) 1631 { 1632 bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE; 1633 1634 return is_zcopy ? skb_uarg(skb) : NULL; 1635 } 1636 1637 static inline bool skb_zcopy_pure(const struct sk_buff *skb) 1638 { 1639 return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY; 1640 } 1641 1642 static inline bool skb_zcopy_managed(const struct sk_buff *skb) 1643 { 1644 return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS; 1645 } 1646 1647 static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1, 1648 const struct sk_buff *skb2) 1649 { 1650 return skb_zcopy_pure(skb1) == skb_zcopy_pure(skb2); 1651 } 1652 1653 static inline void net_zcopy_get(struct ubuf_info *uarg) 1654 { 1655 refcount_inc(&uarg->refcnt); 1656 } 1657 1658 static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg) 1659 { 1660 skb_shinfo(skb)->destructor_arg = uarg; 1661 skb_shinfo(skb)->flags |= uarg->flags; 1662 } 1663 1664 static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, 1665 bool *have_ref) 1666 { 1667 if (skb && uarg && !skb_zcopy(skb)) { 1668 if (unlikely(have_ref && *have_ref)) 1669 *have_ref = false; 1670 else 1671 net_zcopy_get(uarg); 1672 skb_zcopy_init(skb, uarg); 1673 } 1674 } 1675 1676 static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val) 1677 { 1678 skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); 1679 skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; 1680 } 1681 1682 static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb) 1683 { 1684 return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL; 1685 } 1686 1687 static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) 1688 { 1689 return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); 1690 } 1691 1692 static inline void net_zcopy_put(struct ubuf_info *uarg) 1693 { 1694 if (uarg) 1695 uarg->callback(NULL, uarg, true); 1696 } 1697 1698 static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1699 { 1700 if (uarg) { 1701 if (uarg->callback == msg_zerocopy_callback) 1702 msg_zerocopy_put_abort(uarg, have_uref); 1703 else if (have_uref) 1704 net_zcopy_put(uarg); 1705 } 1706 } 1707 1708 /* Release a reference on a zerocopy structure */ 1709 static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) 1710 { 1711 struct ubuf_info *uarg = skb_zcopy(skb); 1712 1713 if (uarg) { 1714 if (!skb_zcopy_is_nouarg(skb)) 1715 uarg->callback(skb, uarg, zerocopy_success); 1716 1717 skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY; 1718 } 1719 } 1720 1721 void __skb_zcopy_downgrade_managed(struct sk_buff *skb); 1722 1723 static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb) 1724 { 1725 if (unlikely(skb_zcopy_managed(skb))) 1726 __skb_zcopy_downgrade_managed(skb); 1727 } 1728 1729 static inline void skb_mark_not_on_list(struct sk_buff *skb) 1730 { 1731 skb->next = NULL; 1732 } 1733 1734 /* Iterate through singly-linked GSO fragments of an skb. */ 1735 #define skb_list_walk_safe(first, skb, next_skb) \ 1736 for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \ 1737 (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL) 1738 1739 static inline void skb_list_del_init(struct sk_buff *skb) 1740 { 1741 __list_del_entry(&skb->list); 1742 skb_mark_not_on_list(skb); 1743 } 1744 1745 /** 1746 * skb_queue_empty - check if a queue is empty 1747 * @list: queue head 1748 * 1749 * Returns true if the queue is empty, false otherwise. 1750 */ 1751 static inline int skb_queue_empty(const struct sk_buff_head *list) 1752 { 1753 return list->next == (const struct sk_buff *) list; 1754 } 1755 1756 /** 1757 * skb_queue_empty_lockless - check if a queue is empty 1758 * @list: queue head 1759 * 1760 * Returns true if the queue is empty, false otherwise. 1761 * This variant can be used in lockless contexts. 1762 */ 1763 static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list) 1764 { 1765 return READ_ONCE(list->next) == (const struct sk_buff *) list; 1766 } 1767 1768 1769 /** 1770 * skb_queue_is_last - check if skb is the last entry in the queue 1771 * @list: queue head 1772 * @skb: buffer 1773 * 1774 * Returns true if @skb is the last buffer on the list. 1775 */ 1776 static inline bool skb_queue_is_last(const struct sk_buff_head *list, 1777 const struct sk_buff *skb) 1778 { 1779 return skb->next == (const struct sk_buff *) list; 1780 } 1781 1782 /** 1783 * skb_queue_is_first - check if skb is the first entry in the queue 1784 * @list: queue head 1785 * @skb: buffer 1786 * 1787 * Returns true if @skb is the first buffer on the list. 1788 */ 1789 static inline bool skb_queue_is_first(const struct sk_buff_head *list, 1790 const struct sk_buff *skb) 1791 { 1792 return skb->prev == (const struct sk_buff *) list; 1793 } 1794 1795 /** 1796 * skb_queue_next - return the next packet in the queue 1797 * @list: queue head 1798 * @skb: current buffer 1799 * 1800 * Return the next packet in @list after @skb. It is only valid to 1801 * call this if skb_queue_is_last() evaluates to false. 1802 */ 1803 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list, 1804 const struct sk_buff *skb) 1805 { 1806 /* This BUG_ON may seem severe, but if we just return then we 1807 * are going to dereference garbage. 1808 */ 1809 BUG_ON(skb_queue_is_last(list, skb)); 1810 return skb->next; 1811 } 1812 1813 /** 1814 * skb_queue_prev - return the prev packet in the queue 1815 * @list: queue head 1816 * @skb: current buffer 1817 * 1818 * Return the prev packet in @list before @skb. It is only valid to 1819 * call this if skb_queue_is_first() evaluates to false. 1820 */ 1821 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list, 1822 const struct sk_buff *skb) 1823 { 1824 /* This BUG_ON may seem severe, but if we just return then we 1825 * are going to dereference garbage. 1826 */ 1827 BUG_ON(skb_queue_is_first(list, skb)); 1828 return skb->prev; 1829 } 1830 1831 /** 1832 * skb_get - reference buffer 1833 * @skb: buffer to reference 1834 * 1835 * Makes another reference to a socket buffer and returns a pointer 1836 * to the buffer. 1837 */ 1838 static inline struct sk_buff *skb_get(struct sk_buff *skb) 1839 { 1840 refcount_inc(&skb->users); 1841 return skb; 1842 } 1843 1844 /* 1845 * If users == 1, we are the only owner and can avoid redundant atomic changes. 1846 */ 1847 1848 /** 1849 * skb_cloned - is the buffer a clone 1850 * @skb: buffer to check 1851 * 1852 * Returns true if the buffer was generated with skb_clone() and is 1853 * one of multiple shared copies of the buffer. Cloned buffers are 1854 * shared data so must not be written to under normal circumstances. 1855 */ 1856 static inline int skb_cloned(const struct sk_buff *skb) 1857 { 1858 return skb->cloned && 1859 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1; 1860 } 1861 1862 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) 1863 { 1864 might_sleep_if(gfpflags_allow_blocking(pri)); 1865 1866 if (skb_cloned(skb)) 1867 return pskb_expand_head(skb, 0, 0, pri); 1868 1869 return 0; 1870 } 1871 1872 /* This variant of skb_unclone() makes sure skb->truesize 1873 * and skb_end_offset() are not changed, whenever a new skb->head is needed. 1874 * 1875 * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X)) 1876 * when various debugging features are in place. 1877 */ 1878 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri); 1879 static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 1880 { 1881 might_sleep_if(gfpflags_allow_blocking(pri)); 1882 1883 if (skb_cloned(skb)) 1884 return __skb_unclone_keeptruesize(skb, pri); 1885 return 0; 1886 } 1887 1888 /** 1889 * skb_header_cloned - is the header a clone 1890 * @skb: buffer to check 1891 * 1892 * Returns true if modifying the header part of the buffer requires 1893 * the data to be copied. 1894 */ 1895 static inline int skb_header_cloned(const struct sk_buff *skb) 1896 { 1897 int dataref; 1898 1899 if (!skb->cloned) 1900 return 0; 1901 1902 dataref = atomic_read(&skb_shinfo(skb)->dataref); 1903 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT); 1904 return dataref != 1; 1905 } 1906 1907 static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri) 1908 { 1909 might_sleep_if(gfpflags_allow_blocking(pri)); 1910 1911 if (skb_header_cloned(skb)) 1912 return pskb_expand_head(skb, 0, 0, pri); 1913 1914 return 0; 1915 } 1916 1917 /** 1918 * __skb_header_release() - allow clones to use the headroom 1919 * @skb: buffer to operate on 1920 * 1921 * See "DOC: dataref and headerless skbs". 1922 */ 1923 static inline void __skb_header_release(struct sk_buff *skb) 1924 { 1925 skb->nohdr = 1; 1926 atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT)); 1927 } 1928 1929 1930 /** 1931 * skb_shared - is the buffer shared 1932 * @skb: buffer to check 1933 * 1934 * Returns true if more than one person has a reference to this 1935 * buffer. 1936 */ 1937 static inline int skb_shared(const struct sk_buff *skb) 1938 { 1939 return refcount_read(&skb->users) != 1; 1940 } 1941 1942 /** 1943 * skb_share_check - check if buffer is shared and if so clone it 1944 * @skb: buffer to check 1945 * @pri: priority for memory allocation 1946 * 1947 * If the buffer is shared the buffer is cloned and the old copy 1948 * drops a reference. A new clone with a single reference is returned. 1949 * If the buffer is not shared the original buffer is returned. When 1950 * being called from interrupt status or with spinlocks held pri must 1951 * be GFP_ATOMIC. 1952 * 1953 * NULL is returned on a memory allocation failure. 1954 */ 1955 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri) 1956 { 1957 might_sleep_if(gfpflags_allow_blocking(pri)); 1958 if (skb_shared(skb)) { 1959 struct sk_buff *nskb = skb_clone(skb, pri); 1960 1961 if (likely(nskb)) 1962 consume_skb(skb); 1963 else 1964 kfree_skb(skb); 1965 skb = nskb; 1966 } 1967 return skb; 1968 } 1969 1970 /* 1971 * Copy shared buffers into a new sk_buff. We effectively do COW on 1972 * packets to handle cases where we have a local reader and forward 1973 * and a couple of other messy ones. The normal one is tcpdumping 1974 * a packet thats being forwarded. 1975 */ 1976 1977 /** 1978 * skb_unshare - make a copy of a shared buffer 1979 * @skb: buffer to check 1980 * @pri: priority for memory allocation 1981 * 1982 * If the socket buffer is a clone then this function creates a new 1983 * copy of the data, drops a reference count on the old copy and returns 1984 * the new copy with the reference count at 1. If the buffer is not a clone 1985 * the original buffer is returned. When called with a spinlock held or 1986 * from interrupt state @pri must be %GFP_ATOMIC 1987 * 1988 * %NULL is returned on a memory allocation failure. 1989 */ 1990 static inline struct sk_buff *skb_unshare(struct sk_buff *skb, 1991 gfp_t pri) 1992 { 1993 might_sleep_if(gfpflags_allow_blocking(pri)); 1994 if (skb_cloned(skb)) { 1995 struct sk_buff *nskb = skb_copy(skb, pri); 1996 1997 /* Free our shared copy */ 1998 if (likely(nskb)) 1999 consume_skb(skb); 2000 else 2001 kfree_skb(skb); 2002 skb = nskb; 2003 } 2004 return skb; 2005 } 2006 2007 /** 2008 * skb_peek - peek at the head of an &sk_buff_head 2009 * @list_: list to peek at 2010 * 2011 * Peek an &sk_buff. Unlike most other operations you _MUST_ 2012 * be careful with this one. A peek leaves the buffer on the 2013 * list and someone else may run off with it. You must hold 2014 * the appropriate locks or have a private queue to do this. 2015 * 2016 * Returns %NULL for an empty list or a pointer to the head element. 2017 * The reference count is not incremented and the reference is therefore 2018 * volatile. Use with caution. 2019 */ 2020 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_) 2021 { 2022 struct sk_buff *skb = list_->next; 2023 2024 if (skb == (struct sk_buff *)list_) 2025 skb = NULL; 2026 return skb; 2027 } 2028 2029 /** 2030 * __skb_peek - peek at the head of a non-empty &sk_buff_head 2031 * @list_: list to peek at 2032 * 2033 * Like skb_peek(), but the caller knows that the list is not empty. 2034 */ 2035 static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_) 2036 { 2037 return list_->next; 2038 } 2039 2040 /** 2041 * skb_peek_next - peek skb following the given one from a queue 2042 * @skb: skb to start from 2043 * @list_: list to peek at 2044 * 2045 * Returns %NULL when the end of the list is met or a pointer to the 2046 * next element. The reference count is not incremented and the 2047 * reference is therefore volatile. Use with caution. 2048 */ 2049 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb, 2050 const struct sk_buff_head *list_) 2051 { 2052 struct sk_buff *next = skb->next; 2053 2054 if (next == (struct sk_buff *)list_) 2055 next = NULL; 2056 return next; 2057 } 2058 2059 /** 2060 * skb_peek_tail - peek at the tail of an &sk_buff_head 2061 * @list_: list to peek at 2062 * 2063 * Peek an &sk_buff. Unlike most other operations you _MUST_ 2064 * be careful with this one. A peek leaves the buffer on the 2065 * list and someone else may run off with it. You must hold 2066 * the appropriate locks or have a private queue to do this. 2067 * 2068 * Returns %NULL for an empty list or a pointer to the tail element. 2069 * The reference count is not incremented and the reference is therefore 2070 * volatile. Use with caution. 2071 */ 2072 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_) 2073 { 2074 struct sk_buff *skb = READ_ONCE(list_->prev); 2075 2076 if (skb == (struct sk_buff *)list_) 2077 skb = NULL; 2078 return skb; 2079 2080 } 2081 2082 /** 2083 * skb_queue_len - get queue length 2084 * @list_: list to measure 2085 * 2086 * Return the length of an &sk_buff queue. 2087 */ 2088 static inline __u32 skb_queue_len(const struct sk_buff_head *list_) 2089 { 2090 return list_->qlen; 2091 } 2092 2093 /** 2094 * skb_queue_len_lockless - get queue length 2095 * @list_: list to measure 2096 * 2097 * Return the length of an &sk_buff queue. 2098 * This variant can be used in lockless contexts. 2099 */ 2100 static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_) 2101 { 2102 return READ_ONCE(list_->qlen); 2103 } 2104 2105 /** 2106 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head 2107 * @list: queue to initialize 2108 * 2109 * This initializes only the list and queue length aspects of 2110 * an sk_buff_head object. This allows to initialize the list 2111 * aspects of an sk_buff_head without reinitializing things like 2112 * the spinlock. It can also be used for on-stack sk_buff_head 2113 * objects where the spinlock is known to not be used. 2114 */ 2115 static inline void __skb_queue_head_init(struct sk_buff_head *list) 2116 { 2117 list->prev = list->next = (struct sk_buff *)list; 2118 list->qlen = 0; 2119 } 2120 2121 /* 2122 * This function creates a split out lock class for each invocation; 2123 * this is needed for now since a whole lot of users of the skb-queue 2124 * infrastructure in drivers have different locking usage (in hardirq) 2125 * than the networking core (in softirq only). In the long run either the 2126 * network layer or drivers should need annotation to consolidate the 2127 * main types of usage into 3 classes. 2128 */ 2129 static inline void skb_queue_head_init(struct sk_buff_head *list) 2130 { 2131 spin_lock_init(&list->lock); 2132 __skb_queue_head_init(list); 2133 } 2134 2135 static inline void skb_queue_head_init_class(struct sk_buff_head *list, 2136 struct lock_class_key *class) 2137 { 2138 skb_queue_head_init(list); 2139 lockdep_set_class(&list->lock, class); 2140 } 2141 2142 /* 2143 * Insert an sk_buff on a list. 2144 * 2145 * The "__skb_xxxx()" functions are the non-atomic ones that 2146 * can only be called with interrupts disabled. 2147 */ 2148 static inline void __skb_insert(struct sk_buff *newsk, 2149 struct sk_buff *prev, struct sk_buff *next, 2150 struct sk_buff_head *list) 2151 { 2152 /* See skb_queue_empty_lockless() and skb_peek_tail() 2153 * for the opposite READ_ONCE() 2154 */ 2155 WRITE_ONCE(newsk->next, next); 2156 WRITE_ONCE(newsk->prev, prev); 2157 WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk); 2158 WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk); 2159 WRITE_ONCE(list->qlen, list->qlen + 1); 2160 } 2161 2162 static inline void __skb_queue_splice(const struct sk_buff_head *list, 2163 struct sk_buff *prev, 2164 struct sk_buff *next) 2165 { 2166 struct sk_buff *first = list->next; 2167 struct sk_buff *last = list->prev; 2168 2169 WRITE_ONCE(first->prev, prev); 2170 WRITE_ONCE(prev->next, first); 2171 2172 WRITE_ONCE(last->next, next); 2173 WRITE_ONCE(next->prev, last); 2174 } 2175 2176 /** 2177 * skb_queue_splice - join two skb lists, this is designed for stacks 2178 * @list: the new list to add 2179 * @head: the place to add it in the first list 2180 */ 2181 static inline void skb_queue_splice(const struct sk_buff_head *list, 2182 struct sk_buff_head *head) 2183 { 2184 if (!skb_queue_empty(list)) { 2185 __skb_queue_splice(list, (struct sk_buff *) head, head->next); 2186 head->qlen += list->qlen; 2187 } 2188 } 2189 2190 /** 2191 * skb_queue_splice_init - join two skb lists and reinitialise the emptied list 2192 * @list: the new list to add 2193 * @head: the place to add it in the first list 2194 * 2195 * The list at @list is reinitialised 2196 */ 2197 static inline void skb_queue_splice_init(struct sk_buff_head *list, 2198 struct sk_buff_head *head) 2199 { 2200 if (!skb_queue_empty(list)) { 2201 __skb_queue_splice(list, (struct sk_buff *) head, head->next); 2202 head->qlen += list->qlen; 2203 __skb_queue_head_init(list); 2204 } 2205 } 2206 2207 /** 2208 * skb_queue_splice_tail - join two skb lists, each list being a queue 2209 * @list: the new list to add 2210 * @head: the place to add it in the first list 2211 */ 2212 static inline void skb_queue_splice_tail(const struct sk_buff_head *list, 2213 struct sk_buff_head *head) 2214 { 2215 if (!skb_queue_empty(list)) { 2216 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); 2217 head->qlen += list->qlen; 2218 } 2219 } 2220 2221 /** 2222 * skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list 2223 * @list: the new list to add 2224 * @head: the place to add it in the first list 2225 * 2226 * Each of the lists is a queue. 2227 * The list at @list is reinitialised 2228 */ 2229 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list, 2230 struct sk_buff_head *head) 2231 { 2232 if (!skb_queue_empty(list)) { 2233 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); 2234 head->qlen += list->qlen; 2235 __skb_queue_head_init(list); 2236 } 2237 } 2238 2239 /** 2240 * __skb_queue_after - queue a buffer at the list head 2241 * @list: list to use 2242 * @prev: place after this buffer 2243 * @newsk: buffer to queue 2244 * 2245 * Queue a buffer int the middle of a list. This function takes no locks 2246 * and you must therefore hold required locks before calling it. 2247 * 2248 * A buffer cannot be placed on two lists at the same time. 2249 */ 2250 static inline void __skb_queue_after(struct sk_buff_head *list, 2251 struct sk_buff *prev, 2252 struct sk_buff *newsk) 2253 { 2254 __skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list); 2255 } 2256 2257 void skb_append(struct sk_buff *old, struct sk_buff *newsk, 2258 struct sk_buff_head *list); 2259 2260 static inline void __skb_queue_before(struct sk_buff_head *list, 2261 struct sk_buff *next, 2262 struct sk_buff *newsk) 2263 { 2264 __skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list); 2265 } 2266 2267 /** 2268 * __skb_queue_head - queue a buffer at the list head 2269 * @list: list to use 2270 * @newsk: buffer to queue 2271 * 2272 * Queue a buffer at the start of a list. This function takes no locks 2273 * and you must therefore hold required locks before calling it. 2274 * 2275 * A buffer cannot be placed on two lists at the same time. 2276 */ 2277 static inline void __skb_queue_head(struct sk_buff_head *list, 2278 struct sk_buff *newsk) 2279 { 2280 __skb_queue_after(list, (struct sk_buff *)list, newsk); 2281 } 2282 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk); 2283 2284 /** 2285 * __skb_queue_tail - queue a buffer at the list tail 2286 * @list: list to use 2287 * @newsk: buffer to queue 2288 * 2289 * Queue a buffer at the end of a list. This function takes no locks 2290 * and you must therefore hold required locks before calling it. 2291 * 2292 * A buffer cannot be placed on two lists at the same time. 2293 */ 2294 static inline void __skb_queue_tail(struct sk_buff_head *list, 2295 struct sk_buff *newsk) 2296 { 2297 __skb_queue_before(list, (struct sk_buff *)list, newsk); 2298 } 2299 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk); 2300 2301 /* 2302 * remove sk_buff from list. _Must_ be called atomically, and with 2303 * the list known.. 2304 */ 2305 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list); 2306 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 2307 { 2308 struct sk_buff *next, *prev; 2309 2310 WRITE_ONCE(list->qlen, list->qlen - 1); 2311 next = skb->next; 2312 prev = skb->prev; 2313 skb->next = skb->prev = NULL; 2314 WRITE_ONCE(next->prev, prev); 2315 WRITE_ONCE(prev->next, next); 2316 } 2317 2318 /** 2319 * __skb_dequeue - remove from the head of the queue 2320 * @list: list to dequeue from 2321 * 2322 * Remove the head of the list. This function does not take any locks 2323 * so must be used with appropriate locks held only. The head item is 2324 * returned or %NULL if the list is empty. 2325 */ 2326 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list) 2327 { 2328 struct sk_buff *skb = skb_peek(list); 2329 if (skb) 2330 __skb_unlink(skb, list); 2331 return skb; 2332 } 2333 struct sk_buff *skb_dequeue(struct sk_buff_head *list); 2334 2335 /** 2336 * __skb_dequeue_tail - remove from the tail of the queue 2337 * @list: list to dequeue from 2338 * 2339 * Remove the tail of the list. This function does not take any locks 2340 * so must be used with appropriate locks held only. The tail item is 2341 * returned or %NULL if the list is empty. 2342 */ 2343 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list) 2344 { 2345 struct sk_buff *skb = skb_peek_tail(list); 2346 if (skb) 2347 __skb_unlink(skb, list); 2348 return skb; 2349 } 2350 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list); 2351 2352 2353 static inline bool skb_is_nonlinear(const struct sk_buff *skb) 2354 { 2355 return skb->data_len; 2356 } 2357 2358 static inline unsigned int skb_headlen(const struct sk_buff *skb) 2359 { 2360 return skb->len - skb->data_len; 2361 } 2362 2363 static inline unsigned int __skb_pagelen(const struct sk_buff *skb) 2364 { 2365 unsigned int i, len = 0; 2366 2367 for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--) 2368 len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2369 return len; 2370 } 2371 2372 static inline unsigned int skb_pagelen(const struct sk_buff *skb) 2373 { 2374 return skb_headlen(skb) + __skb_pagelen(skb); 2375 } 2376 2377 static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, 2378 int i, struct page *page, 2379 int off, int size) 2380 { 2381 skb_frag_t *frag = &shinfo->frags[i]; 2382 2383 /* 2384 * Propagate page pfmemalloc to the skb if we can. The problem is 2385 * that not all callers have unique ownership of the page but rely 2386 * on page_is_pfmemalloc doing the right thing(tm). 2387 */ 2388 frag->bv_page = page; 2389 frag->bv_offset = off; 2390 skb_frag_size_set(frag, size); 2391 } 2392 2393 /** 2394 * skb_len_add - adds a number to len fields of skb 2395 * @skb: buffer to add len to 2396 * @delta: number of bytes to add 2397 */ 2398 static inline void skb_len_add(struct sk_buff *skb, int delta) 2399 { 2400 skb->len += delta; 2401 skb->data_len += delta; 2402 skb->truesize += delta; 2403 } 2404 2405 /** 2406 * __skb_fill_page_desc - initialise a paged fragment in an skb 2407 * @skb: buffer containing fragment to be initialised 2408 * @i: paged fragment index to initialise 2409 * @page: the page to use for this fragment 2410 * @off: the offset to the data with @page 2411 * @size: the length of the data 2412 * 2413 * Initialises the @i'th fragment of @skb to point to &size bytes at 2414 * offset @off within @page. 2415 * 2416 * Does not take any additional reference on the fragment. 2417 */ 2418 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, 2419 struct page *page, int off, int size) 2420 { 2421 __skb_fill_page_desc_noacc(skb_shinfo(skb), i, page, off, size); 2422 page = compound_head(page); 2423 if (page_is_pfmemalloc(page)) 2424 skb->pfmemalloc = true; 2425 } 2426 2427 /** 2428 * skb_fill_page_desc - initialise a paged fragment in an skb 2429 * @skb: buffer containing fragment to be initialised 2430 * @i: paged fragment index to initialise 2431 * @page: the page to use for this fragment 2432 * @off: the offset to the data with @page 2433 * @size: the length of the data 2434 * 2435 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of 2436 * @skb to point to @size bytes at offset @off within @page. In 2437 * addition updates @skb such that @i is the last fragment. 2438 * 2439 * Does not take any additional reference on the fragment. 2440 */ 2441 static inline void skb_fill_page_desc(struct sk_buff *skb, int i, 2442 struct page *page, int off, int size) 2443 { 2444 __skb_fill_page_desc(skb, i, page, off, size); 2445 skb_shinfo(skb)->nr_frags = i + 1; 2446 } 2447 2448 /** 2449 * skb_fill_page_desc_noacc - initialise a paged fragment in an skb 2450 * @skb: buffer containing fragment to be initialised 2451 * @i: paged fragment index to initialise 2452 * @page: the page to use for this fragment 2453 * @off: the offset to the data with @page 2454 * @size: the length of the data 2455 * 2456 * Variant of skb_fill_page_desc() which does not deal with 2457 * pfmemalloc, if page is not owned by us. 2458 */ 2459 static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i, 2460 struct page *page, int off, 2461 int size) 2462 { 2463 struct skb_shared_info *shinfo = skb_shinfo(skb); 2464 2465 __skb_fill_page_desc_noacc(shinfo, i, page, off, size); 2466 shinfo->nr_frags = i + 1; 2467 } 2468 2469 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 2470 int size, unsigned int truesize); 2471 2472 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 2473 unsigned int truesize); 2474 2475 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb)) 2476 2477 #ifdef NET_SKBUFF_DATA_USES_OFFSET 2478 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) 2479 { 2480 return skb->head + skb->tail; 2481 } 2482 2483 static inline void skb_reset_tail_pointer(struct sk_buff *skb) 2484 { 2485 skb->tail = skb->data - skb->head; 2486 } 2487 2488 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) 2489 { 2490 skb_reset_tail_pointer(skb); 2491 skb->tail += offset; 2492 } 2493 2494 #else /* NET_SKBUFF_DATA_USES_OFFSET */ 2495 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) 2496 { 2497 return skb->tail; 2498 } 2499 2500 static inline void skb_reset_tail_pointer(struct sk_buff *skb) 2501 { 2502 skb->tail = skb->data; 2503 } 2504 2505 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) 2506 { 2507 skb->tail = skb->data + offset; 2508 } 2509 2510 #endif /* NET_SKBUFF_DATA_USES_OFFSET */ 2511 2512 static inline void skb_assert_len(struct sk_buff *skb) 2513 { 2514 #ifdef CONFIG_DEBUG_NET 2515 if (WARN_ONCE(!skb->len, "%s\n", __func__)) 2516 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); 2517 #endif /* CONFIG_DEBUG_NET */ 2518 } 2519 2520 /* 2521 * Add data to an sk_buff 2522 */ 2523 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len); 2524 void *skb_put(struct sk_buff *skb, unsigned int len); 2525 static inline void *__skb_put(struct sk_buff *skb, unsigned int len) 2526 { 2527 void *tmp = skb_tail_pointer(skb); 2528 SKB_LINEAR_ASSERT(skb); 2529 skb->tail += len; 2530 skb->len += len; 2531 return tmp; 2532 } 2533 2534 static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len) 2535 { 2536 void *tmp = __skb_put(skb, len); 2537 2538 memset(tmp, 0, len); 2539 return tmp; 2540 } 2541 2542 static inline void *__skb_put_data(struct sk_buff *skb, const void *data, 2543 unsigned int len) 2544 { 2545 void *tmp = __skb_put(skb, len); 2546 2547 memcpy(tmp, data, len); 2548 return tmp; 2549 } 2550 2551 static inline void __skb_put_u8(struct sk_buff *skb, u8 val) 2552 { 2553 *(u8 *)__skb_put(skb, 1) = val; 2554 } 2555 2556 static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len) 2557 { 2558 void *tmp = skb_put(skb, len); 2559 2560 memset(tmp, 0, len); 2561 2562 return tmp; 2563 } 2564 2565 static inline void *skb_put_data(struct sk_buff *skb, const void *data, 2566 unsigned int len) 2567 { 2568 void *tmp = skb_put(skb, len); 2569 2570 memcpy(tmp, data, len); 2571 2572 return tmp; 2573 } 2574 2575 static inline void skb_put_u8(struct sk_buff *skb, u8 val) 2576 { 2577 *(u8 *)skb_put(skb, 1) = val; 2578 } 2579 2580 void *skb_push(struct sk_buff *skb, unsigned int len); 2581 static inline void *__skb_push(struct sk_buff *skb, unsigned int len) 2582 { 2583 skb->data -= len; 2584 skb->len += len; 2585 return skb->data; 2586 } 2587 2588 void *skb_pull(struct sk_buff *skb, unsigned int len); 2589 static inline void *__skb_pull(struct sk_buff *skb, unsigned int len) 2590 { 2591 skb->len -= len; 2592 if (unlikely(skb->len < skb->data_len)) { 2593 #if defined(CONFIG_DEBUG_NET) 2594 skb->len += len; 2595 pr_err("__skb_pull(len=%u)\n", len); 2596 skb_dump(KERN_ERR, skb, false); 2597 #endif 2598 BUG(); 2599 } 2600 return skb->data += len; 2601 } 2602 2603 static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len) 2604 { 2605 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); 2606 } 2607 2608 void *skb_pull_data(struct sk_buff *skb, size_t len); 2609 2610 void *__pskb_pull_tail(struct sk_buff *skb, int delta); 2611 2612 static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len) 2613 { 2614 if (len > skb_headlen(skb) && 2615 !__pskb_pull_tail(skb, len - skb_headlen(skb))) 2616 return NULL; 2617 skb->len -= len; 2618 return skb->data += len; 2619 } 2620 2621 static inline void *pskb_pull(struct sk_buff *skb, unsigned int len) 2622 { 2623 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len); 2624 } 2625 2626 static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len) 2627 { 2628 if (likely(len <= skb_headlen(skb))) 2629 return true; 2630 if (unlikely(len > skb->len)) 2631 return false; 2632 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; 2633 } 2634 2635 void skb_condense(struct sk_buff *skb); 2636 2637 /** 2638 * skb_headroom - bytes at buffer head 2639 * @skb: buffer to check 2640 * 2641 * Return the number of bytes of free space at the head of an &sk_buff. 2642 */ 2643 static inline unsigned int skb_headroom(const struct sk_buff *skb) 2644 { 2645 return skb->data - skb->head; 2646 } 2647 2648 /** 2649 * skb_tailroom - bytes at buffer end 2650 * @skb: buffer to check 2651 * 2652 * Return the number of bytes of free space at the tail of an sk_buff 2653 */ 2654 static inline int skb_tailroom(const struct sk_buff *skb) 2655 { 2656 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail; 2657 } 2658 2659 /** 2660 * skb_availroom - bytes at buffer end 2661 * @skb: buffer to check 2662 * 2663 * Return the number of bytes of free space at the tail of an sk_buff 2664 * allocated by sk_stream_alloc() 2665 */ 2666 static inline int skb_availroom(const struct sk_buff *skb) 2667 { 2668 if (skb_is_nonlinear(skb)) 2669 return 0; 2670 2671 return skb->end - skb->tail - skb->reserved_tailroom; 2672 } 2673 2674 /** 2675 * skb_reserve - adjust headroom 2676 * @skb: buffer to alter 2677 * @len: bytes to move 2678 * 2679 * Increase the headroom of an empty &sk_buff by reducing the tail 2680 * room. This is only allowed for an empty buffer. 2681 */ 2682 static inline void skb_reserve(struct sk_buff *skb, int len) 2683 { 2684 skb->data += len; 2685 skb->tail += len; 2686 } 2687 2688 /** 2689 * skb_tailroom_reserve - adjust reserved_tailroom 2690 * @skb: buffer to alter 2691 * @mtu: maximum amount of headlen permitted 2692 * @needed_tailroom: minimum amount of reserved_tailroom 2693 * 2694 * Set reserved_tailroom so that headlen can be as large as possible but 2695 * not larger than mtu and tailroom cannot be smaller than 2696 * needed_tailroom. 2697 * The required headroom should already have been reserved before using 2698 * this function. 2699 */ 2700 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu, 2701 unsigned int needed_tailroom) 2702 { 2703 SKB_LINEAR_ASSERT(skb); 2704 if (mtu < skb_tailroom(skb) - needed_tailroom) 2705 /* use at most mtu */ 2706 skb->reserved_tailroom = skb_tailroom(skb) - mtu; 2707 else 2708 /* use up to all available space */ 2709 skb->reserved_tailroom = needed_tailroom; 2710 } 2711 2712 #define ENCAP_TYPE_ETHER 0 2713 #define ENCAP_TYPE_IPPROTO 1 2714 2715 static inline void skb_set_inner_protocol(struct sk_buff *skb, 2716 __be16 protocol) 2717 { 2718 skb->inner_protocol = protocol; 2719 skb->inner_protocol_type = ENCAP_TYPE_ETHER; 2720 } 2721 2722 static inline void skb_set_inner_ipproto(struct sk_buff *skb, 2723 __u8 ipproto) 2724 { 2725 skb->inner_ipproto = ipproto; 2726 skb->inner_protocol_type = ENCAP_TYPE_IPPROTO; 2727 } 2728 2729 static inline void skb_reset_inner_headers(struct sk_buff *skb) 2730 { 2731 skb->inner_mac_header = skb->mac_header; 2732 skb->inner_network_header = skb->network_header; 2733 skb->inner_transport_header = skb->transport_header; 2734 } 2735 2736 static inline void skb_reset_mac_len(struct sk_buff *skb) 2737 { 2738 skb->mac_len = skb->network_header - skb->mac_header; 2739 } 2740 2741 static inline unsigned char *skb_inner_transport_header(const struct sk_buff 2742 *skb) 2743 { 2744 return skb->head + skb->inner_transport_header; 2745 } 2746 2747 static inline int skb_inner_transport_offset(const struct sk_buff *skb) 2748 { 2749 return skb_inner_transport_header(skb) - skb->data; 2750 } 2751 2752 static inline void skb_reset_inner_transport_header(struct sk_buff *skb) 2753 { 2754 skb->inner_transport_header = skb->data - skb->head; 2755 } 2756 2757 static inline void skb_set_inner_transport_header(struct sk_buff *skb, 2758 const int offset) 2759 { 2760 skb_reset_inner_transport_header(skb); 2761 skb->inner_transport_header += offset; 2762 } 2763 2764 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb) 2765 { 2766 return skb->head + skb->inner_network_header; 2767 } 2768 2769 static inline void skb_reset_inner_network_header(struct sk_buff *skb) 2770 { 2771 skb->inner_network_header = skb->data - skb->head; 2772 } 2773 2774 static inline void skb_set_inner_network_header(struct sk_buff *skb, 2775 const int offset) 2776 { 2777 skb_reset_inner_network_header(skb); 2778 skb->inner_network_header += offset; 2779 } 2780 2781 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb) 2782 { 2783 return skb->head + skb->inner_mac_header; 2784 } 2785 2786 static inline void skb_reset_inner_mac_header(struct sk_buff *skb) 2787 { 2788 skb->inner_mac_header = skb->data - skb->head; 2789 } 2790 2791 static inline void skb_set_inner_mac_header(struct sk_buff *skb, 2792 const int offset) 2793 { 2794 skb_reset_inner_mac_header(skb); 2795 skb->inner_mac_header += offset; 2796 } 2797 static inline bool skb_transport_header_was_set(const struct sk_buff *skb) 2798 { 2799 return skb->transport_header != (typeof(skb->transport_header))~0U; 2800 } 2801 2802 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) 2803 { 2804 DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb)); 2805 return skb->head + skb->transport_header; 2806 } 2807 2808 static inline void skb_reset_transport_header(struct sk_buff *skb) 2809 { 2810 skb->transport_header = skb->data - skb->head; 2811 } 2812 2813 static inline void skb_set_transport_header(struct sk_buff *skb, 2814 const int offset) 2815 { 2816 skb_reset_transport_header(skb); 2817 skb->transport_header += offset; 2818 } 2819 2820 static inline unsigned char *skb_network_header(const struct sk_buff *skb) 2821 { 2822 return skb->head + skb->network_header; 2823 } 2824 2825 static inline void skb_reset_network_header(struct sk_buff *skb) 2826 { 2827 skb->network_header = skb->data - skb->head; 2828 } 2829 2830 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) 2831 { 2832 skb_reset_network_header(skb); 2833 skb->network_header += offset; 2834 } 2835 2836 static inline int skb_mac_header_was_set(const struct sk_buff *skb) 2837 { 2838 return skb->mac_header != (typeof(skb->mac_header))~0U; 2839 } 2840 2841 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) 2842 { 2843 DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); 2844 return skb->head + skb->mac_header; 2845 } 2846 2847 static inline int skb_mac_offset(const struct sk_buff *skb) 2848 { 2849 return skb_mac_header(skb) - skb->data; 2850 } 2851 2852 static inline u32 skb_mac_header_len(const struct sk_buff *skb) 2853 { 2854 DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); 2855 return skb->network_header - skb->mac_header; 2856 } 2857 2858 static inline void skb_unset_mac_header(struct sk_buff *skb) 2859 { 2860 skb->mac_header = (typeof(skb->mac_header))~0U; 2861 } 2862 2863 static inline void skb_reset_mac_header(struct sk_buff *skb) 2864 { 2865 skb->mac_header = skb->data - skb->head; 2866 } 2867 2868 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) 2869 { 2870 skb_reset_mac_header(skb); 2871 skb->mac_header += offset; 2872 } 2873 2874 static inline void skb_pop_mac_header(struct sk_buff *skb) 2875 { 2876 skb->mac_header = skb->network_header; 2877 } 2878 2879 static inline void skb_probe_transport_header(struct sk_buff *skb) 2880 { 2881 struct flow_keys_basic keys; 2882 2883 if (skb_transport_header_was_set(skb)) 2884 return; 2885 2886 if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, 2887 NULL, 0, 0, 0, 0)) 2888 skb_set_transport_header(skb, keys.control.thoff); 2889 } 2890 2891 static inline void skb_mac_header_rebuild(struct sk_buff *skb) 2892 { 2893 if (skb_mac_header_was_set(skb)) { 2894 const unsigned char *old_mac = skb_mac_header(skb); 2895 2896 skb_set_mac_header(skb, -skb->mac_len); 2897 memmove(skb_mac_header(skb), old_mac, skb->mac_len); 2898 } 2899 } 2900 2901 static inline int skb_checksum_start_offset(const struct sk_buff *skb) 2902 { 2903 return skb->csum_start - skb_headroom(skb); 2904 } 2905 2906 static inline unsigned char *skb_checksum_start(const struct sk_buff *skb) 2907 { 2908 return skb->head + skb->csum_start; 2909 } 2910 2911 static inline int skb_transport_offset(const struct sk_buff *skb) 2912 { 2913 return skb_transport_header(skb) - skb->data; 2914 } 2915 2916 static inline u32 skb_network_header_len(const struct sk_buff *skb) 2917 { 2918 return skb->transport_header - skb->network_header; 2919 } 2920 2921 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb) 2922 { 2923 return skb->inner_transport_header - skb->inner_network_header; 2924 } 2925 2926 static inline int skb_network_offset(const struct sk_buff *skb) 2927 { 2928 return skb_network_header(skb) - skb->data; 2929 } 2930 2931 static inline int skb_inner_network_offset(const struct sk_buff *skb) 2932 { 2933 return skb_inner_network_header(skb) - skb->data; 2934 } 2935 2936 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len) 2937 { 2938 return pskb_may_pull(skb, skb_network_offset(skb) + len); 2939 } 2940 2941 /* 2942 * CPUs often take a performance hit when accessing unaligned memory 2943 * locations. The actual performance hit varies, it can be small if the 2944 * hardware handles it or large if we have to take an exception and fix it 2945 * in software. 2946 * 2947 * Since an ethernet header is 14 bytes network drivers often end up with 2948 * the IP header at an unaligned offset. The IP header can be aligned by 2949 * shifting the start of the packet by 2 bytes. Drivers should do this 2950 * with: 2951 * 2952 * skb_reserve(skb, NET_IP_ALIGN); 2953 * 2954 * The downside to this alignment of the IP header is that the DMA is now 2955 * unaligned. On some architectures the cost of an unaligned DMA is high 2956 * and this cost outweighs the gains made by aligning the IP header. 2957 * 2958 * Since this trade off varies between architectures, we allow NET_IP_ALIGN 2959 * to be overridden. 2960 */ 2961 #ifndef NET_IP_ALIGN 2962 #define NET_IP_ALIGN 2 2963 #endif 2964 2965 /* 2966 * The networking layer reserves some headroom in skb data (via 2967 * dev_alloc_skb). This is used to avoid having to reallocate skb data when 2968 * the header has to grow. In the default case, if the header has to grow 2969 * 32 bytes or less we avoid the reallocation. 2970 * 2971 * Unfortunately this headroom changes the DMA alignment of the resulting 2972 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive 2973 * on some architectures. An architecture can override this value, 2974 * perhaps setting it to a cacheline in size (since that will maintain 2975 * cacheline alignment of the DMA). It must be a power of 2. 2976 * 2977 * Various parts of the networking layer expect at least 32 bytes of 2978 * headroom, you should not reduce this. 2979 * 2980 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS) 2981 * to reduce average number of cache lines per packet. 2982 * get_rps_cpu() for example only access one 64 bytes aligned block : 2983 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8) 2984 */ 2985 #ifndef NET_SKB_PAD 2986 #define NET_SKB_PAD max(32, L1_CACHE_BYTES) 2987 #endif 2988 2989 int ___pskb_trim(struct sk_buff *skb, unsigned int len); 2990 2991 static inline void __skb_set_length(struct sk_buff *skb, unsigned int len) 2992 { 2993 if (WARN_ON(skb_is_nonlinear(skb))) 2994 return; 2995 skb->len = len; 2996 skb_set_tail_pointer(skb, len); 2997 } 2998 2999 static inline void __skb_trim(struct sk_buff *skb, unsigned int len) 3000 { 3001 __skb_set_length(skb, len); 3002 } 3003 3004 void skb_trim(struct sk_buff *skb, unsigned int len); 3005 3006 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) 3007 { 3008 if (skb->data_len) 3009 return ___pskb_trim(skb, len); 3010 __skb_trim(skb, len); 3011 return 0; 3012 } 3013 3014 static inline int pskb_trim(struct sk_buff *skb, unsigned int len) 3015 { 3016 return (len < skb->len) ? __pskb_trim(skb, len) : 0; 3017 } 3018 3019 /** 3020 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer 3021 * @skb: buffer to alter 3022 * @len: new length 3023 * 3024 * This is identical to pskb_trim except that the caller knows that 3025 * the skb is not cloned so we should never get an error due to out- 3026 * of-memory. 3027 */ 3028 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len) 3029 { 3030 int err = pskb_trim(skb, len); 3031 BUG_ON(err); 3032 } 3033 3034 static inline int __skb_grow(struct sk_buff *skb, unsigned int len) 3035 { 3036 unsigned int diff = len - skb->len; 3037 3038 if (skb_tailroom(skb) < diff) { 3039 int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb), 3040 GFP_ATOMIC); 3041 if (ret) 3042 return ret; 3043 } 3044 __skb_set_length(skb, len); 3045 return 0; 3046 } 3047 3048 /** 3049 * skb_orphan - orphan a buffer 3050 * @skb: buffer to orphan 3051 * 3052 * If a buffer currently has an owner then we call the owner's 3053 * destructor function and make the @skb unowned. The buffer continues 3054 * to exist but is no longer charged to its former owner. 3055 */ 3056 static inline void skb_orphan(struct sk_buff *skb) 3057 { 3058 if (skb->destructor) { 3059 skb->destructor(skb); 3060 skb->destructor = NULL; 3061 skb->sk = NULL; 3062 } else { 3063 BUG_ON(skb->sk); 3064 } 3065 } 3066 3067 /** 3068 * skb_orphan_frags - orphan the frags contained in a buffer 3069 * @skb: buffer to orphan frags from 3070 * @gfp_mask: allocation mask for replacement pages 3071 * 3072 * For each frag in the SKB which needs a destructor (i.e. has an 3073 * owner) create a copy of that frag and release the original 3074 * page by calling the destructor. 3075 */ 3076 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) 3077 { 3078 if (likely(!skb_zcopy(skb))) 3079 return 0; 3080 if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN) 3081 return 0; 3082 return skb_copy_ubufs(skb, gfp_mask); 3083 } 3084 3085 /* Frags must be orphaned, even if refcounted, if skb might loop to rx path */ 3086 static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask) 3087 { 3088 if (likely(!skb_zcopy(skb))) 3089 return 0; 3090 return skb_copy_ubufs(skb, gfp_mask); 3091 } 3092 3093 /** 3094 * __skb_queue_purge - empty a list 3095 * @list: list to empty 3096 * 3097 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3098 * the list and one reference dropped. This function does not take the 3099 * list lock and the caller must hold the relevant locks to use it. 3100 */ 3101 static inline void __skb_queue_purge(struct sk_buff_head *list) 3102 { 3103 struct sk_buff *skb; 3104 while ((skb = __skb_dequeue(list)) != NULL) 3105 kfree_skb(skb); 3106 } 3107 void skb_queue_purge(struct sk_buff_head *list); 3108 3109 unsigned int skb_rbtree_purge(struct rb_root *root); 3110 3111 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask); 3112 3113 /** 3114 * netdev_alloc_frag - allocate a page fragment 3115 * @fragsz: fragment size 3116 * 3117 * Allocates a frag from a page for receive buffer. 3118 * Uses GFP_ATOMIC allocations. 3119 */ 3120 static inline void *netdev_alloc_frag(unsigned int fragsz) 3121 { 3122 return __netdev_alloc_frag_align(fragsz, ~0u); 3123 } 3124 3125 static inline void *netdev_alloc_frag_align(unsigned int fragsz, 3126 unsigned int align) 3127 { 3128 WARN_ON_ONCE(!is_power_of_2(align)); 3129 return __netdev_alloc_frag_align(fragsz, -align); 3130 } 3131 3132 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length, 3133 gfp_t gfp_mask); 3134 3135 /** 3136 * netdev_alloc_skb - allocate an skbuff for rx on a specific device 3137 * @dev: network device to receive on 3138 * @length: length to allocate 3139 * 3140 * Allocate a new &sk_buff and assign it a usage count of one. The 3141 * buffer has unspecified headroom built in. Users should allocate 3142 * the headroom they think they need without accounting for the 3143 * built in space. The built in space is used for optimisations. 3144 * 3145 * %NULL is returned if there is no free memory. Although this function 3146 * allocates memory it can be called from an interrupt. 3147 */ 3148 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev, 3149 unsigned int length) 3150 { 3151 return __netdev_alloc_skb(dev, length, GFP_ATOMIC); 3152 } 3153 3154 /* legacy helper around __netdev_alloc_skb() */ 3155 static inline struct sk_buff *__dev_alloc_skb(unsigned int length, 3156 gfp_t gfp_mask) 3157 { 3158 return __netdev_alloc_skb(NULL, length, gfp_mask); 3159 } 3160 3161 /* legacy helper around netdev_alloc_skb() */ 3162 static inline struct sk_buff *dev_alloc_skb(unsigned int length) 3163 { 3164 return netdev_alloc_skb(NULL, length); 3165 } 3166 3167 3168 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev, 3169 unsigned int length, gfp_t gfp) 3170 { 3171 struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp); 3172 3173 if (NET_IP_ALIGN && skb) 3174 skb_reserve(skb, NET_IP_ALIGN); 3175 return skb; 3176 } 3177 3178 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, 3179 unsigned int length) 3180 { 3181 return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC); 3182 } 3183 3184 static inline void skb_free_frag(void *addr) 3185 { 3186 page_frag_free(addr); 3187 } 3188 3189 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask); 3190 3191 static inline void *napi_alloc_frag(unsigned int fragsz) 3192 { 3193 return __napi_alloc_frag_align(fragsz, ~0u); 3194 } 3195 3196 static inline void *napi_alloc_frag_align(unsigned int fragsz, 3197 unsigned int align) 3198 { 3199 WARN_ON_ONCE(!is_power_of_2(align)); 3200 return __napi_alloc_frag_align(fragsz, -align); 3201 } 3202 3203 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, 3204 unsigned int length, gfp_t gfp_mask); 3205 static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi, 3206 unsigned int length) 3207 { 3208 return __napi_alloc_skb(napi, length, GFP_ATOMIC); 3209 } 3210 void napi_consume_skb(struct sk_buff *skb, int budget); 3211 3212 void napi_skb_free_stolen_head(struct sk_buff *skb); 3213 void __kfree_skb_defer(struct sk_buff *skb); 3214 3215 /** 3216 * __dev_alloc_pages - allocate page for network Rx 3217 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx 3218 * @order: size of the allocation 3219 * 3220 * Allocate a new page. 3221 * 3222 * %NULL is returned if there is no free memory. 3223 */ 3224 static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, 3225 unsigned int order) 3226 { 3227 /* This piece of code contains several assumptions. 3228 * 1. This is for device Rx, therefor a cold page is preferred. 3229 * 2. The expectation is the user wants a compound page. 3230 * 3. If requesting a order 0 page it will not be compound 3231 * due to the check to see if order has a value in prep_new_page 3232 * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to 3233 * code in gfp_to_alloc_flags that should be enforcing this. 3234 */ 3235 gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; 3236 3237 return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); 3238 } 3239 3240 static inline struct page *dev_alloc_pages(unsigned int order) 3241 { 3242 return __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, order); 3243 } 3244 3245 /** 3246 * __dev_alloc_page - allocate a page for network Rx 3247 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx 3248 * 3249 * Allocate a new page. 3250 * 3251 * %NULL is returned if there is no free memory. 3252 */ 3253 static inline struct page *__dev_alloc_page(gfp_t gfp_mask) 3254 { 3255 return __dev_alloc_pages(gfp_mask, 0); 3256 } 3257 3258 static inline struct page *dev_alloc_page(void) 3259 { 3260 return dev_alloc_pages(0); 3261 } 3262 3263 /** 3264 * dev_page_is_reusable - check whether a page can be reused for network Rx 3265 * @page: the page to test 3266 * 3267 * A page shouldn't be considered for reusing/recycling if it was allocated 3268 * under memory pressure or at a distant memory node. 3269 * 3270 * Returns false if this page should be returned to page allocator, true 3271 * otherwise. 3272 */ 3273 static inline bool dev_page_is_reusable(const struct page *page) 3274 { 3275 return likely(page_to_nid(page) == numa_mem_id() && 3276 !page_is_pfmemalloc(page)); 3277 } 3278 3279 /** 3280 * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page 3281 * @page: The page that was allocated from skb_alloc_page 3282 * @skb: The skb that may need pfmemalloc set 3283 */ 3284 static inline void skb_propagate_pfmemalloc(const struct page *page, 3285 struct sk_buff *skb) 3286 { 3287 if (page_is_pfmemalloc(page)) 3288 skb->pfmemalloc = true; 3289 } 3290 3291 /** 3292 * skb_frag_off() - Returns the offset of a skb fragment 3293 * @frag: the paged fragment 3294 */ 3295 static inline unsigned int skb_frag_off(const skb_frag_t *frag) 3296 { 3297 return frag->bv_offset; 3298 } 3299 3300 /** 3301 * skb_frag_off_add() - Increments the offset of a skb fragment by @delta 3302 * @frag: skb fragment 3303 * @delta: value to add 3304 */ 3305 static inline void skb_frag_off_add(skb_frag_t *frag, int delta) 3306 { 3307 frag->bv_offset += delta; 3308 } 3309 3310 /** 3311 * skb_frag_off_set() - Sets the offset of a skb fragment 3312 * @frag: skb fragment 3313 * @offset: offset of fragment 3314 */ 3315 static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset) 3316 { 3317 frag->bv_offset = offset; 3318 } 3319 3320 /** 3321 * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment 3322 * @fragto: skb fragment where offset is set 3323 * @fragfrom: skb fragment offset is copied from 3324 */ 3325 static inline void skb_frag_off_copy(skb_frag_t *fragto, 3326 const skb_frag_t *fragfrom) 3327 { 3328 fragto->bv_offset = fragfrom->bv_offset; 3329 } 3330 3331 /** 3332 * skb_frag_page - retrieve the page referred to by a paged fragment 3333 * @frag: the paged fragment 3334 * 3335 * Returns the &struct page associated with @frag. 3336 */ 3337 static inline struct page *skb_frag_page(const skb_frag_t *frag) 3338 { 3339 return frag->bv_page; 3340 } 3341 3342 /** 3343 * __skb_frag_ref - take an addition reference on a paged fragment. 3344 * @frag: the paged fragment 3345 * 3346 * Takes an additional reference on the paged fragment @frag. 3347 */ 3348 static inline void __skb_frag_ref(skb_frag_t *frag) 3349 { 3350 get_page(skb_frag_page(frag)); 3351 } 3352 3353 /** 3354 * skb_frag_ref - take an addition reference on a paged fragment of an skb. 3355 * @skb: the buffer 3356 * @f: the fragment offset. 3357 * 3358 * Takes an additional reference on the @f'th paged fragment of @skb. 3359 */ 3360 static inline void skb_frag_ref(struct sk_buff *skb, int f) 3361 { 3362 __skb_frag_ref(&skb_shinfo(skb)->frags[f]); 3363 } 3364 3365 /** 3366 * __skb_frag_unref - release a reference on a paged fragment. 3367 * @frag: the paged fragment 3368 * @recycle: recycle the page if allocated via page_pool 3369 * 3370 * Releases a reference on the paged fragment @frag 3371 * or recycles the page via the page_pool API. 3372 */ 3373 static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle) 3374 { 3375 struct page *page = skb_frag_page(frag); 3376 3377 #ifdef CONFIG_PAGE_POOL 3378 if (recycle && page_pool_return_skb_page(page)) 3379 return; 3380 #endif 3381 put_page(page); 3382 } 3383 3384 /** 3385 * skb_frag_unref - release a reference on a paged fragment of an skb. 3386 * @skb: the buffer 3387 * @f: the fragment offset 3388 * 3389 * Releases a reference on the @f'th paged fragment of @skb. 3390 */ 3391 static inline void skb_frag_unref(struct sk_buff *skb, int f) 3392 { 3393 struct skb_shared_info *shinfo = skb_shinfo(skb); 3394 3395 if (!skb_zcopy_managed(skb)) 3396 __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle); 3397 } 3398 3399 /** 3400 * skb_frag_address - gets the address of the data contained in a paged fragment 3401 * @frag: the paged fragment buffer 3402 * 3403 * Returns the address of the data within @frag. The page must already 3404 * be mapped. 3405 */ 3406 static inline void *skb_frag_address(const skb_frag_t *frag) 3407 { 3408 return page_address(skb_frag_page(frag)) + skb_frag_off(frag); 3409 } 3410 3411 /** 3412 * skb_frag_address_safe - gets the address of the data contained in a paged fragment 3413 * @frag: the paged fragment buffer 3414 * 3415 * Returns the address of the data within @frag. Checks that the page 3416 * is mapped and returns %NULL otherwise. 3417 */ 3418 static inline void *skb_frag_address_safe(const skb_frag_t *frag) 3419 { 3420 void *ptr = page_address(skb_frag_page(frag)); 3421 if (unlikely(!ptr)) 3422 return NULL; 3423 3424 return ptr + skb_frag_off(frag); 3425 } 3426 3427 /** 3428 * skb_frag_page_copy() - sets the page in a fragment from another fragment 3429 * @fragto: skb fragment where page is set 3430 * @fragfrom: skb fragment page is copied from 3431 */ 3432 static inline void skb_frag_page_copy(skb_frag_t *fragto, 3433 const skb_frag_t *fragfrom) 3434 { 3435 fragto->bv_page = fragfrom->bv_page; 3436 } 3437 3438 /** 3439 * __skb_frag_set_page - sets the page contained in a paged fragment 3440 * @frag: the paged fragment 3441 * @page: the page to set 3442 * 3443 * Sets the fragment @frag to contain @page. 3444 */ 3445 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page) 3446 { 3447 frag->bv_page = page; 3448 } 3449 3450 /** 3451 * skb_frag_set_page - sets the page contained in a paged fragment of an skb 3452 * @skb: the buffer 3453 * @f: the fragment offset 3454 * @page: the page to set 3455 * 3456 * Sets the @f'th fragment of @skb to contain @page. 3457 */ 3458 static inline void skb_frag_set_page(struct sk_buff *skb, int f, 3459 struct page *page) 3460 { 3461 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page); 3462 } 3463 3464 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); 3465 3466 /** 3467 * skb_frag_dma_map - maps a paged fragment via the DMA API 3468 * @dev: the device to map the fragment to 3469 * @frag: the paged fragment to map 3470 * @offset: the offset within the fragment (starting at the 3471 * fragment's own offset) 3472 * @size: the number of bytes to map 3473 * @dir: the direction of the mapping (``PCI_DMA_*``) 3474 * 3475 * Maps the page associated with @frag to @device. 3476 */ 3477 static inline dma_addr_t skb_frag_dma_map(struct device *dev, 3478 const skb_frag_t *frag, 3479 size_t offset, size_t size, 3480 enum dma_data_direction dir) 3481 { 3482 return dma_map_page(dev, skb_frag_page(frag), 3483 skb_frag_off(frag) + offset, size, dir); 3484 } 3485 3486 static inline struct sk_buff *pskb_copy(struct sk_buff *skb, 3487 gfp_t gfp_mask) 3488 { 3489 return __pskb_copy(skb, skb_headroom(skb), gfp_mask); 3490 } 3491 3492 3493 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb, 3494 gfp_t gfp_mask) 3495 { 3496 return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true); 3497 } 3498 3499 3500 /** 3501 * skb_clone_writable - is the header of a clone writable 3502 * @skb: buffer to check 3503 * @len: length up to which to write 3504 * 3505 * Returns true if modifying the header part of the cloned buffer 3506 * does not requires the data to be copied. 3507 */ 3508 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len) 3509 { 3510 return !skb_header_cloned(skb) && 3511 skb_headroom(skb) + len <= skb->hdr_len; 3512 } 3513 3514 static inline int skb_try_make_writable(struct sk_buff *skb, 3515 unsigned int write_len) 3516 { 3517 return skb_cloned(skb) && !skb_clone_writable(skb, write_len) && 3518 pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 3519 } 3520 3521 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom, 3522 int cloned) 3523 { 3524 int delta = 0; 3525 3526 if (headroom > skb_headroom(skb)) 3527 delta = headroom - skb_headroom(skb); 3528 3529 if (delta || cloned) 3530 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0, 3531 GFP_ATOMIC); 3532 return 0; 3533 } 3534 3535 /** 3536 * skb_cow - copy header of skb when it is required 3537 * @skb: buffer to cow 3538 * @headroom: needed headroom 3539 * 3540 * If the skb passed lacks sufficient headroom or its data part 3541 * is shared, data is reallocated. If reallocation fails, an error 3542 * is returned and original skb is not changed. 3543 * 3544 * The result is skb with writable area skb->head...skb->tail 3545 * and at least @headroom of space at head. 3546 */ 3547 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom) 3548 { 3549 return __skb_cow(skb, headroom, skb_cloned(skb)); 3550 } 3551 3552 /** 3553 * skb_cow_head - skb_cow but only making the head writable 3554 * @skb: buffer to cow 3555 * @headroom: needed headroom 3556 * 3557 * This function is identical to skb_cow except that we replace the 3558 * skb_cloned check by skb_header_cloned. It should be used when 3559 * you only need to push on some header and do not need to modify 3560 * the data. 3561 */ 3562 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom) 3563 { 3564 return __skb_cow(skb, headroom, skb_header_cloned(skb)); 3565 } 3566 3567 /** 3568 * skb_padto - pad an skbuff up to a minimal size 3569 * @skb: buffer to pad 3570 * @len: minimal length 3571 * 3572 * Pads up a buffer to ensure the trailing bytes exist and are 3573 * blanked. If the buffer already contains sufficient data it 3574 * is untouched. Otherwise it is extended. Returns zero on 3575 * success. The skb is freed on error. 3576 */ 3577 static inline int skb_padto(struct sk_buff *skb, unsigned int len) 3578 { 3579 unsigned int size = skb->len; 3580 if (likely(size >= len)) 3581 return 0; 3582 return skb_pad(skb, len - size); 3583 } 3584 3585 /** 3586 * __skb_put_padto - increase size and pad an skbuff up to a minimal size 3587 * @skb: buffer to pad 3588 * @len: minimal length 3589 * @free_on_error: free buffer on error 3590 * 3591 * Pads up a buffer to ensure the trailing bytes exist and are 3592 * blanked. If the buffer already contains sufficient data it 3593 * is untouched. Otherwise it is extended. Returns zero on 3594 * success. The skb is freed on error if @free_on_error is true. 3595 */ 3596 static inline int __must_check __skb_put_padto(struct sk_buff *skb, 3597 unsigned int len, 3598 bool free_on_error) 3599 { 3600 unsigned int size = skb->len; 3601 3602 if (unlikely(size < len)) { 3603 len -= size; 3604 if (__skb_pad(skb, len, free_on_error)) 3605 return -ENOMEM; 3606 __skb_put(skb, len); 3607 } 3608 return 0; 3609 } 3610 3611 /** 3612 * skb_put_padto - increase size and pad an skbuff up to a minimal size 3613 * @skb: buffer to pad 3614 * @len: minimal length 3615 * 3616 * Pads up a buffer to ensure the trailing bytes exist and are 3617 * blanked. If the buffer already contains sufficient data it 3618 * is untouched. Otherwise it is extended. Returns zero on 3619 * success. The skb is freed on error. 3620 */ 3621 static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len) 3622 { 3623 return __skb_put_padto(skb, len, true); 3624 } 3625 3626 static inline int skb_add_data(struct sk_buff *skb, 3627 struct iov_iter *from, int copy) 3628 { 3629 const int off = skb->len; 3630 3631 if (skb->ip_summed == CHECKSUM_NONE) { 3632 __wsum csum = 0; 3633 if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy, 3634 &csum, from)) { 3635 skb->csum = csum_block_add(skb->csum, csum, off); 3636 return 0; 3637 } 3638 } else if (copy_from_iter_full(skb_put(skb, copy), copy, from)) 3639 return 0; 3640 3641 __skb_trim(skb, off); 3642 return -EFAULT; 3643 } 3644 3645 static inline bool skb_can_coalesce(struct sk_buff *skb, int i, 3646 const struct page *page, int off) 3647 { 3648 if (skb_zcopy(skb)) 3649 return false; 3650 if (i) { 3651 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 3652 3653 return page == skb_frag_page(frag) && 3654 off == skb_frag_off(frag) + skb_frag_size(frag); 3655 } 3656 return false; 3657 } 3658 3659 static inline int __skb_linearize(struct sk_buff *skb) 3660 { 3661 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM; 3662 } 3663 3664 /** 3665 * skb_linearize - convert paged skb to linear one 3666 * @skb: buffer to linarize 3667 * 3668 * If there is no free memory -ENOMEM is returned, otherwise zero 3669 * is returned and the old skb data released. 3670 */ 3671 static inline int skb_linearize(struct sk_buff *skb) 3672 { 3673 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0; 3674 } 3675 3676 /** 3677 * skb_has_shared_frag - can any frag be overwritten 3678 * @skb: buffer to test 3679 * 3680 * Return true if the skb has at least one frag that might be modified 3681 * by an external entity (as in vmsplice()/sendfile()) 3682 */ 3683 static inline bool skb_has_shared_frag(const struct sk_buff *skb) 3684 { 3685 return skb_is_nonlinear(skb) && 3686 skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 3687 } 3688 3689 /** 3690 * skb_linearize_cow - make sure skb is linear and writable 3691 * @skb: buffer to process 3692 * 3693 * If there is no free memory -ENOMEM is returned, otherwise zero 3694 * is returned and the old skb data released. 3695 */ 3696 static inline int skb_linearize_cow(struct sk_buff *skb) 3697 { 3698 return skb_is_nonlinear(skb) || skb_cloned(skb) ? 3699 __skb_linearize(skb) : 0; 3700 } 3701 3702 static __always_inline void 3703 __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, 3704 unsigned int off) 3705 { 3706 if (skb->ip_summed == CHECKSUM_COMPLETE) 3707 skb->csum = csum_block_sub(skb->csum, 3708 csum_partial(start, len, 0), off); 3709 else if (skb->ip_summed == CHECKSUM_PARTIAL && 3710 skb_checksum_start_offset(skb) < 0) 3711 skb->ip_summed = CHECKSUM_NONE; 3712 } 3713 3714 /** 3715 * skb_postpull_rcsum - update checksum for received skb after pull 3716 * @skb: buffer to update 3717 * @start: start of data before pull 3718 * @len: length of data pulled 3719 * 3720 * After doing a pull on a received packet, you need to call this to 3721 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to 3722 * CHECKSUM_NONE so that it can be recomputed from scratch. 3723 */ 3724 static inline void skb_postpull_rcsum(struct sk_buff *skb, 3725 const void *start, unsigned int len) 3726 { 3727 if (skb->ip_summed == CHECKSUM_COMPLETE) 3728 skb->csum = wsum_negate(csum_partial(start, len, 3729 wsum_negate(skb->csum))); 3730 else if (skb->ip_summed == CHECKSUM_PARTIAL && 3731 skb_checksum_start_offset(skb) < 0) 3732 skb->ip_summed = CHECKSUM_NONE; 3733 } 3734 3735 static __always_inline void 3736 __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, 3737 unsigned int off) 3738 { 3739 if (skb->ip_summed == CHECKSUM_COMPLETE) 3740 skb->csum = csum_block_add(skb->csum, 3741 csum_partial(start, len, 0), off); 3742 } 3743 3744 /** 3745 * skb_postpush_rcsum - update checksum for received skb after push 3746 * @skb: buffer to update 3747 * @start: start of data after push 3748 * @len: length of data pushed 3749 * 3750 * After doing a push on a received packet, you need to call this to 3751 * update the CHECKSUM_COMPLETE checksum. 3752 */ 3753 static inline void skb_postpush_rcsum(struct sk_buff *skb, 3754 const void *start, unsigned int len) 3755 { 3756 __skb_postpush_rcsum(skb, start, len, 0); 3757 } 3758 3759 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); 3760 3761 /** 3762 * skb_push_rcsum - push skb and update receive checksum 3763 * @skb: buffer to update 3764 * @len: length of data pulled 3765 * 3766 * This function performs an skb_push on the packet and updates 3767 * the CHECKSUM_COMPLETE checksum. It should be used on 3768 * receive path processing instead of skb_push unless you know 3769 * that the checksum difference is zero (e.g., a valid IP header) 3770 * or you are setting ip_summed to CHECKSUM_NONE. 3771 */ 3772 static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len) 3773 { 3774 skb_push(skb, len); 3775 skb_postpush_rcsum(skb, skb->data, len); 3776 return skb->data; 3777 } 3778 3779 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len); 3780 /** 3781 * pskb_trim_rcsum - trim received skb and update checksum 3782 * @skb: buffer to trim 3783 * @len: new length 3784 * 3785 * This is exactly the same as pskb_trim except that it ensures the 3786 * checksum of received packets are still valid after the operation. 3787 * It can change skb pointers. 3788 */ 3789 3790 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) 3791 { 3792 if (likely(len >= skb->len)) 3793 return 0; 3794 return pskb_trim_rcsum_slow(skb, len); 3795 } 3796 3797 static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len) 3798 { 3799 if (skb->ip_summed == CHECKSUM_COMPLETE) 3800 skb->ip_summed = CHECKSUM_NONE; 3801 __skb_trim(skb, len); 3802 return 0; 3803 } 3804 3805 static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len) 3806 { 3807 if (skb->ip_summed == CHECKSUM_COMPLETE) 3808 skb->ip_summed = CHECKSUM_NONE; 3809 return __skb_grow(skb, len); 3810 } 3811 3812 #define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode) 3813 #define skb_rb_first(root) rb_to_skb(rb_first(root)) 3814 #define skb_rb_last(root) rb_to_skb(rb_last(root)) 3815 #define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode)) 3816 #define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode)) 3817 3818 #define skb_queue_walk(queue, skb) \ 3819 for (skb = (queue)->next; \ 3820 skb != (struct sk_buff *)(queue); \ 3821 skb = skb->next) 3822 3823 #define skb_queue_walk_safe(queue, skb, tmp) \ 3824 for (skb = (queue)->next, tmp = skb->next; \ 3825 skb != (struct sk_buff *)(queue); \ 3826 skb = tmp, tmp = skb->next) 3827 3828 #define skb_queue_walk_from(queue, skb) \ 3829 for (; skb != (struct sk_buff *)(queue); \ 3830 skb = skb->next) 3831 3832 #define skb_rbtree_walk(skb, root) \ 3833 for (skb = skb_rb_first(root); skb != NULL; \ 3834 skb = skb_rb_next(skb)) 3835 3836 #define skb_rbtree_walk_from(skb) \ 3837 for (; skb != NULL; \ 3838 skb = skb_rb_next(skb)) 3839 3840 #define skb_rbtree_walk_from_safe(skb, tmp) \ 3841 for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL); \ 3842 skb = tmp) 3843 3844 #define skb_queue_walk_from_safe(queue, skb, tmp) \ 3845 for (tmp = skb->next; \ 3846 skb != (struct sk_buff *)(queue); \ 3847 skb = tmp, tmp = skb->next) 3848 3849 #define skb_queue_reverse_walk(queue, skb) \ 3850 for (skb = (queue)->prev; \ 3851 skb != (struct sk_buff *)(queue); \ 3852 skb = skb->prev) 3853 3854 #define skb_queue_reverse_walk_safe(queue, skb, tmp) \ 3855 for (skb = (queue)->prev, tmp = skb->prev; \ 3856 skb != (struct sk_buff *)(queue); \ 3857 skb = tmp, tmp = skb->prev) 3858 3859 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \ 3860 for (tmp = skb->prev; \ 3861 skb != (struct sk_buff *)(queue); \ 3862 skb = tmp, tmp = skb->prev) 3863 3864 static inline bool skb_has_frag_list(const struct sk_buff *skb) 3865 { 3866 return skb_shinfo(skb)->frag_list != NULL; 3867 } 3868 3869 static inline void skb_frag_list_init(struct sk_buff *skb) 3870 { 3871 skb_shinfo(skb)->frag_list = NULL; 3872 } 3873 3874 #define skb_walk_frags(skb, iter) \ 3875 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next) 3876 3877 3878 int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue, 3879 int *err, long *timeo_p, 3880 const struct sk_buff *skb); 3881 struct sk_buff *__skb_try_recv_from_queue(struct sock *sk, 3882 struct sk_buff_head *queue, 3883 unsigned int flags, 3884 int *off, int *err, 3885 struct sk_buff **last); 3886 struct sk_buff *__skb_try_recv_datagram(struct sock *sk, 3887 struct sk_buff_head *queue, 3888 unsigned int flags, int *off, int *err, 3889 struct sk_buff **last); 3890 struct sk_buff *__skb_recv_datagram(struct sock *sk, 3891 struct sk_buff_head *sk_queue, 3892 unsigned int flags, int *off, int *err); 3893 struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err); 3894 __poll_t datagram_poll(struct file *file, struct socket *sock, 3895 struct poll_table_struct *wait); 3896 int skb_copy_datagram_iter(const struct sk_buff *from, int offset, 3897 struct iov_iter *to, int size); 3898 static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset, 3899 struct msghdr *msg, int size) 3900 { 3901 return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size); 3902 } 3903 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen, 3904 struct msghdr *msg); 3905 int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset, 3906 struct iov_iter *to, int len, 3907 struct ahash_request *hash); 3908 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset, 3909 struct iov_iter *from, int len); 3910 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm); 3911 void skb_free_datagram(struct sock *sk, struct sk_buff *skb); 3912 void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len); 3913 static inline void skb_free_datagram_locked(struct sock *sk, 3914 struct sk_buff *skb) 3915 { 3916 __skb_free_datagram_locked(sk, skb, 0); 3917 } 3918 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags); 3919 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len); 3920 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len); 3921 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to, 3922 int len); 3923 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 3924 struct pipe_inode_info *pipe, unsigned int len, 3925 unsigned int flags); 3926 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 3927 int len); 3928 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len); 3929 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to); 3930 unsigned int skb_zerocopy_headlen(const struct sk_buff *from); 3931 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from, 3932 int len, int hlen); 3933 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len); 3934 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen); 3935 void skb_scrub_packet(struct sk_buff *skb, bool xnet); 3936 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu); 3937 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len); 3938 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features); 3939 struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features, 3940 unsigned int offset); 3941 struct sk_buff *skb_vlan_untag(struct sk_buff *skb); 3942 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len); 3943 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci); 3944 int skb_vlan_pop(struct sk_buff *skb); 3945 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci); 3946 int skb_eth_pop(struct sk_buff *skb); 3947 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 3948 const unsigned char *src); 3949 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 3950 int mac_len, bool ethernet); 3951 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 3952 bool ethernet); 3953 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse); 3954 int skb_mpls_dec_ttl(struct sk_buff *skb); 3955 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy, 3956 gfp_t gfp); 3957 3958 static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len) 3959 { 3960 return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT; 3961 } 3962 3963 static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len) 3964 { 3965 return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT; 3966 } 3967 3968 struct skb_checksum_ops { 3969 __wsum (*update)(const void *mem, int len, __wsum wsum); 3970 __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len); 3971 }; 3972 3973 extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly; 3974 3975 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 3976 __wsum csum, const struct skb_checksum_ops *ops); 3977 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, 3978 __wsum csum); 3979 3980 static inline void * __must_check 3981 __skb_header_pointer(const struct sk_buff *skb, int offset, int len, 3982 const void *data, int hlen, void *buffer) 3983 { 3984 if (likely(hlen - offset >= len)) 3985 return (void *)data + offset; 3986 3987 if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0)) 3988 return NULL; 3989 3990 return buffer; 3991 } 3992 3993 static inline void * __must_check 3994 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer) 3995 { 3996 return __skb_header_pointer(skb, offset, len, skb->data, 3997 skb_headlen(skb), buffer); 3998 } 3999 4000 /** 4001 * skb_needs_linearize - check if we need to linearize a given skb 4002 * depending on the given device features. 4003 * @skb: socket buffer to check 4004 * @features: net device features 4005 * 4006 * Returns true if either: 4007 * 1. skb has frag_list and the device doesn't support FRAGLIST, or 4008 * 2. skb is fragmented and the device does not support SG. 4009 */ 4010 static inline bool skb_needs_linearize(struct sk_buff *skb, 4011 netdev_features_t features) 4012 { 4013 return skb_is_nonlinear(skb) && 4014 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) || 4015 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG))); 4016 } 4017 4018 static inline void skb_copy_from_linear_data(const struct sk_buff *skb, 4019 void *to, 4020 const unsigned int len) 4021 { 4022 memcpy(to, skb->data, len); 4023 } 4024 4025 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb, 4026 const int offset, void *to, 4027 const unsigned int len) 4028 { 4029 memcpy(to, skb->data + offset, len); 4030 } 4031 4032 static inline void skb_copy_to_linear_data(struct sk_buff *skb, 4033 const void *from, 4034 const unsigned int len) 4035 { 4036 memcpy(skb->data, from, len); 4037 } 4038 4039 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb, 4040 const int offset, 4041 const void *from, 4042 const unsigned int len) 4043 { 4044 memcpy(skb->data + offset, from, len); 4045 } 4046 4047 void skb_init(void); 4048 4049 static inline ktime_t skb_get_ktime(const struct sk_buff *skb) 4050 { 4051 return skb->tstamp; 4052 } 4053 4054 /** 4055 * skb_get_timestamp - get timestamp from a skb 4056 * @skb: skb to get stamp from 4057 * @stamp: pointer to struct __kernel_old_timeval to store stamp in 4058 * 4059 * Timestamps are stored in the skb as offsets to a base timestamp. 4060 * This function converts the offset back to a struct timeval and stores 4061 * it in stamp. 4062 */ 4063 static inline void skb_get_timestamp(const struct sk_buff *skb, 4064 struct __kernel_old_timeval *stamp) 4065 { 4066 *stamp = ns_to_kernel_old_timeval(skb->tstamp); 4067 } 4068 4069 static inline void skb_get_new_timestamp(const struct sk_buff *skb, 4070 struct __kernel_sock_timeval *stamp) 4071 { 4072 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); 4073 4074 stamp->tv_sec = ts.tv_sec; 4075 stamp->tv_usec = ts.tv_nsec / 1000; 4076 } 4077 4078 static inline void skb_get_timestampns(const struct sk_buff *skb, 4079 struct __kernel_old_timespec *stamp) 4080 { 4081 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); 4082 4083 stamp->tv_sec = ts.tv_sec; 4084 stamp->tv_nsec = ts.tv_nsec; 4085 } 4086 4087 static inline void skb_get_new_timestampns(const struct sk_buff *skb, 4088 struct __kernel_timespec *stamp) 4089 { 4090 struct timespec64 ts = ktime_to_timespec64(skb->tstamp); 4091 4092 stamp->tv_sec = ts.tv_sec; 4093 stamp->tv_nsec = ts.tv_nsec; 4094 } 4095 4096 static inline void __net_timestamp(struct sk_buff *skb) 4097 { 4098 skb->tstamp = ktime_get_real(); 4099 skb->mono_delivery_time = 0; 4100 } 4101 4102 static inline ktime_t net_timedelta(ktime_t t) 4103 { 4104 return ktime_sub(ktime_get_real(), t); 4105 } 4106 4107 static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt, 4108 bool mono) 4109 { 4110 skb->tstamp = kt; 4111 skb->mono_delivery_time = kt && mono; 4112 } 4113 4114 DECLARE_STATIC_KEY_FALSE(netstamp_needed_key); 4115 4116 /* It is used in the ingress path to clear the delivery_time. 4117 * If needed, set the skb->tstamp to the (rcv) timestamp. 4118 */ 4119 static inline void skb_clear_delivery_time(struct sk_buff *skb) 4120 { 4121 if (skb->mono_delivery_time) { 4122 skb->mono_delivery_time = 0; 4123 if (static_branch_unlikely(&netstamp_needed_key)) 4124 skb->tstamp = ktime_get_real(); 4125 else 4126 skb->tstamp = 0; 4127 } 4128 } 4129 4130 static inline void skb_clear_tstamp(struct sk_buff *skb) 4131 { 4132 if (skb->mono_delivery_time) 4133 return; 4134 4135 skb->tstamp = 0; 4136 } 4137 4138 static inline ktime_t skb_tstamp(const struct sk_buff *skb) 4139 { 4140 if (skb->mono_delivery_time) 4141 return 0; 4142 4143 return skb->tstamp; 4144 } 4145 4146 static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond) 4147 { 4148 if (!skb->mono_delivery_time && skb->tstamp) 4149 return skb->tstamp; 4150 4151 if (static_branch_unlikely(&netstamp_needed_key) || cond) 4152 return ktime_get_real(); 4153 4154 return 0; 4155 } 4156 4157 static inline u8 skb_metadata_len(const struct sk_buff *skb) 4158 { 4159 return skb_shinfo(skb)->meta_len; 4160 } 4161 4162 static inline void *skb_metadata_end(const struct sk_buff *skb) 4163 { 4164 return skb_mac_header(skb); 4165 } 4166 4167 static inline bool __skb_metadata_differs(const struct sk_buff *skb_a, 4168 const struct sk_buff *skb_b, 4169 u8 meta_len) 4170 { 4171 const void *a = skb_metadata_end(skb_a); 4172 const void *b = skb_metadata_end(skb_b); 4173 /* Using more efficient varaiant than plain call to memcmp(). */ 4174 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64 4175 u64 diffs = 0; 4176 4177 switch (meta_len) { 4178 #define __it(x, op) (x -= sizeof(u##op)) 4179 #define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op)) 4180 case 32: diffs |= __it_diff(a, b, 64); 4181 fallthrough; 4182 case 24: diffs |= __it_diff(a, b, 64); 4183 fallthrough; 4184 case 16: diffs |= __it_diff(a, b, 64); 4185 fallthrough; 4186 case 8: diffs |= __it_diff(a, b, 64); 4187 break; 4188 case 28: diffs |= __it_diff(a, b, 64); 4189 fallthrough; 4190 case 20: diffs |= __it_diff(a, b, 64); 4191 fallthrough; 4192 case 12: diffs |= __it_diff(a, b, 64); 4193 fallthrough; 4194 case 4: diffs |= __it_diff(a, b, 32); 4195 break; 4196 } 4197 return diffs; 4198 #else 4199 return memcmp(a - meta_len, b - meta_len, meta_len); 4200 #endif 4201 } 4202 4203 static inline bool skb_metadata_differs(const struct sk_buff *skb_a, 4204 const struct sk_buff *skb_b) 4205 { 4206 u8 len_a = skb_metadata_len(skb_a); 4207 u8 len_b = skb_metadata_len(skb_b); 4208 4209 if (!(len_a | len_b)) 4210 return false; 4211 4212 return len_a != len_b ? 4213 true : __skb_metadata_differs(skb_a, skb_b, len_a); 4214 } 4215 4216 static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len) 4217 { 4218 skb_shinfo(skb)->meta_len = meta_len; 4219 } 4220 4221 static inline void skb_metadata_clear(struct sk_buff *skb) 4222 { 4223 skb_metadata_set(skb, 0); 4224 } 4225 4226 struct sk_buff *skb_clone_sk(struct sk_buff *skb); 4227 4228 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING 4229 4230 void skb_clone_tx_timestamp(struct sk_buff *skb); 4231 bool skb_defer_rx_timestamp(struct sk_buff *skb); 4232 4233 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */ 4234 4235 static inline void skb_clone_tx_timestamp(struct sk_buff *skb) 4236 { 4237 } 4238 4239 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb) 4240 { 4241 return false; 4242 } 4243 4244 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */ 4245 4246 /** 4247 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps 4248 * 4249 * PHY drivers may accept clones of transmitted packets for 4250 * timestamping via their phy_driver.txtstamp method. These drivers 4251 * must call this function to return the skb back to the stack with a 4252 * timestamp. 4253 * 4254 * @skb: clone of the original outgoing packet 4255 * @hwtstamps: hardware time stamps 4256 * 4257 */ 4258 void skb_complete_tx_timestamp(struct sk_buff *skb, 4259 struct skb_shared_hwtstamps *hwtstamps); 4260 4261 void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb, 4262 struct skb_shared_hwtstamps *hwtstamps, 4263 struct sock *sk, int tstype); 4264 4265 /** 4266 * skb_tstamp_tx - queue clone of skb with send time stamps 4267 * @orig_skb: the original outgoing packet 4268 * @hwtstamps: hardware time stamps, may be NULL if not available 4269 * 4270 * If the skb has a socket associated, then this function clones the 4271 * skb (thus sharing the actual data and optional structures), stores 4272 * the optional hardware time stamping information (if non NULL) or 4273 * generates a software time stamp (otherwise), then queues the clone 4274 * to the error queue of the socket. Errors are silently ignored. 4275 */ 4276 void skb_tstamp_tx(struct sk_buff *orig_skb, 4277 struct skb_shared_hwtstamps *hwtstamps); 4278 4279 /** 4280 * skb_tx_timestamp() - Driver hook for transmit timestamping 4281 * 4282 * Ethernet MAC Drivers should call this function in their hard_xmit() 4283 * function immediately before giving the sk_buff to the MAC hardware. 4284 * 4285 * Specifically, one should make absolutely sure that this function is 4286 * called before TX completion of this packet can trigger. Otherwise 4287 * the packet could potentially already be freed. 4288 * 4289 * @skb: A socket buffer. 4290 */ 4291 static inline void skb_tx_timestamp(struct sk_buff *skb) 4292 { 4293 skb_clone_tx_timestamp(skb); 4294 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP) 4295 skb_tstamp_tx(skb, NULL); 4296 } 4297 4298 /** 4299 * skb_complete_wifi_ack - deliver skb with wifi status 4300 * 4301 * @skb: the original outgoing packet 4302 * @acked: ack status 4303 * 4304 */ 4305 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked); 4306 4307 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len); 4308 __sum16 __skb_checksum_complete(struct sk_buff *skb); 4309 4310 static inline int skb_csum_unnecessary(const struct sk_buff *skb) 4311 { 4312 return ((skb->ip_summed == CHECKSUM_UNNECESSARY) || 4313 skb->csum_valid || 4314 (skb->ip_summed == CHECKSUM_PARTIAL && 4315 skb_checksum_start_offset(skb) >= 0)); 4316 } 4317 4318 /** 4319 * skb_checksum_complete - Calculate checksum of an entire packet 4320 * @skb: packet to process 4321 * 4322 * This function calculates the checksum over the entire packet plus 4323 * the value of skb->csum. The latter can be used to supply the 4324 * checksum of a pseudo header as used by TCP/UDP. It returns the 4325 * checksum. 4326 * 4327 * For protocols that contain complete checksums such as ICMP/TCP/UDP, 4328 * this function can be used to verify that checksum on received 4329 * packets. In that case the function should return zero if the 4330 * checksum is correct. In particular, this function will return zero 4331 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the 4332 * hardware has already verified the correctness of the checksum. 4333 */ 4334 static inline __sum16 skb_checksum_complete(struct sk_buff *skb) 4335 { 4336 return skb_csum_unnecessary(skb) ? 4337 0 : __skb_checksum_complete(skb); 4338 } 4339 4340 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb) 4341 { 4342 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 4343 if (skb->csum_level == 0) 4344 skb->ip_summed = CHECKSUM_NONE; 4345 else 4346 skb->csum_level--; 4347 } 4348 } 4349 4350 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb) 4351 { 4352 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 4353 if (skb->csum_level < SKB_MAX_CSUM_LEVEL) 4354 skb->csum_level++; 4355 } else if (skb->ip_summed == CHECKSUM_NONE) { 4356 skb->ip_summed = CHECKSUM_UNNECESSARY; 4357 skb->csum_level = 0; 4358 } 4359 } 4360 4361 static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb) 4362 { 4363 if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 4364 skb->ip_summed = CHECKSUM_NONE; 4365 skb->csum_level = 0; 4366 } 4367 } 4368 4369 /* Check if we need to perform checksum complete validation. 4370 * 4371 * Returns true if checksum complete is needed, false otherwise 4372 * (either checksum is unnecessary or zero checksum is allowed). 4373 */ 4374 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb, 4375 bool zero_okay, 4376 __sum16 check) 4377 { 4378 if (skb_csum_unnecessary(skb) || (zero_okay && !check)) { 4379 skb->csum_valid = 1; 4380 __skb_decr_checksum_unnecessary(skb); 4381 return false; 4382 } 4383 4384 return true; 4385 } 4386 4387 /* For small packets <= CHECKSUM_BREAK perform checksum complete directly 4388 * in checksum_init. 4389 */ 4390 #define CHECKSUM_BREAK 76 4391 4392 /* Unset checksum-complete 4393 * 4394 * Unset checksum complete can be done when packet is being modified 4395 * (uncompressed for instance) and checksum-complete value is 4396 * invalidated. 4397 */ 4398 static inline void skb_checksum_complete_unset(struct sk_buff *skb) 4399 { 4400 if (skb->ip_summed == CHECKSUM_COMPLETE) 4401 skb->ip_summed = CHECKSUM_NONE; 4402 } 4403 4404 /* Validate (init) checksum based on checksum complete. 4405 * 4406 * Return values: 4407 * 0: checksum is validated or try to in skb_checksum_complete. In the latter 4408 * case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo 4409 * checksum is stored in skb->csum for use in __skb_checksum_complete 4410 * non-zero: value of invalid checksum 4411 * 4412 */ 4413 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb, 4414 bool complete, 4415 __wsum psum) 4416 { 4417 if (skb->ip_summed == CHECKSUM_COMPLETE) { 4418 if (!csum_fold(csum_add(psum, skb->csum))) { 4419 skb->csum_valid = 1; 4420 return 0; 4421 } 4422 } 4423 4424 skb->csum = psum; 4425 4426 if (complete || skb->len <= CHECKSUM_BREAK) { 4427 __sum16 csum; 4428 4429 csum = __skb_checksum_complete(skb); 4430 skb->csum_valid = !csum; 4431 return csum; 4432 } 4433 4434 return 0; 4435 } 4436 4437 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto) 4438 { 4439 return 0; 4440 } 4441 4442 /* Perform checksum validate (init). Note that this is a macro since we only 4443 * want to calculate the pseudo header which is an input function if necessary. 4444 * First we try to validate without any computation (checksum unnecessary) and 4445 * then calculate based on checksum complete calling the function to compute 4446 * pseudo header. 4447 * 4448 * Return values: 4449 * 0: checksum is validated or try to in skb_checksum_complete 4450 * non-zero: value of invalid checksum 4451 */ 4452 #define __skb_checksum_validate(skb, proto, complete, \ 4453 zero_okay, check, compute_pseudo) \ 4454 ({ \ 4455 __sum16 __ret = 0; \ 4456 skb->csum_valid = 0; \ 4457 if (__skb_checksum_validate_needed(skb, zero_okay, check)) \ 4458 __ret = __skb_checksum_validate_complete(skb, \ 4459 complete, compute_pseudo(skb, proto)); \ 4460 __ret; \ 4461 }) 4462 4463 #define skb_checksum_init(skb, proto, compute_pseudo) \ 4464 __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo) 4465 4466 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \ 4467 __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo) 4468 4469 #define skb_checksum_validate(skb, proto, compute_pseudo) \ 4470 __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo) 4471 4472 #define skb_checksum_validate_zero_check(skb, proto, check, \ 4473 compute_pseudo) \ 4474 __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo) 4475 4476 #define skb_checksum_simple_validate(skb) \ 4477 __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo) 4478 4479 static inline bool __skb_checksum_convert_check(struct sk_buff *skb) 4480 { 4481 return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid); 4482 } 4483 4484 static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo) 4485 { 4486 skb->csum = ~pseudo; 4487 skb->ip_summed = CHECKSUM_COMPLETE; 4488 } 4489 4490 #define skb_checksum_try_convert(skb, proto, compute_pseudo) \ 4491 do { \ 4492 if (__skb_checksum_convert_check(skb)) \ 4493 __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \ 4494 } while (0) 4495 4496 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr, 4497 u16 start, u16 offset) 4498 { 4499 skb->ip_summed = CHECKSUM_PARTIAL; 4500 skb->csum_start = ((unsigned char *)ptr + start) - skb->head; 4501 skb->csum_offset = offset - start; 4502 } 4503 4504 /* Update skbuf and packet to reflect the remote checksum offload operation. 4505 * When called, ptr indicates the starting point for skb->csum when 4506 * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete 4507 * here, skb_postpull_rcsum is done so skb->csum start is ptr. 4508 */ 4509 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr, 4510 int start, int offset, bool nopartial) 4511 { 4512 __wsum delta; 4513 4514 if (!nopartial) { 4515 skb_remcsum_adjust_partial(skb, ptr, start, offset); 4516 return; 4517 } 4518 4519 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) { 4520 __skb_checksum_complete(skb); 4521 skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data); 4522 } 4523 4524 delta = remcsum_adjust(ptr, skb->csum, start, offset); 4525 4526 /* Adjust skb->csum since we changed the packet */ 4527 skb->csum = csum_add(skb->csum, delta); 4528 } 4529 4530 static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb) 4531 { 4532 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 4533 return (void *)(skb->_nfct & NFCT_PTRMASK); 4534 #else 4535 return NULL; 4536 #endif 4537 } 4538 4539 static inline unsigned long skb_get_nfct(const struct sk_buff *skb) 4540 { 4541 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 4542 return skb->_nfct; 4543 #else 4544 return 0UL; 4545 #endif 4546 } 4547 4548 static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct) 4549 { 4550 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 4551 skb->slow_gro |= !!nfct; 4552 skb->_nfct = nfct; 4553 #endif 4554 } 4555 4556 #ifdef CONFIG_SKB_EXTENSIONS 4557 enum skb_ext_id { 4558 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4559 SKB_EXT_BRIDGE_NF, 4560 #endif 4561 #ifdef CONFIG_XFRM 4562 SKB_EXT_SEC_PATH, 4563 #endif 4564 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4565 TC_SKB_EXT, 4566 #endif 4567 #if IS_ENABLED(CONFIG_MPTCP) 4568 SKB_EXT_MPTCP, 4569 #endif 4570 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4571 SKB_EXT_MCTP, 4572 #endif 4573 SKB_EXT_NUM, /* must be last */ 4574 }; 4575 4576 /** 4577 * struct skb_ext - sk_buff extensions 4578 * @refcnt: 1 on allocation, deallocated on 0 4579 * @offset: offset to add to @data to obtain extension address 4580 * @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units 4581 * @data: start of extension data, variable sized 4582 * 4583 * Note: offsets/lengths are stored in chunks of 8 bytes, this allows 4584 * to use 'u8' types while allowing up to 2kb worth of extension data. 4585 */ 4586 struct skb_ext { 4587 refcount_t refcnt; 4588 u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */ 4589 u8 chunks; /* same */ 4590 char data[] __aligned(8); 4591 }; 4592 4593 struct skb_ext *__skb_ext_alloc(gfp_t flags); 4594 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 4595 struct skb_ext *ext); 4596 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id); 4597 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id); 4598 void __skb_ext_put(struct skb_ext *ext); 4599 4600 static inline void skb_ext_put(struct sk_buff *skb) 4601 { 4602 if (skb->active_extensions) 4603 __skb_ext_put(skb->extensions); 4604 } 4605 4606 static inline void __skb_ext_copy(struct sk_buff *dst, 4607 const struct sk_buff *src) 4608 { 4609 dst->active_extensions = src->active_extensions; 4610 4611 if (src->active_extensions) { 4612 struct skb_ext *ext = src->extensions; 4613 4614 refcount_inc(&ext->refcnt); 4615 dst->extensions = ext; 4616 } 4617 } 4618 4619 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src) 4620 { 4621 skb_ext_put(dst); 4622 __skb_ext_copy(dst, src); 4623 } 4624 4625 static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i) 4626 { 4627 return !!ext->offset[i]; 4628 } 4629 4630 static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id) 4631 { 4632 return skb->active_extensions & (1 << id); 4633 } 4634 4635 static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 4636 { 4637 if (skb_ext_exist(skb, id)) 4638 __skb_ext_del(skb, id); 4639 } 4640 4641 static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id) 4642 { 4643 if (skb_ext_exist(skb, id)) { 4644 struct skb_ext *ext = skb->extensions; 4645 4646 return (void *)ext + (ext->offset[id] << 3); 4647 } 4648 4649 return NULL; 4650 } 4651 4652 static inline void skb_ext_reset(struct sk_buff *skb) 4653 { 4654 if (unlikely(skb->active_extensions)) { 4655 __skb_ext_put(skb->extensions); 4656 skb->active_extensions = 0; 4657 } 4658 } 4659 4660 static inline bool skb_has_extensions(struct sk_buff *skb) 4661 { 4662 return unlikely(skb->active_extensions); 4663 } 4664 #else 4665 static inline void skb_ext_put(struct sk_buff *skb) {} 4666 static inline void skb_ext_reset(struct sk_buff *skb) {} 4667 static inline void skb_ext_del(struct sk_buff *skb, int unused) {} 4668 static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {} 4669 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {} 4670 static inline bool skb_has_extensions(struct sk_buff *skb) { return false; } 4671 #endif /* CONFIG_SKB_EXTENSIONS */ 4672 4673 static inline void nf_reset_ct(struct sk_buff *skb) 4674 { 4675 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 4676 nf_conntrack_put(skb_nfct(skb)); 4677 skb->_nfct = 0; 4678 #endif 4679 } 4680 4681 static inline void nf_reset_trace(struct sk_buff *skb) 4682 { 4683 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES) 4684 skb->nf_trace = 0; 4685 #endif 4686 } 4687 4688 static inline void ipvs_reset(struct sk_buff *skb) 4689 { 4690 #if IS_ENABLED(CONFIG_IP_VS) 4691 skb->ipvs_property = 0; 4692 #endif 4693 } 4694 4695 /* Note: This doesn't put any conntrack info in dst. */ 4696 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src, 4697 bool copy) 4698 { 4699 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 4700 dst->_nfct = src->_nfct; 4701 nf_conntrack_get(skb_nfct(src)); 4702 #endif 4703 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES) 4704 if (copy) 4705 dst->nf_trace = src->nf_trace; 4706 #endif 4707 } 4708 4709 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src) 4710 { 4711 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 4712 nf_conntrack_put(skb_nfct(dst)); 4713 #endif 4714 dst->slow_gro = src->slow_gro; 4715 __nf_copy(dst, src, true); 4716 } 4717 4718 #ifdef CONFIG_NETWORK_SECMARK 4719 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from) 4720 { 4721 to->secmark = from->secmark; 4722 } 4723 4724 static inline void skb_init_secmark(struct sk_buff *skb) 4725 { 4726 skb->secmark = 0; 4727 } 4728 #else 4729 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from) 4730 { } 4731 4732 static inline void skb_init_secmark(struct sk_buff *skb) 4733 { } 4734 #endif 4735 4736 static inline int secpath_exists(const struct sk_buff *skb) 4737 { 4738 #ifdef CONFIG_XFRM 4739 return skb_ext_exist(skb, SKB_EXT_SEC_PATH); 4740 #else 4741 return 0; 4742 #endif 4743 } 4744 4745 static inline bool skb_irq_freeable(const struct sk_buff *skb) 4746 { 4747 return !skb->destructor && 4748 !secpath_exists(skb) && 4749 !skb_nfct(skb) && 4750 !skb->_skb_refdst && 4751 !skb_has_frag_list(skb); 4752 } 4753 4754 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping) 4755 { 4756 skb->queue_mapping = queue_mapping; 4757 } 4758 4759 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb) 4760 { 4761 return skb->queue_mapping; 4762 } 4763 4764 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from) 4765 { 4766 to->queue_mapping = from->queue_mapping; 4767 } 4768 4769 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue) 4770 { 4771 skb->queue_mapping = rx_queue + 1; 4772 } 4773 4774 static inline u16 skb_get_rx_queue(const struct sk_buff *skb) 4775 { 4776 return skb->queue_mapping - 1; 4777 } 4778 4779 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb) 4780 { 4781 return skb->queue_mapping != 0; 4782 } 4783 4784 static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val) 4785 { 4786 skb->dst_pending_confirm = val; 4787 } 4788 4789 static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb) 4790 { 4791 return skb->dst_pending_confirm != 0; 4792 } 4793 4794 static inline struct sec_path *skb_sec_path(const struct sk_buff *skb) 4795 { 4796 #ifdef CONFIG_XFRM 4797 return skb_ext_find(skb, SKB_EXT_SEC_PATH); 4798 #else 4799 return NULL; 4800 #endif 4801 } 4802 4803 /* Keeps track of mac header offset relative to skb->head. 4804 * It is useful for TSO of Tunneling protocol. e.g. GRE. 4805 * For non-tunnel skb it points to skb_mac_header() and for 4806 * tunnel skb it points to outer mac header. 4807 * Keeps track of level of encapsulation of network headers. 4808 */ 4809 struct skb_gso_cb { 4810 union { 4811 int mac_offset; 4812 int data_offset; 4813 }; 4814 int encap_level; 4815 __wsum csum; 4816 __u16 csum_start; 4817 }; 4818 #define SKB_GSO_CB_OFFSET 32 4819 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET)) 4820 4821 static inline int skb_tnl_header_len(const struct sk_buff *inner_skb) 4822 { 4823 return (skb_mac_header(inner_skb) - inner_skb->head) - 4824 SKB_GSO_CB(inner_skb)->mac_offset; 4825 } 4826 4827 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra) 4828 { 4829 int new_headroom, headroom; 4830 int ret; 4831 4832 headroom = skb_headroom(skb); 4833 ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC); 4834 if (ret) 4835 return ret; 4836 4837 new_headroom = skb_headroom(skb); 4838 SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom); 4839 return 0; 4840 } 4841 4842 static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res) 4843 { 4844 /* Do not update partial checksums if remote checksum is enabled. */ 4845 if (skb->remcsum_offload) 4846 return; 4847 4848 SKB_GSO_CB(skb)->csum = res; 4849 SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head; 4850 } 4851 4852 /* Compute the checksum for a gso segment. First compute the checksum value 4853 * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and 4854 * then add in skb->csum (checksum from csum_start to end of packet). 4855 * skb->csum and csum_start are then updated to reflect the checksum of the 4856 * resultant packet starting from the transport header-- the resultant checksum 4857 * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo 4858 * header. 4859 */ 4860 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res) 4861 { 4862 unsigned char *csum_start = skb_transport_header(skb); 4863 int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start; 4864 __wsum partial = SKB_GSO_CB(skb)->csum; 4865 4866 SKB_GSO_CB(skb)->csum = res; 4867 SKB_GSO_CB(skb)->csum_start = csum_start - skb->head; 4868 4869 return csum_fold(csum_partial(csum_start, plen, partial)); 4870 } 4871 4872 static inline bool skb_is_gso(const struct sk_buff *skb) 4873 { 4874 return skb_shinfo(skb)->gso_size; 4875 } 4876 4877 /* Note: Should be called only if skb_is_gso(skb) is true */ 4878 static inline bool skb_is_gso_v6(const struct sk_buff *skb) 4879 { 4880 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; 4881 } 4882 4883 /* Note: Should be called only if skb_is_gso(skb) is true */ 4884 static inline bool skb_is_gso_sctp(const struct sk_buff *skb) 4885 { 4886 return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP; 4887 } 4888 4889 /* Note: Should be called only if skb_is_gso(skb) is true */ 4890 static inline bool skb_is_gso_tcp(const struct sk_buff *skb) 4891 { 4892 return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6); 4893 } 4894 4895 static inline void skb_gso_reset(struct sk_buff *skb) 4896 { 4897 skb_shinfo(skb)->gso_size = 0; 4898 skb_shinfo(skb)->gso_segs = 0; 4899 skb_shinfo(skb)->gso_type = 0; 4900 } 4901 4902 static inline void skb_increase_gso_size(struct skb_shared_info *shinfo, 4903 u16 increment) 4904 { 4905 if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS)) 4906 return; 4907 shinfo->gso_size += increment; 4908 } 4909 4910 static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo, 4911 u16 decrement) 4912 { 4913 if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS)) 4914 return; 4915 shinfo->gso_size -= decrement; 4916 } 4917 4918 void __skb_warn_lro_forwarding(const struct sk_buff *skb); 4919 4920 static inline bool skb_warn_if_lro(const struct sk_buff *skb) 4921 { 4922 /* LRO sets gso_size but not gso_type, whereas if GSO is really 4923 * wanted then gso_type will be set. */ 4924 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4925 4926 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 && 4927 unlikely(shinfo->gso_type == 0)) { 4928 __skb_warn_lro_forwarding(skb); 4929 return true; 4930 } 4931 return false; 4932 } 4933 4934 static inline void skb_forward_csum(struct sk_buff *skb) 4935 { 4936 /* Unfortunately we don't support this one. Any brave souls? */ 4937 if (skb->ip_summed == CHECKSUM_COMPLETE) 4938 skb->ip_summed = CHECKSUM_NONE; 4939 } 4940 4941 /** 4942 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE 4943 * @skb: skb to check 4944 * 4945 * fresh skbs have their ip_summed set to CHECKSUM_NONE. 4946 * Instead of forcing ip_summed to CHECKSUM_NONE, we can 4947 * use this helper, to document places where we make this assertion. 4948 */ 4949 static inline void skb_checksum_none_assert(const struct sk_buff *skb) 4950 { 4951 DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE); 4952 } 4953 4954 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off); 4955 4956 int skb_checksum_setup(struct sk_buff *skb, bool recalculate); 4957 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 4958 unsigned int transport_len, 4959 __sum16(*skb_chkf)(struct sk_buff *skb)); 4960 4961 /** 4962 * skb_head_is_locked - Determine if the skb->head is locked down 4963 * @skb: skb to check 4964 * 4965 * The head on skbs build around a head frag can be removed if they are 4966 * not cloned. This function returns true if the skb head is locked down 4967 * due to either being allocated via kmalloc, or by being a clone with 4968 * multiple references to the head. 4969 */ 4970 static inline bool skb_head_is_locked(const struct sk_buff *skb) 4971 { 4972 return !skb->head_frag || skb_cloned(skb); 4973 } 4974 4975 /* Local Checksum Offload. 4976 * Compute outer checksum based on the assumption that the 4977 * inner checksum will be offloaded later. 4978 * See Documentation/networking/checksum-offloads.rst for 4979 * explanation of how this works. 4980 * Fill in outer checksum adjustment (e.g. with sum of outer 4981 * pseudo-header) before calling. 4982 * Also ensure that inner checksum is in linear data area. 4983 */ 4984 static inline __wsum lco_csum(struct sk_buff *skb) 4985 { 4986 unsigned char *csum_start = skb_checksum_start(skb); 4987 unsigned char *l4_hdr = skb_transport_header(skb); 4988 __wsum partial; 4989 4990 /* Start with complement of inner checksum adjustment */ 4991 partial = ~csum_unfold(*(__force __sum16 *)(csum_start + 4992 skb->csum_offset)); 4993 4994 /* Add in checksum of our headers (incl. outer checksum 4995 * adjustment filled in by caller) and return result. 4996 */ 4997 return csum_partial(l4_hdr, csum_start - l4_hdr, partial); 4998 } 4999 5000 static inline bool skb_is_redirected(const struct sk_buff *skb) 5001 { 5002 return skb->redirected; 5003 } 5004 5005 static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress) 5006 { 5007 skb->redirected = 1; 5008 #ifdef CONFIG_NET_REDIRECT 5009 skb->from_ingress = from_ingress; 5010 if (skb->from_ingress) 5011 skb_clear_tstamp(skb); 5012 #endif 5013 } 5014 5015 static inline void skb_reset_redirect(struct sk_buff *skb) 5016 { 5017 skb->redirected = 0; 5018 } 5019 5020 static inline bool skb_csum_is_sctp(struct sk_buff *skb) 5021 { 5022 return skb->csum_not_inet; 5023 } 5024 5025 static inline void skb_set_kcov_handle(struct sk_buff *skb, 5026 const u64 kcov_handle) 5027 { 5028 #ifdef CONFIG_KCOV 5029 skb->kcov_handle = kcov_handle; 5030 #endif 5031 } 5032 5033 static inline u64 skb_get_kcov_handle(struct sk_buff *skb) 5034 { 5035 #ifdef CONFIG_KCOV 5036 return skb->kcov_handle; 5037 #else 5038 return 0; 5039 #endif 5040 } 5041 5042 #ifdef CONFIG_PAGE_POOL 5043 static inline void skb_mark_for_recycle(struct sk_buff *skb) 5044 { 5045 skb->pp_recycle = 1; 5046 } 5047 #endif 5048 5049 static inline bool skb_pp_recycle(struct sk_buff *skb, void *data) 5050 { 5051 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 5052 return false; 5053 return page_pool_return_skb_page(virt_to_page(data)); 5054 } 5055 5056 #endif /* __KERNEL__ */ 5057 #endif /* _LINUX_SKBUFF_H */ 5058