1 /* 2 * Definitions for the 'struct sk_buff' memory handlers. 3 * 4 * Authors: 5 * Alan Cox, <[email protected]> 6 * Florian La Roche, <[email protected]> 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License 10 * as published by the Free Software Foundation; either version 11 * 2 of the License, or (at your option) any later version. 12 */ 13 14 #ifndef _LINUX_SKBUFF_H 15 #define _LINUX_SKBUFF_H 16 17 #include <linux/kernel.h> 18 #include <linux/kmemcheck.h> 19 #include <linux/compiler.h> 20 #include <linux/time.h> 21 #include <linux/cache.h> 22 23 #include <linux/atomic.h> 24 #include <asm/types.h> 25 #include <linux/spinlock.h> 26 #include <linux/net.h> 27 #include <linux/textsearch.h> 28 #include <net/checksum.h> 29 #include <linux/rcupdate.h> 30 #include <linux/dmaengine.h> 31 #include <linux/hrtimer.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/netdev_features.h> 34 35 /* Don't change this without changing skb_csum_unnecessary! */ 36 #define CHECKSUM_NONE 0 37 #define CHECKSUM_UNNECESSARY 1 38 #define CHECKSUM_COMPLETE 2 39 #define CHECKSUM_PARTIAL 3 40 41 #define SKB_DATA_ALIGN(X) (((X) + (SMP_CACHE_BYTES - 1)) & \ 42 ~(SMP_CACHE_BYTES - 1)) 43 #define SKB_WITH_OVERHEAD(X) \ 44 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 45 #define SKB_MAX_ORDER(X, ORDER) \ 46 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X)) 47 #define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0)) 48 #define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2)) 49 50 /* return minimum truesize of one skb containing X bytes of data */ 51 #define SKB_TRUESIZE(X) ((X) + \ 52 SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \ 53 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 54 55 /* A. Checksumming of received packets by device. 56 * 57 * NONE: device failed to checksum this packet. 58 * skb->csum is undefined. 59 * 60 * UNNECESSARY: device parsed packet and wouldbe verified checksum. 61 * skb->csum is undefined. 62 * It is bad option, but, unfortunately, many of vendors do this. 63 * Apparently with secret goal to sell you new device, when you 64 * will add new protocol to your host. F.e. IPv6. 8) 65 * 66 * COMPLETE: the most generic way. Device supplied checksum of _all_ 67 * the packet as seen by netif_rx in skb->csum. 68 * NOTE: Even if device supports only some protocols, but 69 * is able to produce some skb->csum, it MUST use COMPLETE, 70 * not UNNECESSARY. 71 * 72 * PARTIAL: identical to the case for output below. This may occur 73 * on a packet received directly from another Linux OS, e.g., 74 * a virtualised Linux kernel on the same host. The packet can 75 * be treated in the same way as UNNECESSARY except that on 76 * output (i.e., forwarding) the checksum must be filled in 77 * by the OS or the hardware. 78 * 79 * B. Checksumming on output. 80 * 81 * NONE: skb is checksummed by protocol or csum is not required. 82 * 83 * PARTIAL: device is required to csum packet as seen by hard_start_xmit 84 * from skb->csum_start to the end and to record the checksum 85 * at skb->csum_start + skb->csum_offset. 86 * 87 * Device must show its capabilities in dev->features, set 88 * at device setup time. 89 * NETIF_F_HW_CSUM - it is clever device, it is able to checksum 90 * everything. 91 * NETIF_F_IP_CSUM - device is dumb. It is able to csum only 92 * TCP/UDP over IPv4. Sigh. Vendors like this 93 * way by an unknown reason. Though, see comment above 94 * about CHECKSUM_UNNECESSARY. 8) 95 * NETIF_F_IPV6_CSUM about as dumb as the last one but does IPv6 instead. 96 * 97 * UNNECESSARY: device will do per protocol specific csum. Protocol drivers 98 * that do not want net to perform the checksum calculation should use 99 * this flag in their outgoing skbs. 100 * NETIF_F_FCOE_CRC this indicates the device can do FCoE FC CRC 101 * offload. Correspondingly, the FCoE protocol driver 102 * stack should use CHECKSUM_UNNECESSARY. 103 * 104 * Any questions? No questions, good. --ANK 105 */ 106 107 struct net_device; 108 struct scatterlist; 109 struct pipe_inode_info; 110 111 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 112 struct nf_conntrack { 113 atomic_t use; 114 }; 115 #endif 116 117 #ifdef CONFIG_BRIDGE_NETFILTER 118 struct nf_bridge_info { 119 atomic_t use; 120 struct net_device *physindev; 121 struct net_device *physoutdev; 122 unsigned int mask; 123 unsigned long data[32 / sizeof(unsigned long)]; 124 }; 125 #endif 126 127 struct sk_buff_head { 128 /* These two members must be first. */ 129 struct sk_buff *next; 130 struct sk_buff *prev; 131 132 __u32 qlen; 133 spinlock_t lock; 134 }; 135 136 struct sk_buff; 137 138 /* To allow 64K frame to be packed as single skb without frag_list we 139 * require 64K/PAGE_SIZE pages plus 1 additional page to allow for 140 * buffers which do not start on a page boundary. 141 * 142 * Since GRO uses frags we allocate at least 16 regardless of page 143 * size. 144 */ 145 #if (65536/PAGE_SIZE + 1) < 16 146 #define MAX_SKB_FRAGS 16UL 147 #else 148 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1) 149 #endif 150 151 typedef struct skb_frag_struct skb_frag_t; 152 153 struct skb_frag_struct { 154 struct { 155 struct page *p; 156 } page; 157 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) 158 __u32 page_offset; 159 __u32 size; 160 #else 161 __u16 page_offset; 162 __u16 size; 163 #endif 164 }; 165 166 static inline unsigned int skb_frag_size(const skb_frag_t *frag) 167 { 168 return frag->size; 169 } 170 171 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size) 172 { 173 frag->size = size; 174 } 175 176 static inline void skb_frag_size_add(skb_frag_t *frag, int delta) 177 { 178 frag->size += delta; 179 } 180 181 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta) 182 { 183 frag->size -= delta; 184 } 185 186 #define HAVE_HW_TIME_STAMP 187 188 /** 189 * struct skb_shared_hwtstamps - hardware time stamps 190 * @hwtstamp: hardware time stamp transformed into duration 191 * since arbitrary point in time 192 * @syststamp: hwtstamp transformed to system time base 193 * 194 * Software time stamps generated by ktime_get_real() are stored in 195 * skb->tstamp. The relation between the different kinds of time 196 * stamps is as follows: 197 * 198 * syststamp and tstamp can be compared against each other in 199 * arbitrary combinations. The accuracy of a 200 * syststamp/tstamp/"syststamp from other device" comparison is 201 * limited by the accuracy of the transformation into system time 202 * base. This depends on the device driver and its underlying 203 * hardware. 204 * 205 * hwtstamps can only be compared against other hwtstamps from 206 * the same device. 207 * 208 * This structure is attached to packets as part of the 209 * &skb_shared_info. Use skb_hwtstamps() to get a pointer. 210 */ 211 struct skb_shared_hwtstamps { 212 ktime_t hwtstamp; 213 ktime_t syststamp; 214 }; 215 216 /* Definitions for tx_flags in struct skb_shared_info */ 217 enum { 218 /* generate hardware time stamp */ 219 SKBTX_HW_TSTAMP = 1 << 0, 220 221 /* generate software time stamp */ 222 SKBTX_SW_TSTAMP = 1 << 1, 223 224 /* device driver is going to provide hardware time stamp */ 225 SKBTX_IN_PROGRESS = 1 << 2, 226 227 /* ensure the originating sk reference is available on driver level */ 228 SKBTX_DRV_NEEDS_SK_REF = 1 << 3, 229 230 /* device driver supports TX zero-copy buffers */ 231 SKBTX_DEV_ZEROCOPY = 1 << 4, 232 233 /* generate wifi status information (where possible) */ 234 SKBTX_WIFI_STATUS = 1 << 5, 235 }; 236 237 /* 238 * The callback notifies userspace to release buffers when skb DMA is done in 239 * lower device, the skb last reference should be 0 when calling this. 240 * The desc is used to track userspace buffer index. 241 */ 242 struct ubuf_info { 243 void (*callback)(void *); 244 void *arg; 245 unsigned long desc; 246 }; 247 248 /* This data is invariant across clones and lives at 249 * the end of the header data, ie. at skb->end. 250 */ 251 struct skb_shared_info { 252 unsigned char nr_frags; 253 __u8 tx_flags; 254 unsigned short gso_size; 255 /* Warning: this field is not always filled in (UFO)! */ 256 unsigned short gso_segs; 257 unsigned short gso_type; 258 struct sk_buff *frag_list; 259 struct skb_shared_hwtstamps hwtstamps; 260 __be32 ip6_frag_id; 261 262 /* 263 * Warning : all fields before dataref are cleared in __alloc_skb() 264 */ 265 atomic_t dataref; 266 267 /* Intermediate layers must ensure that destructor_arg 268 * remains valid until skb destructor */ 269 void * destructor_arg; 270 271 /* must be last field, see pskb_expand_head() */ 272 skb_frag_t frags[MAX_SKB_FRAGS]; 273 }; 274 275 /* We divide dataref into two halves. The higher 16 bits hold references 276 * to the payload part of skb->data. The lower 16 bits hold references to 277 * the entire skb->data. A clone of a headerless skb holds the length of 278 * the header in skb->hdr_len. 279 * 280 * All users must obey the rule that the skb->data reference count must be 281 * greater than or equal to the payload reference count. 282 * 283 * Holding a reference to the payload part means that the user does not 284 * care about modifications to the header part of skb->data. 285 */ 286 #define SKB_DATAREF_SHIFT 16 287 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1) 288 289 290 enum { 291 SKB_FCLONE_UNAVAILABLE, 292 SKB_FCLONE_ORIG, 293 SKB_FCLONE_CLONE, 294 }; 295 296 enum { 297 SKB_GSO_TCPV4 = 1 << 0, 298 SKB_GSO_UDP = 1 << 1, 299 300 /* This indicates the skb is from an untrusted source. */ 301 SKB_GSO_DODGY = 1 << 2, 302 303 /* This indicates the tcp segment has CWR set. */ 304 SKB_GSO_TCP_ECN = 1 << 3, 305 306 SKB_GSO_TCPV6 = 1 << 4, 307 308 SKB_GSO_FCOE = 1 << 5, 309 }; 310 311 #if BITS_PER_LONG > 32 312 #define NET_SKBUFF_DATA_USES_OFFSET 1 313 #endif 314 315 #ifdef NET_SKBUFF_DATA_USES_OFFSET 316 typedef unsigned int sk_buff_data_t; 317 #else 318 typedef unsigned char *sk_buff_data_t; 319 #endif 320 321 #if defined(CONFIG_NF_DEFRAG_IPV4) || defined(CONFIG_NF_DEFRAG_IPV4_MODULE) || \ 322 defined(CONFIG_NF_DEFRAG_IPV6) || defined(CONFIG_NF_DEFRAG_IPV6_MODULE) 323 #define NET_SKBUFF_NF_DEFRAG_NEEDED 1 324 #endif 325 326 /** 327 * struct sk_buff - socket buffer 328 * @next: Next buffer in list 329 * @prev: Previous buffer in list 330 * @tstamp: Time we arrived 331 * @sk: Socket we are owned by 332 * @dev: Device we arrived on/are leaving by 333 * @cb: Control buffer. Free for use by every layer. Put private vars here 334 * @_skb_refdst: destination entry (with norefcount bit) 335 * @sp: the security path, used for xfrm 336 * @len: Length of actual data 337 * @data_len: Data length 338 * @mac_len: Length of link layer header 339 * @hdr_len: writable header length of cloned skb 340 * @csum: Checksum (must include start/offset pair) 341 * @csum_start: Offset from skb->head where checksumming should start 342 * @csum_offset: Offset from csum_start where checksum should be stored 343 * @priority: Packet queueing priority 344 * @local_df: allow local fragmentation 345 * @cloned: Head may be cloned (check refcnt to be sure) 346 * @ip_summed: Driver fed us an IP checksum 347 * @nohdr: Payload reference only, must not modify header 348 * @nfctinfo: Relationship of this skb to the connection 349 * @pkt_type: Packet class 350 * @fclone: skbuff clone status 351 * @ipvs_property: skbuff is owned by ipvs 352 * @peeked: this packet has been seen already, so stats have been 353 * done for it, don't do them again 354 * @nf_trace: netfilter packet trace flag 355 * @protocol: Packet protocol from driver 356 * @destructor: Destruct function 357 * @nfct: Associated connection, if any 358 * @nfct_reasm: netfilter conntrack re-assembly pointer 359 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c 360 * @skb_iif: ifindex of device we arrived on 361 * @tc_index: Traffic control index 362 * @tc_verd: traffic control verdict 363 * @rxhash: the packet hash computed on receive 364 * @queue_mapping: Queue mapping for multiqueue devices 365 * @ndisc_nodetype: router type (from link layer) 366 * @ooo_okay: allow the mapping of a socket to a queue to be changed 367 * @l4_rxhash: indicate rxhash is a canonical 4-tuple hash over transport 368 * ports. 369 * @wifi_acked_valid: wifi_acked was set 370 * @wifi_acked: whether frame was acked on wifi or not 371 * @no_fcs: Request NIC to treat last 4 bytes as Ethernet FCS 372 * @dma_cookie: a cookie to one of several possible DMA operations 373 * done by skb DMA functions 374 * @secmark: security marking 375 * @mark: Generic packet mark 376 * @dropcount: total number of sk_receive_queue overflows 377 * @vlan_tci: vlan tag control information 378 * @transport_header: Transport layer header 379 * @network_header: Network layer header 380 * @mac_header: Link layer header 381 * @tail: Tail pointer 382 * @end: End pointer 383 * @head: Head of buffer 384 * @data: Data head pointer 385 * @truesize: Buffer size 386 * @users: User count - see {datagram,tcp}.c 387 */ 388 389 struct sk_buff { 390 /* These two members must be first. */ 391 struct sk_buff *next; 392 struct sk_buff *prev; 393 394 ktime_t tstamp; 395 396 struct sock *sk; 397 struct net_device *dev; 398 399 /* 400 * This is the control buffer. It is free to use for every 401 * layer. Please put your private variables there. If you 402 * want to keep them across layers you have to do a skb_clone() 403 * first. This is owned by whoever has the skb queued ATM. 404 */ 405 char cb[48] __aligned(8); 406 407 unsigned long _skb_refdst; 408 #ifdef CONFIG_XFRM 409 struct sec_path *sp; 410 #endif 411 unsigned int len, 412 data_len; 413 __u16 mac_len, 414 hdr_len; 415 union { 416 __wsum csum; 417 struct { 418 __u16 csum_start; 419 __u16 csum_offset; 420 }; 421 }; 422 __u32 priority; 423 kmemcheck_bitfield_begin(flags1); 424 __u8 local_df:1, 425 cloned:1, 426 ip_summed:2, 427 nohdr:1, 428 nfctinfo:3; 429 __u8 pkt_type:3, 430 fclone:2, 431 ipvs_property:1, 432 peeked:1, 433 nf_trace:1; 434 kmemcheck_bitfield_end(flags1); 435 __be16 protocol; 436 437 void (*destructor)(struct sk_buff *skb); 438 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 439 struct nf_conntrack *nfct; 440 #endif 441 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED 442 struct sk_buff *nfct_reasm; 443 #endif 444 #ifdef CONFIG_BRIDGE_NETFILTER 445 struct nf_bridge_info *nf_bridge; 446 #endif 447 448 int skb_iif; 449 450 __u32 rxhash; 451 452 __u16 vlan_tci; 453 454 #ifdef CONFIG_NET_SCHED 455 __u16 tc_index; /* traffic control index */ 456 #ifdef CONFIG_NET_CLS_ACT 457 __u16 tc_verd; /* traffic control verdict */ 458 #endif 459 #endif 460 461 __u16 queue_mapping; 462 kmemcheck_bitfield_begin(flags2); 463 #ifdef CONFIG_IPV6_NDISC_NODETYPE 464 __u8 ndisc_nodetype:2; 465 #endif 466 __u8 ooo_okay:1; 467 __u8 l4_rxhash:1; 468 __u8 wifi_acked_valid:1; 469 __u8 wifi_acked:1; 470 __u8 no_fcs:1; 471 /* 9/11 bit hole (depending on ndisc_nodetype presence) */ 472 kmemcheck_bitfield_end(flags2); 473 474 #ifdef CONFIG_NET_DMA 475 dma_cookie_t dma_cookie; 476 #endif 477 #ifdef CONFIG_NETWORK_SECMARK 478 __u32 secmark; 479 #endif 480 union { 481 __u32 mark; 482 __u32 dropcount; 483 }; 484 485 sk_buff_data_t transport_header; 486 sk_buff_data_t network_header; 487 sk_buff_data_t mac_header; 488 /* These elements must be at the end, see alloc_skb() for details. */ 489 sk_buff_data_t tail; 490 sk_buff_data_t end; 491 unsigned char *head, 492 *data; 493 unsigned int truesize; 494 atomic_t users; 495 }; 496 497 #ifdef __KERNEL__ 498 /* 499 * Handling routines are only of interest to the kernel 500 */ 501 #include <linux/slab.h> 502 503 #include <asm/system.h> 504 505 /* 506 * skb might have a dst pointer attached, refcounted or not. 507 * _skb_refdst low order bit is set if refcount was _not_ taken 508 */ 509 #define SKB_DST_NOREF 1UL 510 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF) 511 512 /** 513 * skb_dst - returns skb dst_entry 514 * @skb: buffer 515 * 516 * Returns skb dst_entry, regardless of reference taken or not. 517 */ 518 static inline struct dst_entry *skb_dst(const struct sk_buff *skb) 519 { 520 /* If refdst was not refcounted, check we still are in a 521 * rcu_read_lock section 522 */ 523 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) && 524 !rcu_read_lock_held() && 525 !rcu_read_lock_bh_held()); 526 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK); 527 } 528 529 /** 530 * skb_dst_set - sets skb dst 531 * @skb: buffer 532 * @dst: dst entry 533 * 534 * Sets skb dst, assuming a reference was taken on dst and should 535 * be released by skb_dst_drop() 536 */ 537 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst) 538 { 539 skb->_skb_refdst = (unsigned long)dst; 540 } 541 542 extern void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst); 543 544 /** 545 * skb_dst_is_noref - Test if skb dst isn't refcounted 546 * @skb: buffer 547 */ 548 static inline bool skb_dst_is_noref(const struct sk_buff *skb) 549 { 550 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb); 551 } 552 553 static inline struct rtable *skb_rtable(const struct sk_buff *skb) 554 { 555 return (struct rtable *)skb_dst(skb); 556 } 557 558 extern void kfree_skb(struct sk_buff *skb); 559 extern void consume_skb(struct sk_buff *skb); 560 extern void __kfree_skb(struct sk_buff *skb); 561 extern struct sk_buff *__alloc_skb(unsigned int size, 562 gfp_t priority, int fclone, int node); 563 extern struct sk_buff *build_skb(void *data); 564 static inline struct sk_buff *alloc_skb(unsigned int size, 565 gfp_t priority) 566 { 567 return __alloc_skb(size, priority, 0, NUMA_NO_NODE); 568 } 569 570 static inline struct sk_buff *alloc_skb_fclone(unsigned int size, 571 gfp_t priority) 572 { 573 return __alloc_skb(size, priority, 1, NUMA_NO_NODE); 574 } 575 576 extern void skb_recycle(struct sk_buff *skb); 577 extern bool skb_recycle_check(struct sk_buff *skb, int skb_size); 578 579 extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src); 580 extern int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask); 581 extern struct sk_buff *skb_clone(struct sk_buff *skb, 582 gfp_t priority); 583 extern struct sk_buff *skb_copy(const struct sk_buff *skb, 584 gfp_t priority); 585 extern struct sk_buff *__pskb_copy(struct sk_buff *skb, 586 int headroom, gfp_t gfp_mask); 587 588 extern int pskb_expand_head(struct sk_buff *skb, 589 int nhead, int ntail, 590 gfp_t gfp_mask); 591 extern struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, 592 unsigned int headroom); 593 extern struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 594 int newheadroom, int newtailroom, 595 gfp_t priority); 596 extern int skb_to_sgvec(struct sk_buff *skb, 597 struct scatterlist *sg, int offset, 598 int len); 599 extern int skb_cow_data(struct sk_buff *skb, int tailbits, 600 struct sk_buff **trailer); 601 extern int skb_pad(struct sk_buff *skb, int pad); 602 #define dev_kfree_skb(a) consume_skb(a) 603 604 extern int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 605 int getfrag(void *from, char *to, int offset, 606 int len,int odd, struct sk_buff *skb), 607 void *from, int length); 608 609 struct skb_seq_state { 610 __u32 lower_offset; 611 __u32 upper_offset; 612 __u32 frag_idx; 613 __u32 stepped_offset; 614 struct sk_buff *root_skb; 615 struct sk_buff *cur_skb; 616 __u8 *frag_data; 617 }; 618 619 extern void skb_prepare_seq_read(struct sk_buff *skb, 620 unsigned int from, unsigned int to, 621 struct skb_seq_state *st); 622 extern unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 623 struct skb_seq_state *st); 624 extern void skb_abort_seq_read(struct skb_seq_state *st); 625 626 extern unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 627 unsigned int to, struct ts_config *config, 628 struct ts_state *state); 629 630 extern void __skb_get_rxhash(struct sk_buff *skb); 631 static inline __u32 skb_get_rxhash(struct sk_buff *skb) 632 { 633 if (!skb->rxhash) 634 __skb_get_rxhash(skb); 635 636 return skb->rxhash; 637 } 638 639 #ifdef NET_SKBUFF_DATA_USES_OFFSET 640 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) 641 { 642 return skb->head + skb->end; 643 } 644 #else 645 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) 646 { 647 return skb->end; 648 } 649 #endif 650 651 /* Internal */ 652 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB))) 653 654 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) 655 { 656 return &skb_shinfo(skb)->hwtstamps; 657 } 658 659 /** 660 * skb_queue_empty - check if a queue is empty 661 * @list: queue head 662 * 663 * Returns true if the queue is empty, false otherwise. 664 */ 665 static inline int skb_queue_empty(const struct sk_buff_head *list) 666 { 667 return list->next == (struct sk_buff *)list; 668 } 669 670 /** 671 * skb_queue_is_last - check if skb is the last entry in the queue 672 * @list: queue head 673 * @skb: buffer 674 * 675 * Returns true if @skb is the last buffer on the list. 676 */ 677 static inline bool skb_queue_is_last(const struct sk_buff_head *list, 678 const struct sk_buff *skb) 679 { 680 return skb->next == (struct sk_buff *)list; 681 } 682 683 /** 684 * skb_queue_is_first - check if skb is the first entry in the queue 685 * @list: queue head 686 * @skb: buffer 687 * 688 * Returns true if @skb is the first buffer on the list. 689 */ 690 static inline bool skb_queue_is_first(const struct sk_buff_head *list, 691 const struct sk_buff *skb) 692 { 693 return skb->prev == (struct sk_buff *)list; 694 } 695 696 /** 697 * skb_queue_next - return the next packet in the queue 698 * @list: queue head 699 * @skb: current buffer 700 * 701 * Return the next packet in @list after @skb. It is only valid to 702 * call this if skb_queue_is_last() evaluates to false. 703 */ 704 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list, 705 const struct sk_buff *skb) 706 { 707 /* This BUG_ON may seem severe, but if we just return then we 708 * are going to dereference garbage. 709 */ 710 BUG_ON(skb_queue_is_last(list, skb)); 711 return skb->next; 712 } 713 714 /** 715 * skb_queue_prev - return the prev packet in the queue 716 * @list: queue head 717 * @skb: current buffer 718 * 719 * Return the prev packet in @list before @skb. It is only valid to 720 * call this if skb_queue_is_first() evaluates to false. 721 */ 722 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list, 723 const struct sk_buff *skb) 724 { 725 /* This BUG_ON may seem severe, but if we just return then we 726 * are going to dereference garbage. 727 */ 728 BUG_ON(skb_queue_is_first(list, skb)); 729 return skb->prev; 730 } 731 732 /** 733 * skb_get - reference buffer 734 * @skb: buffer to reference 735 * 736 * Makes another reference to a socket buffer and returns a pointer 737 * to the buffer. 738 */ 739 static inline struct sk_buff *skb_get(struct sk_buff *skb) 740 { 741 atomic_inc(&skb->users); 742 return skb; 743 } 744 745 /* 746 * If users == 1, we are the only owner and are can avoid redundant 747 * atomic change. 748 */ 749 750 /** 751 * skb_cloned - is the buffer a clone 752 * @skb: buffer to check 753 * 754 * Returns true if the buffer was generated with skb_clone() and is 755 * one of multiple shared copies of the buffer. Cloned buffers are 756 * shared data so must not be written to under normal circumstances. 757 */ 758 static inline int skb_cloned(const struct sk_buff *skb) 759 { 760 return skb->cloned && 761 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1; 762 } 763 764 /** 765 * skb_header_cloned - is the header a clone 766 * @skb: buffer to check 767 * 768 * Returns true if modifying the header part of the buffer requires 769 * the data to be copied. 770 */ 771 static inline int skb_header_cloned(const struct sk_buff *skb) 772 { 773 int dataref; 774 775 if (!skb->cloned) 776 return 0; 777 778 dataref = atomic_read(&skb_shinfo(skb)->dataref); 779 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT); 780 return dataref != 1; 781 } 782 783 /** 784 * skb_header_release - release reference to header 785 * @skb: buffer to operate on 786 * 787 * Drop a reference to the header part of the buffer. This is done 788 * by acquiring a payload reference. You must not read from the header 789 * part of skb->data after this. 790 */ 791 static inline void skb_header_release(struct sk_buff *skb) 792 { 793 BUG_ON(skb->nohdr); 794 skb->nohdr = 1; 795 atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref); 796 } 797 798 /** 799 * skb_shared - is the buffer shared 800 * @skb: buffer to check 801 * 802 * Returns true if more than one person has a reference to this 803 * buffer. 804 */ 805 static inline int skb_shared(const struct sk_buff *skb) 806 { 807 return atomic_read(&skb->users) != 1; 808 } 809 810 /** 811 * skb_share_check - check if buffer is shared and if so clone it 812 * @skb: buffer to check 813 * @pri: priority for memory allocation 814 * 815 * If the buffer is shared the buffer is cloned and the old copy 816 * drops a reference. A new clone with a single reference is returned. 817 * If the buffer is not shared the original buffer is returned. When 818 * being called from interrupt status or with spinlocks held pri must 819 * be GFP_ATOMIC. 820 * 821 * NULL is returned on a memory allocation failure. 822 */ 823 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, 824 gfp_t pri) 825 { 826 might_sleep_if(pri & __GFP_WAIT); 827 if (skb_shared(skb)) { 828 struct sk_buff *nskb = skb_clone(skb, pri); 829 kfree_skb(skb); 830 skb = nskb; 831 } 832 return skb; 833 } 834 835 /* 836 * Copy shared buffers into a new sk_buff. We effectively do COW on 837 * packets to handle cases where we have a local reader and forward 838 * and a couple of other messy ones. The normal one is tcpdumping 839 * a packet thats being forwarded. 840 */ 841 842 /** 843 * skb_unshare - make a copy of a shared buffer 844 * @skb: buffer to check 845 * @pri: priority for memory allocation 846 * 847 * If the socket buffer is a clone then this function creates a new 848 * copy of the data, drops a reference count on the old copy and returns 849 * the new copy with the reference count at 1. If the buffer is not a clone 850 * the original buffer is returned. When called with a spinlock held or 851 * from interrupt state @pri must be %GFP_ATOMIC 852 * 853 * %NULL is returned on a memory allocation failure. 854 */ 855 static inline struct sk_buff *skb_unshare(struct sk_buff *skb, 856 gfp_t pri) 857 { 858 might_sleep_if(pri & __GFP_WAIT); 859 if (skb_cloned(skb)) { 860 struct sk_buff *nskb = skb_copy(skb, pri); 861 kfree_skb(skb); /* Free our shared copy */ 862 skb = nskb; 863 } 864 return skb; 865 } 866 867 /** 868 * skb_peek - peek at the head of an &sk_buff_head 869 * @list_: list to peek at 870 * 871 * Peek an &sk_buff. Unlike most other operations you _MUST_ 872 * be careful with this one. A peek leaves the buffer on the 873 * list and someone else may run off with it. You must hold 874 * the appropriate locks or have a private queue to do this. 875 * 876 * Returns %NULL for an empty list or a pointer to the head element. 877 * The reference count is not incremented and the reference is therefore 878 * volatile. Use with caution. 879 */ 880 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_) 881 { 882 struct sk_buff *list = ((const struct sk_buff *)list_)->next; 883 if (list == (struct sk_buff *)list_) 884 list = NULL; 885 return list; 886 } 887 888 /** 889 * skb_peek_next - peek skb following the given one from a queue 890 * @skb: skb to start from 891 * @list_: list to peek at 892 * 893 * Returns %NULL when the end of the list is met or a pointer to the 894 * next element. The reference count is not incremented and the 895 * reference is therefore volatile. Use with caution. 896 */ 897 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb, 898 const struct sk_buff_head *list_) 899 { 900 struct sk_buff *next = skb->next; 901 if (next == (struct sk_buff *)list_) 902 next = NULL; 903 return next; 904 } 905 906 /** 907 * skb_peek_tail - peek at the tail of an &sk_buff_head 908 * @list_: list to peek at 909 * 910 * Peek an &sk_buff. Unlike most other operations you _MUST_ 911 * be careful with this one. A peek leaves the buffer on the 912 * list and someone else may run off with it. You must hold 913 * the appropriate locks or have a private queue to do this. 914 * 915 * Returns %NULL for an empty list or a pointer to the tail element. 916 * The reference count is not incremented and the reference is therefore 917 * volatile. Use with caution. 918 */ 919 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_) 920 { 921 struct sk_buff *list = ((const struct sk_buff *)list_)->prev; 922 if (list == (struct sk_buff *)list_) 923 list = NULL; 924 return list; 925 } 926 927 /** 928 * skb_queue_len - get queue length 929 * @list_: list to measure 930 * 931 * Return the length of an &sk_buff queue. 932 */ 933 static inline __u32 skb_queue_len(const struct sk_buff_head *list_) 934 { 935 return list_->qlen; 936 } 937 938 /** 939 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head 940 * @list: queue to initialize 941 * 942 * This initializes only the list and queue length aspects of 943 * an sk_buff_head object. This allows to initialize the list 944 * aspects of an sk_buff_head without reinitializing things like 945 * the spinlock. It can also be used for on-stack sk_buff_head 946 * objects where the spinlock is known to not be used. 947 */ 948 static inline void __skb_queue_head_init(struct sk_buff_head *list) 949 { 950 list->prev = list->next = (struct sk_buff *)list; 951 list->qlen = 0; 952 } 953 954 /* 955 * This function creates a split out lock class for each invocation; 956 * this is needed for now since a whole lot of users of the skb-queue 957 * infrastructure in drivers have different locking usage (in hardirq) 958 * than the networking core (in softirq only). In the long run either the 959 * network layer or drivers should need annotation to consolidate the 960 * main types of usage into 3 classes. 961 */ 962 static inline void skb_queue_head_init(struct sk_buff_head *list) 963 { 964 spin_lock_init(&list->lock); 965 __skb_queue_head_init(list); 966 } 967 968 static inline void skb_queue_head_init_class(struct sk_buff_head *list, 969 struct lock_class_key *class) 970 { 971 skb_queue_head_init(list); 972 lockdep_set_class(&list->lock, class); 973 } 974 975 /* 976 * Insert an sk_buff on a list. 977 * 978 * The "__skb_xxxx()" functions are the non-atomic ones that 979 * can only be called with interrupts disabled. 980 */ 981 extern void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list); 982 static inline void __skb_insert(struct sk_buff *newsk, 983 struct sk_buff *prev, struct sk_buff *next, 984 struct sk_buff_head *list) 985 { 986 newsk->next = next; 987 newsk->prev = prev; 988 next->prev = prev->next = newsk; 989 list->qlen++; 990 } 991 992 static inline void __skb_queue_splice(const struct sk_buff_head *list, 993 struct sk_buff *prev, 994 struct sk_buff *next) 995 { 996 struct sk_buff *first = list->next; 997 struct sk_buff *last = list->prev; 998 999 first->prev = prev; 1000 prev->next = first; 1001 1002 last->next = next; 1003 next->prev = last; 1004 } 1005 1006 /** 1007 * skb_queue_splice - join two skb lists, this is designed for stacks 1008 * @list: the new list to add 1009 * @head: the place to add it in the first list 1010 */ 1011 static inline void skb_queue_splice(const struct sk_buff_head *list, 1012 struct sk_buff_head *head) 1013 { 1014 if (!skb_queue_empty(list)) { 1015 __skb_queue_splice(list, (struct sk_buff *) head, head->next); 1016 head->qlen += list->qlen; 1017 } 1018 } 1019 1020 /** 1021 * skb_queue_splice - join two skb lists and reinitialise the emptied list 1022 * @list: the new list to add 1023 * @head: the place to add it in the first list 1024 * 1025 * The list at @list is reinitialised 1026 */ 1027 static inline void skb_queue_splice_init(struct sk_buff_head *list, 1028 struct sk_buff_head *head) 1029 { 1030 if (!skb_queue_empty(list)) { 1031 __skb_queue_splice(list, (struct sk_buff *) head, head->next); 1032 head->qlen += list->qlen; 1033 __skb_queue_head_init(list); 1034 } 1035 } 1036 1037 /** 1038 * skb_queue_splice_tail - join two skb lists, each list being a queue 1039 * @list: the new list to add 1040 * @head: the place to add it in the first list 1041 */ 1042 static inline void skb_queue_splice_tail(const struct sk_buff_head *list, 1043 struct sk_buff_head *head) 1044 { 1045 if (!skb_queue_empty(list)) { 1046 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); 1047 head->qlen += list->qlen; 1048 } 1049 } 1050 1051 /** 1052 * skb_queue_splice_tail - join two skb lists and reinitialise the emptied list 1053 * @list: the new list to add 1054 * @head: the place to add it in the first list 1055 * 1056 * Each of the lists is a queue. 1057 * The list at @list is reinitialised 1058 */ 1059 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list, 1060 struct sk_buff_head *head) 1061 { 1062 if (!skb_queue_empty(list)) { 1063 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); 1064 head->qlen += list->qlen; 1065 __skb_queue_head_init(list); 1066 } 1067 } 1068 1069 /** 1070 * __skb_queue_after - queue a buffer at the list head 1071 * @list: list to use 1072 * @prev: place after this buffer 1073 * @newsk: buffer to queue 1074 * 1075 * Queue a buffer int the middle of a list. This function takes no locks 1076 * and you must therefore hold required locks before calling it. 1077 * 1078 * A buffer cannot be placed on two lists at the same time. 1079 */ 1080 static inline void __skb_queue_after(struct sk_buff_head *list, 1081 struct sk_buff *prev, 1082 struct sk_buff *newsk) 1083 { 1084 __skb_insert(newsk, prev, prev->next, list); 1085 } 1086 1087 extern void skb_append(struct sk_buff *old, struct sk_buff *newsk, 1088 struct sk_buff_head *list); 1089 1090 static inline void __skb_queue_before(struct sk_buff_head *list, 1091 struct sk_buff *next, 1092 struct sk_buff *newsk) 1093 { 1094 __skb_insert(newsk, next->prev, next, list); 1095 } 1096 1097 /** 1098 * __skb_queue_head - queue a buffer at the list head 1099 * @list: list to use 1100 * @newsk: buffer to queue 1101 * 1102 * Queue a buffer at the start of a list. This function takes no locks 1103 * and you must therefore hold required locks before calling it. 1104 * 1105 * A buffer cannot be placed on two lists at the same time. 1106 */ 1107 extern void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk); 1108 static inline void __skb_queue_head(struct sk_buff_head *list, 1109 struct sk_buff *newsk) 1110 { 1111 __skb_queue_after(list, (struct sk_buff *)list, newsk); 1112 } 1113 1114 /** 1115 * __skb_queue_tail - queue a buffer at the list tail 1116 * @list: list to use 1117 * @newsk: buffer to queue 1118 * 1119 * Queue a buffer at the end of a list. This function takes no locks 1120 * and you must therefore hold required locks before calling it. 1121 * 1122 * A buffer cannot be placed on two lists at the same time. 1123 */ 1124 extern void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk); 1125 static inline void __skb_queue_tail(struct sk_buff_head *list, 1126 struct sk_buff *newsk) 1127 { 1128 __skb_queue_before(list, (struct sk_buff *)list, newsk); 1129 } 1130 1131 /* 1132 * remove sk_buff from list. _Must_ be called atomically, and with 1133 * the list known.. 1134 */ 1135 extern void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list); 1136 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 1137 { 1138 struct sk_buff *next, *prev; 1139 1140 list->qlen--; 1141 next = skb->next; 1142 prev = skb->prev; 1143 skb->next = skb->prev = NULL; 1144 next->prev = prev; 1145 prev->next = next; 1146 } 1147 1148 /** 1149 * __skb_dequeue - remove from the head of the queue 1150 * @list: list to dequeue from 1151 * 1152 * Remove the head of the list. This function does not take any locks 1153 * so must be used with appropriate locks held only. The head item is 1154 * returned or %NULL if the list is empty. 1155 */ 1156 extern struct sk_buff *skb_dequeue(struct sk_buff_head *list); 1157 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list) 1158 { 1159 struct sk_buff *skb = skb_peek(list); 1160 if (skb) 1161 __skb_unlink(skb, list); 1162 return skb; 1163 } 1164 1165 /** 1166 * __skb_dequeue_tail - remove from the tail of the queue 1167 * @list: list to dequeue from 1168 * 1169 * Remove the tail of the list. This function does not take any locks 1170 * so must be used with appropriate locks held only. The tail item is 1171 * returned or %NULL if the list is empty. 1172 */ 1173 extern struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list); 1174 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list) 1175 { 1176 struct sk_buff *skb = skb_peek_tail(list); 1177 if (skb) 1178 __skb_unlink(skb, list); 1179 return skb; 1180 } 1181 1182 1183 static inline bool skb_is_nonlinear(const struct sk_buff *skb) 1184 { 1185 return skb->data_len; 1186 } 1187 1188 static inline unsigned int skb_headlen(const struct sk_buff *skb) 1189 { 1190 return skb->len - skb->data_len; 1191 } 1192 1193 static inline int skb_pagelen(const struct sk_buff *skb) 1194 { 1195 int i, len = 0; 1196 1197 for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--) 1198 len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 1199 return len + skb_headlen(skb); 1200 } 1201 1202 /** 1203 * __skb_fill_page_desc - initialise a paged fragment in an skb 1204 * @skb: buffer containing fragment to be initialised 1205 * @i: paged fragment index to initialise 1206 * @page: the page to use for this fragment 1207 * @off: the offset to the data with @page 1208 * @size: the length of the data 1209 * 1210 * Initialises the @i'th fragment of @skb to point to &size bytes at 1211 * offset @off within @page. 1212 * 1213 * Does not take any additional reference on the fragment. 1214 */ 1215 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, 1216 struct page *page, int off, int size) 1217 { 1218 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1219 1220 frag->page.p = page; 1221 frag->page_offset = off; 1222 skb_frag_size_set(frag, size); 1223 } 1224 1225 /** 1226 * skb_fill_page_desc - initialise a paged fragment in an skb 1227 * @skb: buffer containing fragment to be initialised 1228 * @i: paged fragment index to initialise 1229 * @page: the page to use for this fragment 1230 * @off: the offset to the data with @page 1231 * @size: the length of the data 1232 * 1233 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of 1234 * @skb to point to &size bytes at offset @off within @page. In 1235 * addition updates @skb such that @i is the last fragment. 1236 * 1237 * Does not take any additional reference on the fragment. 1238 */ 1239 static inline void skb_fill_page_desc(struct sk_buff *skb, int i, 1240 struct page *page, int off, int size) 1241 { 1242 __skb_fill_page_desc(skb, i, page, off, size); 1243 skb_shinfo(skb)->nr_frags = i + 1; 1244 } 1245 1246 extern void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, 1247 int off, int size); 1248 1249 #define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags) 1250 #define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb)) 1251 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb)) 1252 1253 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1254 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) 1255 { 1256 return skb->head + skb->tail; 1257 } 1258 1259 static inline void skb_reset_tail_pointer(struct sk_buff *skb) 1260 { 1261 skb->tail = skb->data - skb->head; 1262 } 1263 1264 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) 1265 { 1266 skb_reset_tail_pointer(skb); 1267 skb->tail += offset; 1268 } 1269 #else /* NET_SKBUFF_DATA_USES_OFFSET */ 1270 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) 1271 { 1272 return skb->tail; 1273 } 1274 1275 static inline void skb_reset_tail_pointer(struct sk_buff *skb) 1276 { 1277 skb->tail = skb->data; 1278 } 1279 1280 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) 1281 { 1282 skb->tail = skb->data + offset; 1283 } 1284 1285 #endif /* NET_SKBUFF_DATA_USES_OFFSET */ 1286 1287 /* 1288 * Add data to an sk_buff 1289 */ 1290 extern unsigned char *skb_put(struct sk_buff *skb, unsigned int len); 1291 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len) 1292 { 1293 unsigned char *tmp = skb_tail_pointer(skb); 1294 SKB_LINEAR_ASSERT(skb); 1295 skb->tail += len; 1296 skb->len += len; 1297 return tmp; 1298 } 1299 1300 extern unsigned char *skb_push(struct sk_buff *skb, unsigned int len); 1301 static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len) 1302 { 1303 skb->data -= len; 1304 skb->len += len; 1305 return skb->data; 1306 } 1307 1308 extern unsigned char *skb_pull(struct sk_buff *skb, unsigned int len); 1309 static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len) 1310 { 1311 skb->len -= len; 1312 BUG_ON(skb->len < skb->data_len); 1313 return skb->data += len; 1314 } 1315 1316 static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len) 1317 { 1318 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); 1319 } 1320 1321 extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta); 1322 1323 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len) 1324 { 1325 if (len > skb_headlen(skb) && 1326 !__pskb_pull_tail(skb, len - skb_headlen(skb))) 1327 return NULL; 1328 skb->len -= len; 1329 return skb->data += len; 1330 } 1331 1332 static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len) 1333 { 1334 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len); 1335 } 1336 1337 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len) 1338 { 1339 if (likely(len <= skb_headlen(skb))) 1340 return 1; 1341 if (unlikely(len > skb->len)) 1342 return 0; 1343 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; 1344 } 1345 1346 /** 1347 * skb_headroom - bytes at buffer head 1348 * @skb: buffer to check 1349 * 1350 * Return the number of bytes of free space at the head of an &sk_buff. 1351 */ 1352 static inline unsigned int skb_headroom(const struct sk_buff *skb) 1353 { 1354 return skb->data - skb->head; 1355 } 1356 1357 /** 1358 * skb_tailroom - bytes at buffer end 1359 * @skb: buffer to check 1360 * 1361 * Return the number of bytes of free space at the tail of an sk_buff 1362 */ 1363 static inline int skb_tailroom(const struct sk_buff *skb) 1364 { 1365 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail; 1366 } 1367 1368 /** 1369 * skb_reserve - adjust headroom 1370 * @skb: buffer to alter 1371 * @len: bytes to move 1372 * 1373 * Increase the headroom of an empty &sk_buff by reducing the tail 1374 * room. This is only allowed for an empty buffer. 1375 */ 1376 static inline void skb_reserve(struct sk_buff *skb, int len) 1377 { 1378 skb->data += len; 1379 skb->tail += len; 1380 } 1381 1382 static inline void skb_reset_mac_len(struct sk_buff *skb) 1383 { 1384 skb->mac_len = skb->network_header - skb->mac_header; 1385 } 1386 1387 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1388 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) 1389 { 1390 return skb->head + skb->transport_header; 1391 } 1392 1393 static inline void skb_reset_transport_header(struct sk_buff *skb) 1394 { 1395 skb->transport_header = skb->data - skb->head; 1396 } 1397 1398 static inline void skb_set_transport_header(struct sk_buff *skb, 1399 const int offset) 1400 { 1401 skb_reset_transport_header(skb); 1402 skb->transport_header += offset; 1403 } 1404 1405 static inline unsigned char *skb_network_header(const struct sk_buff *skb) 1406 { 1407 return skb->head + skb->network_header; 1408 } 1409 1410 static inline void skb_reset_network_header(struct sk_buff *skb) 1411 { 1412 skb->network_header = skb->data - skb->head; 1413 } 1414 1415 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) 1416 { 1417 skb_reset_network_header(skb); 1418 skb->network_header += offset; 1419 } 1420 1421 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) 1422 { 1423 return skb->head + skb->mac_header; 1424 } 1425 1426 static inline int skb_mac_header_was_set(const struct sk_buff *skb) 1427 { 1428 return skb->mac_header != ~0U; 1429 } 1430 1431 static inline void skb_reset_mac_header(struct sk_buff *skb) 1432 { 1433 skb->mac_header = skb->data - skb->head; 1434 } 1435 1436 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) 1437 { 1438 skb_reset_mac_header(skb); 1439 skb->mac_header += offset; 1440 } 1441 1442 #else /* NET_SKBUFF_DATA_USES_OFFSET */ 1443 1444 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) 1445 { 1446 return skb->transport_header; 1447 } 1448 1449 static inline void skb_reset_transport_header(struct sk_buff *skb) 1450 { 1451 skb->transport_header = skb->data; 1452 } 1453 1454 static inline void skb_set_transport_header(struct sk_buff *skb, 1455 const int offset) 1456 { 1457 skb->transport_header = skb->data + offset; 1458 } 1459 1460 static inline unsigned char *skb_network_header(const struct sk_buff *skb) 1461 { 1462 return skb->network_header; 1463 } 1464 1465 static inline void skb_reset_network_header(struct sk_buff *skb) 1466 { 1467 skb->network_header = skb->data; 1468 } 1469 1470 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) 1471 { 1472 skb->network_header = skb->data + offset; 1473 } 1474 1475 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) 1476 { 1477 return skb->mac_header; 1478 } 1479 1480 static inline int skb_mac_header_was_set(const struct sk_buff *skb) 1481 { 1482 return skb->mac_header != NULL; 1483 } 1484 1485 static inline void skb_reset_mac_header(struct sk_buff *skb) 1486 { 1487 skb->mac_header = skb->data; 1488 } 1489 1490 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) 1491 { 1492 skb->mac_header = skb->data + offset; 1493 } 1494 #endif /* NET_SKBUFF_DATA_USES_OFFSET */ 1495 1496 static inline void skb_mac_header_rebuild(struct sk_buff *skb) 1497 { 1498 if (skb_mac_header_was_set(skb)) { 1499 const unsigned char *old_mac = skb_mac_header(skb); 1500 1501 skb_set_mac_header(skb, -skb->mac_len); 1502 memmove(skb_mac_header(skb), old_mac, skb->mac_len); 1503 } 1504 } 1505 1506 static inline int skb_checksum_start_offset(const struct sk_buff *skb) 1507 { 1508 return skb->csum_start - skb_headroom(skb); 1509 } 1510 1511 static inline int skb_transport_offset(const struct sk_buff *skb) 1512 { 1513 return skb_transport_header(skb) - skb->data; 1514 } 1515 1516 static inline u32 skb_network_header_len(const struct sk_buff *skb) 1517 { 1518 return skb->transport_header - skb->network_header; 1519 } 1520 1521 static inline int skb_network_offset(const struct sk_buff *skb) 1522 { 1523 return skb_network_header(skb) - skb->data; 1524 } 1525 1526 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len) 1527 { 1528 return pskb_may_pull(skb, skb_network_offset(skb) + len); 1529 } 1530 1531 /* 1532 * CPUs often take a performance hit when accessing unaligned memory 1533 * locations. The actual performance hit varies, it can be small if the 1534 * hardware handles it or large if we have to take an exception and fix it 1535 * in software. 1536 * 1537 * Since an ethernet header is 14 bytes network drivers often end up with 1538 * the IP header at an unaligned offset. The IP header can be aligned by 1539 * shifting the start of the packet by 2 bytes. Drivers should do this 1540 * with: 1541 * 1542 * skb_reserve(skb, NET_IP_ALIGN); 1543 * 1544 * The downside to this alignment of the IP header is that the DMA is now 1545 * unaligned. On some architectures the cost of an unaligned DMA is high 1546 * and this cost outweighs the gains made by aligning the IP header. 1547 * 1548 * Since this trade off varies between architectures, we allow NET_IP_ALIGN 1549 * to be overridden. 1550 */ 1551 #ifndef NET_IP_ALIGN 1552 #define NET_IP_ALIGN 2 1553 #endif 1554 1555 /* 1556 * The networking layer reserves some headroom in skb data (via 1557 * dev_alloc_skb). This is used to avoid having to reallocate skb data when 1558 * the header has to grow. In the default case, if the header has to grow 1559 * 32 bytes or less we avoid the reallocation. 1560 * 1561 * Unfortunately this headroom changes the DMA alignment of the resulting 1562 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive 1563 * on some architectures. An architecture can override this value, 1564 * perhaps setting it to a cacheline in size (since that will maintain 1565 * cacheline alignment of the DMA). It must be a power of 2. 1566 * 1567 * Various parts of the networking layer expect at least 32 bytes of 1568 * headroom, you should not reduce this. 1569 * 1570 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS) 1571 * to reduce average number of cache lines per packet. 1572 * get_rps_cpus() for example only access one 64 bytes aligned block : 1573 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8) 1574 */ 1575 #ifndef NET_SKB_PAD 1576 #define NET_SKB_PAD max(32, L1_CACHE_BYTES) 1577 #endif 1578 1579 extern int ___pskb_trim(struct sk_buff *skb, unsigned int len); 1580 1581 static inline void __skb_trim(struct sk_buff *skb, unsigned int len) 1582 { 1583 if (unlikely(skb_is_nonlinear(skb))) { 1584 WARN_ON(1); 1585 return; 1586 } 1587 skb->len = len; 1588 skb_set_tail_pointer(skb, len); 1589 } 1590 1591 extern void skb_trim(struct sk_buff *skb, unsigned int len); 1592 1593 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) 1594 { 1595 if (skb->data_len) 1596 return ___pskb_trim(skb, len); 1597 __skb_trim(skb, len); 1598 return 0; 1599 } 1600 1601 static inline int pskb_trim(struct sk_buff *skb, unsigned int len) 1602 { 1603 return (len < skb->len) ? __pskb_trim(skb, len) : 0; 1604 } 1605 1606 /** 1607 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer 1608 * @skb: buffer to alter 1609 * @len: new length 1610 * 1611 * This is identical to pskb_trim except that the caller knows that 1612 * the skb is not cloned so we should never get an error due to out- 1613 * of-memory. 1614 */ 1615 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len) 1616 { 1617 int err = pskb_trim(skb, len); 1618 BUG_ON(err); 1619 } 1620 1621 /** 1622 * skb_orphan - orphan a buffer 1623 * @skb: buffer to orphan 1624 * 1625 * If a buffer currently has an owner then we call the owner's 1626 * destructor function and make the @skb unowned. The buffer continues 1627 * to exist but is no longer charged to its former owner. 1628 */ 1629 static inline void skb_orphan(struct sk_buff *skb) 1630 { 1631 if (skb->destructor) 1632 skb->destructor(skb); 1633 skb->destructor = NULL; 1634 skb->sk = NULL; 1635 } 1636 1637 /** 1638 * __skb_queue_purge - empty a list 1639 * @list: list to empty 1640 * 1641 * Delete all buffers on an &sk_buff list. Each buffer is removed from 1642 * the list and one reference dropped. This function does not take the 1643 * list lock and the caller must hold the relevant locks to use it. 1644 */ 1645 extern void skb_queue_purge(struct sk_buff_head *list); 1646 static inline void __skb_queue_purge(struct sk_buff_head *list) 1647 { 1648 struct sk_buff *skb; 1649 while ((skb = __skb_dequeue(list)) != NULL) 1650 kfree_skb(skb); 1651 } 1652 1653 /** 1654 * __dev_alloc_skb - allocate an skbuff for receiving 1655 * @length: length to allocate 1656 * @gfp_mask: get_free_pages mask, passed to alloc_skb 1657 * 1658 * Allocate a new &sk_buff and assign it a usage count of one. The 1659 * buffer has unspecified headroom built in. Users should allocate 1660 * the headroom they think they need without accounting for the 1661 * built in space. The built in space is used for optimisations. 1662 * 1663 * %NULL is returned if there is no free memory. 1664 */ 1665 static inline struct sk_buff *__dev_alloc_skb(unsigned int length, 1666 gfp_t gfp_mask) 1667 { 1668 struct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask); 1669 if (likely(skb)) 1670 skb_reserve(skb, NET_SKB_PAD); 1671 return skb; 1672 } 1673 1674 extern struct sk_buff *dev_alloc_skb(unsigned int length); 1675 1676 extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 1677 unsigned int length, gfp_t gfp_mask); 1678 1679 /** 1680 * netdev_alloc_skb - allocate an skbuff for rx on a specific device 1681 * @dev: network device to receive on 1682 * @length: length to allocate 1683 * 1684 * Allocate a new &sk_buff and assign it a usage count of one. The 1685 * buffer has unspecified headroom built in. Users should allocate 1686 * the headroom they think they need without accounting for the 1687 * built in space. The built in space is used for optimisations. 1688 * 1689 * %NULL is returned if there is no free memory. Although this function 1690 * allocates memory it can be called from an interrupt. 1691 */ 1692 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev, 1693 unsigned int length) 1694 { 1695 return __netdev_alloc_skb(dev, length, GFP_ATOMIC); 1696 } 1697 1698 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev, 1699 unsigned int length, gfp_t gfp) 1700 { 1701 struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp); 1702 1703 if (NET_IP_ALIGN && skb) 1704 skb_reserve(skb, NET_IP_ALIGN); 1705 return skb; 1706 } 1707 1708 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, 1709 unsigned int length) 1710 { 1711 return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC); 1712 } 1713 1714 /** 1715 * skb_frag_page - retrieve the page refered to by a paged fragment 1716 * @frag: the paged fragment 1717 * 1718 * Returns the &struct page associated with @frag. 1719 */ 1720 static inline struct page *skb_frag_page(const skb_frag_t *frag) 1721 { 1722 return frag->page.p; 1723 } 1724 1725 /** 1726 * __skb_frag_ref - take an addition reference on a paged fragment. 1727 * @frag: the paged fragment 1728 * 1729 * Takes an additional reference on the paged fragment @frag. 1730 */ 1731 static inline void __skb_frag_ref(skb_frag_t *frag) 1732 { 1733 get_page(skb_frag_page(frag)); 1734 } 1735 1736 /** 1737 * skb_frag_ref - take an addition reference on a paged fragment of an skb. 1738 * @skb: the buffer 1739 * @f: the fragment offset. 1740 * 1741 * Takes an additional reference on the @f'th paged fragment of @skb. 1742 */ 1743 static inline void skb_frag_ref(struct sk_buff *skb, int f) 1744 { 1745 __skb_frag_ref(&skb_shinfo(skb)->frags[f]); 1746 } 1747 1748 /** 1749 * __skb_frag_unref - release a reference on a paged fragment. 1750 * @frag: the paged fragment 1751 * 1752 * Releases a reference on the paged fragment @frag. 1753 */ 1754 static inline void __skb_frag_unref(skb_frag_t *frag) 1755 { 1756 put_page(skb_frag_page(frag)); 1757 } 1758 1759 /** 1760 * skb_frag_unref - release a reference on a paged fragment of an skb. 1761 * @skb: the buffer 1762 * @f: the fragment offset 1763 * 1764 * Releases a reference on the @f'th paged fragment of @skb. 1765 */ 1766 static inline void skb_frag_unref(struct sk_buff *skb, int f) 1767 { 1768 __skb_frag_unref(&skb_shinfo(skb)->frags[f]); 1769 } 1770 1771 /** 1772 * skb_frag_address - gets the address of the data contained in a paged fragment 1773 * @frag: the paged fragment buffer 1774 * 1775 * Returns the address of the data within @frag. The page must already 1776 * be mapped. 1777 */ 1778 static inline void *skb_frag_address(const skb_frag_t *frag) 1779 { 1780 return page_address(skb_frag_page(frag)) + frag->page_offset; 1781 } 1782 1783 /** 1784 * skb_frag_address_safe - gets the address of the data contained in a paged fragment 1785 * @frag: the paged fragment buffer 1786 * 1787 * Returns the address of the data within @frag. Checks that the page 1788 * is mapped and returns %NULL otherwise. 1789 */ 1790 static inline void *skb_frag_address_safe(const skb_frag_t *frag) 1791 { 1792 void *ptr = page_address(skb_frag_page(frag)); 1793 if (unlikely(!ptr)) 1794 return NULL; 1795 1796 return ptr + frag->page_offset; 1797 } 1798 1799 /** 1800 * __skb_frag_set_page - sets the page contained in a paged fragment 1801 * @frag: the paged fragment 1802 * @page: the page to set 1803 * 1804 * Sets the fragment @frag to contain @page. 1805 */ 1806 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page) 1807 { 1808 frag->page.p = page; 1809 } 1810 1811 /** 1812 * skb_frag_set_page - sets the page contained in a paged fragment of an skb 1813 * @skb: the buffer 1814 * @f: the fragment offset 1815 * @page: the page to set 1816 * 1817 * Sets the @f'th fragment of @skb to contain @page. 1818 */ 1819 static inline void skb_frag_set_page(struct sk_buff *skb, int f, 1820 struct page *page) 1821 { 1822 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page); 1823 } 1824 1825 /** 1826 * skb_frag_dma_map - maps a paged fragment via the DMA API 1827 * @dev: the device to map the fragment to 1828 * @frag: the paged fragment to map 1829 * @offset: the offset within the fragment (starting at the 1830 * fragment's own offset) 1831 * @size: the number of bytes to map 1832 * @dir: the direction of the mapping (%PCI_DMA_*) 1833 * 1834 * Maps the page associated with @frag to @device. 1835 */ 1836 static inline dma_addr_t skb_frag_dma_map(struct device *dev, 1837 const skb_frag_t *frag, 1838 size_t offset, size_t size, 1839 enum dma_data_direction dir) 1840 { 1841 return dma_map_page(dev, skb_frag_page(frag), 1842 frag->page_offset + offset, size, dir); 1843 } 1844 1845 static inline struct sk_buff *pskb_copy(struct sk_buff *skb, 1846 gfp_t gfp_mask) 1847 { 1848 return __pskb_copy(skb, skb_headroom(skb), gfp_mask); 1849 } 1850 1851 /** 1852 * skb_clone_writable - is the header of a clone writable 1853 * @skb: buffer to check 1854 * @len: length up to which to write 1855 * 1856 * Returns true if modifying the header part of the cloned buffer 1857 * does not requires the data to be copied. 1858 */ 1859 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len) 1860 { 1861 return !skb_header_cloned(skb) && 1862 skb_headroom(skb) + len <= skb->hdr_len; 1863 } 1864 1865 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom, 1866 int cloned) 1867 { 1868 int delta = 0; 1869 1870 if (headroom < NET_SKB_PAD) 1871 headroom = NET_SKB_PAD; 1872 if (headroom > skb_headroom(skb)) 1873 delta = headroom - skb_headroom(skb); 1874 1875 if (delta || cloned) 1876 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0, 1877 GFP_ATOMIC); 1878 return 0; 1879 } 1880 1881 /** 1882 * skb_cow - copy header of skb when it is required 1883 * @skb: buffer to cow 1884 * @headroom: needed headroom 1885 * 1886 * If the skb passed lacks sufficient headroom or its data part 1887 * is shared, data is reallocated. If reallocation fails, an error 1888 * is returned and original skb is not changed. 1889 * 1890 * The result is skb with writable area skb->head...skb->tail 1891 * and at least @headroom of space at head. 1892 */ 1893 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom) 1894 { 1895 return __skb_cow(skb, headroom, skb_cloned(skb)); 1896 } 1897 1898 /** 1899 * skb_cow_head - skb_cow but only making the head writable 1900 * @skb: buffer to cow 1901 * @headroom: needed headroom 1902 * 1903 * This function is identical to skb_cow except that we replace the 1904 * skb_cloned check by skb_header_cloned. It should be used when 1905 * you only need to push on some header and do not need to modify 1906 * the data. 1907 */ 1908 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom) 1909 { 1910 return __skb_cow(skb, headroom, skb_header_cloned(skb)); 1911 } 1912 1913 /** 1914 * skb_padto - pad an skbuff up to a minimal size 1915 * @skb: buffer to pad 1916 * @len: minimal length 1917 * 1918 * Pads up a buffer to ensure the trailing bytes exist and are 1919 * blanked. If the buffer already contains sufficient data it 1920 * is untouched. Otherwise it is extended. Returns zero on 1921 * success. The skb is freed on error. 1922 */ 1923 1924 static inline int skb_padto(struct sk_buff *skb, unsigned int len) 1925 { 1926 unsigned int size = skb->len; 1927 if (likely(size >= len)) 1928 return 0; 1929 return skb_pad(skb, len - size); 1930 } 1931 1932 static inline int skb_add_data(struct sk_buff *skb, 1933 char __user *from, int copy) 1934 { 1935 const int off = skb->len; 1936 1937 if (skb->ip_summed == CHECKSUM_NONE) { 1938 int err = 0; 1939 __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy), 1940 copy, 0, &err); 1941 if (!err) { 1942 skb->csum = csum_block_add(skb->csum, csum, off); 1943 return 0; 1944 } 1945 } else if (!copy_from_user(skb_put(skb, copy), from, copy)) 1946 return 0; 1947 1948 __skb_trim(skb, off); 1949 return -EFAULT; 1950 } 1951 1952 static inline int skb_can_coalesce(struct sk_buff *skb, int i, 1953 const struct page *page, int off) 1954 { 1955 if (i) { 1956 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; 1957 1958 return page == skb_frag_page(frag) && 1959 off == frag->page_offset + skb_frag_size(frag); 1960 } 1961 return 0; 1962 } 1963 1964 static inline int __skb_linearize(struct sk_buff *skb) 1965 { 1966 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM; 1967 } 1968 1969 /** 1970 * skb_linearize - convert paged skb to linear one 1971 * @skb: buffer to linarize 1972 * 1973 * If there is no free memory -ENOMEM is returned, otherwise zero 1974 * is returned and the old skb data released. 1975 */ 1976 static inline int skb_linearize(struct sk_buff *skb) 1977 { 1978 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0; 1979 } 1980 1981 /** 1982 * skb_linearize_cow - make sure skb is linear and writable 1983 * @skb: buffer to process 1984 * 1985 * If there is no free memory -ENOMEM is returned, otherwise zero 1986 * is returned and the old skb data released. 1987 */ 1988 static inline int skb_linearize_cow(struct sk_buff *skb) 1989 { 1990 return skb_is_nonlinear(skb) || skb_cloned(skb) ? 1991 __skb_linearize(skb) : 0; 1992 } 1993 1994 /** 1995 * skb_postpull_rcsum - update checksum for received skb after pull 1996 * @skb: buffer to update 1997 * @start: start of data before pull 1998 * @len: length of data pulled 1999 * 2000 * After doing a pull on a received packet, you need to call this to 2001 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to 2002 * CHECKSUM_NONE so that it can be recomputed from scratch. 2003 */ 2004 2005 static inline void skb_postpull_rcsum(struct sk_buff *skb, 2006 const void *start, unsigned int len) 2007 { 2008 if (skb->ip_summed == CHECKSUM_COMPLETE) 2009 skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0)); 2010 } 2011 2012 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); 2013 2014 /** 2015 * pskb_trim_rcsum - trim received skb and update checksum 2016 * @skb: buffer to trim 2017 * @len: new length 2018 * 2019 * This is exactly the same as pskb_trim except that it ensures the 2020 * checksum of received packets are still valid after the operation. 2021 */ 2022 2023 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) 2024 { 2025 if (likely(len >= skb->len)) 2026 return 0; 2027 if (skb->ip_summed == CHECKSUM_COMPLETE) 2028 skb->ip_summed = CHECKSUM_NONE; 2029 return __pskb_trim(skb, len); 2030 } 2031 2032 #define skb_queue_walk(queue, skb) \ 2033 for (skb = (queue)->next; \ 2034 skb != (struct sk_buff *)(queue); \ 2035 skb = skb->next) 2036 2037 #define skb_queue_walk_safe(queue, skb, tmp) \ 2038 for (skb = (queue)->next, tmp = skb->next; \ 2039 skb != (struct sk_buff *)(queue); \ 2040 skb = tmp, tmp = skb->next) 2041 2042 #define skb_queue_walk_from(queue, skb) \ 2043 for (; skb != (struct sk_buff *)(queue); \ 2044 skb = skb->next) 2045 2046 #define skb_queue_walk_from_safe(queue, skb, tmp) \ 2047 for (tmp = skb->next; \ 2048 skb != (struct sk_buff *)(queue); \ 2049 skb = tmp, tmp = skb->next) 2050 2051 #define skb_queue_reverse_walk(queue, skb) \ 2052 for (skb = (queue)->prev; \ 2053 skb != (struct sk_buff *)(queue); \ 2054 skb = skb->prev) 2055 2056 #define skb_queue_reverse_walk_safe(queue, skb, tmp) \ 2057 for (skb = (queue)->prev, tmp = skb->prev; \ 2058 skb != (struct sk_buff *)(queue); \ 2059 skb = tmp, tmp = skb->prev) 2060 2061 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \ 2062 for (tmp = skb->prev; \ 2063 skb != (struct sk_buff *)(queue); \ 2064 skb = tmp, tmp = skb->prev) 2065 2066 static inline bool skb_has_frag_list(const struct sk_buff *skb) 2067 { 2068 return skb_shinfo(skb)->frag_list != NULL; 2069 } 2070 2071 static inline void skb_frag_list_init(struct sk_buff *skb) 2072 { 2073 skb_shinfo(skb)->frag_list = NULL; 2074 } 2075 2076 static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag) 2077 { 2078 frag->next = skb_shinfo(skb)->frag_list; 2079 skb_shinfo(skb)->frag_list = frag; 2080 } 2081 2082 #define skb_walk_frags(skb, iter) \ 2083 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next) 2084 2085 extern struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags, 2086 int *peeked, int *off, int *err); 2087 extern struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, 2088 int noblock, int *err); 2089 extern unsigned int datagram_poll(struct file *file, struct socket *sock, 2090 struct poll_table_struct *wait); 2091 extern int skb_copy_datagram_iovec(const struct sk_buff *from, 2092 int offset, struct iovec *to, 2093 int size); 2094 extern int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, 2095 int hlen, 2096 struct iovec *iov); 2097 extern int skb_copy_datagram_from_iovec(struct sk_buff *skb, 2098 int offset, 2099 const struct iovec *from, 2100 int from_offset, 2101 int len); 2102 extern int skb_copy_datagram_const_iovec(const struct sk_buff *from, 2103 int offset, 2104 const struct iovec *to, 2105 int to_offset, 2106 int size); 2107 extern void skb_free_datagram(struct sock *sk, struct sk_buff *skb); 2108 extern void skb_free_datagram_locked(struct sock *sk, 2109 struct sk_buff *skb); 2110 extern int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, 2111 unsigned int flags); 2112 extern __wsum skb_checksum(const struct sk_buff *skb, int offset, 2113 int len, __wsum csum); 2114 extern int skb_copy_bits(const struct sk_buff *skb, int offset, 2115 void *to, int len); 2116 extern int skb_store_bits(struct sk_buff *skb, int offset, 2117 const void *from, int len); 2118 extern __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, 2119 int offset, u8 *to, int len, 2120 __wsum csum); 2121 extern int skb_splice_bits(struct sk_buff *skb, 2122 unsigned int offset, 2123 struct pipe_inode_info *pipe, 2124 unsigned int len, 2125 unsigned int flags); 2126 extern void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to); 2127 extern void skb_split(struct sk_buff *skb, 2128 struct sk_buff *skb1, const u32 len); 2129 extern int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, 2130 int shiftlen); 2131 2132 extern struct sk_buff *skb_segment(struct sk_buff *skb, 2133 netdev_features_t features); 2134 2135 static inline void *skb_header_pointer(const struct sk_buff *skb, int offset, 2136 int len, void *buffer) 2137 { 2138 int hlen = skb_headlen(skb); 2139 2140 if (hlen - offset >= len) 2141 return skb->data + offset; 2142 2143 if (skb_copy_bits(skb, offset, buffer, len) < 0) 2144 return NULL; 2145 2146 return buffer; 2147 } 2148 2149 static inline void skb_copy_from_linear_data(const struct sk_buff *skb, 2150 void *to, 2151 const unsigned int len) 2152 { 2153 memcpy(to, skb->data, len); 2154 } 2155 2156 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb, 2157 const int offset, void *to, 2158 const unsigned int len) 2159 { 2160 memcpy(to, skb->data + offset, len); 2161 } 2162 2163 static inline void skb_copy_to_linear_data(struct sk_buff *skb, 2164 const void *from, 2165 const unsigned int len) 2166 { 2167 memcpy(skb->data, from, len); 2168 } 2169 2170 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb, 2171 const int offset, 2172 const void *from, 2173 const unsigned int len) 2174 { 2175 memcpy(skb->data + offset, from, len); 2176 } 2177 2178 extern void skb_init(void); 2179 2180 static inline ktime_t skb_get_ktime(const struct sk_buff *skb) 2181 { 2182 return skb->tstamp; 2183 } 2184 2185 /** 2186 * skb_get_timestamp - get timestamp from a skb 2187 * @skb: skb to get stamp from 2188 * @stamp: pointer to struct timeval to store stamp in 2189 * 2190 * Timestamps are stored in the skb as offsets to a base timestamp. 2191 * This function converts the offset back to a struct timeval and stores 2192 * it in stamp. 2193 */ 2194 static inline void skb_get_timestamp(const struct sk_buff *skb, 2195 struct timeval *stamp) 2196 { 2197 *stamp = ktime_to_timeval(skb->tstamp); 2198 } 2199 2200 static inline void skb_get_timestampns(const struct sk_buff *skb, 2201 struct timespec *stamp) 2202 { 2203 *stamp = ktime_to_timespec(skb->tstamp); 2204 } 2205 2206 static inline void __net_timestamp(struct sk_buff *skb) 2207 { 2208 skb->tstamp = ktime_get_real(); 2209 } 2210 2211 static inline ktime_t net_timedelta(ktime_t t) 2212 { 2213 return ktime_sub(ktime_get_real(), t); 2214 } 2215 2216 static inline ktime_t net_invalid_timestamp(void) 2217 { 2218 return ktime_set(0, 0); 2219 } 2220 2221 extern void skb_timestamping_init(void); 2222 2223 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING 2224 2225 extern void skb_clone_tx_timestamp(struct sk_buff *skb); 2226 extern bool skb_defer_rx_timestamp(struct sk_buff *skb); 2227 2228 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */ 2229 2230 static inline void skb_clone_tx_timestamp(struct sk_buff *skb) 2231 { 2232 } 2233 2234 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb) 2235 { 2236 return false; 2237 } 2238 2239 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */ 2240 2241 /** 2242 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps 2243 * 2244 * PHY drivers may accept clones of transmitted packets for 2245 * timestamping via their phy_driver.txtstamp method. These drivers 2246 * must call this function to return the skb back to the stack, with 2247 * or without a timestamp. 2248 * 2249 * @skb: clone of the the original outgoing packet 2250 * @hwtstamps: hardware time stamps, may be NULL if not available 2251 * 2252 */ 2253 void skb_complete_tx_timestamp(struct sk_buff *skb, 2254 struct skb_shared_hwtstamps *hwtstamps); 2255 2256 /** 2257 * skb_tstamp_tx - queue clone of skb with send time stamps 2258 * @orig_skb: the original outgoing packet 2259 * @hwtstamps: hardware time stamps, may be NULL if not available 2260 * 2261 * If the skb has a socket associated, then this function clones the 2262 * skb (thus sharing the actual data and optional structures), stores 2263 * the optional hardware time stamping information (if non NULL) or 2264 * generates a software time stamp (otherwise), then queues the clone 2265 * to the error queue of the socket. Errors are silently ignored. 2266 */ 2267 extern void skb_tstamp_tx(struct sk_buff *orig_skb, 2268 struct skb_shared_hwtstamps *hwtstamps); 2269 2270 static inline void sw_tx_timestamp(struct sk_buff *skb) 2271 { 2272 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP && 2273 !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) 2274 skb_tstamp_tx(skb, NULL); 2275 } 2276 2277 /** 2278 * skb_tx_timestamp() - Driver hook for transmit timestamping 2279 * 2280 * Ethernet MAC Drivers should call this function in their hard_xmit() 2281 * function immediately before giving the sk_buff to the MAC hardware. 2282 * 2283 * @skb: A socket buffer. 2284 */ 2285 static inline void skb_tx_timestamp(struct sk_buff *skb) 2286 { 2287 skb_clone_tx_timestamp(skb); 2288 sw_tx_timestamp(skb); 2289 } 2290 2291 /** 2292 * skb_complete_wifi_ack - deliver skb with wifi status 2293 * 2294 * @skb: the original outgoing packet 2295 * @acked: ack status 2296 * 2297 */ 2298 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked); 2299 2300 extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len); 2301 extern __sum16 __skb_checksum_complete(struct sk_buff *skb); 2302 2303 static inline int skb_csum_unnecessary(const struct sk_buff *skb) 2304 { 2305 return skb->ip_summed & CHECKSUM_UNNECESSARY; 2306 } 2307 2308 /** 2309 * skb_checksum_complete - Calculate checksum of an entire packet 2310 * @skb: packet to process 2311 * 2312 * This function calculates the checksum over the entire packet plus 2313 * the value of skb->csum. The latter can be used to supply the 2314 * checksum of a pseudo header as used by TCP/UDP. It returns the 2315 * checksum. 2316 * 2317 * For protocols that contain complete checksums such as ICMP/TCP/UDP, 2318 * this function can be used to verify that checksum on received 2319 * packets. In that case the function should return zero if the 2320 * checksum is correct. In particular, this function will return zero 2321 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the 2322 * hardware has already verified the correctness of the checksum. 2323 */ 2324 static inline __sum16 skb_checksum_complete(struct sk_buff *skb) 2325 { 2326 return skb_csum_unnecessary(skb) ? 2327 0 : __skb_checksum_complete(skb); 2328 } 2329 2330 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2331 extern void nf_conntrack_destroy(struct nf_conntrack *nfct); 2332 static inline void nf_conntrack_put(struct nf_conntrack *nfct) 2333 { 2334 if (nfct && atomic_dec_and_test(&nfct->use)) 2335 nf_conntrack_destroy(nfct); 2336 } 2337 static inline void nf_conntrack_get(struct nf_conntrack *nfct) 2338 { 2339 if (nfct) 2340 atomic_inc(&nfct->use); 2341 } 2342 #endif 2343 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED 2344 static inline void nf_conntrack_get_reasm(struct sk_buff *skb) 2345 { 2346 if (skb) 2347 atomic_inc(&skb->users); 2348 } 2349 static inline void nf_conntrack_put_reasm(struct sk_buff *skb) 2350 { 2351 if (skb) 2352 kfree_skb(skb); 2353 } 2354 #endif 2355 #ifdef CONFIG_BRIDGE_NETFILTER 2356 static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge) 2357 { 2358 if (nf_bridge && atomic_dec_and_test(&nf_bridge->use)) 2359 kfree(nf_bridge); 2360 } 2361 static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge) 2362 { 2363 if (nf_bridge) 2364 atomic_inc(&nf_bridge->use); 2365 } 2366 #endif /* CONFIG_BRIDGE_NETFILTER */ 2367 static inline void nf_reset(struct sk_buff *skb) 2368 { 2369 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2370 nf_conntrack_put(skb->nfct); 2371 skb->nfct = NULL; 2372 #endif 2373 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED 2374 nf_conntrack_put_reasm(skb->nfct_reasm); 2375 skb->nfct_reasm = NULL; 2376 #endif 2377 #ifdef CONFIG_BRIDGE_NETFILTER 2378 nf_bridge_put(skb->nf_bridge); 2379 skb->nf_bridge = NULL; 2380 #endif 2381 } 2382 2383 /* Note: This doesn't put any conntrack and bridge info in dst. */ 2384 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src) 2385 { 2386 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2387 dst->nfct = src->nfct; 2388 nf_conntrack_get(src->nfct); 2389 dst->nfctinfo = src->nfctinfo; 2390 #endif 2391 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED 2392 dst->nfct_reasm = src->nfct_reasm; 2393 nf_conntrack_get_reasm(src->nfct_reasm); 2394 #endif 2395 #ifdef CONFIG_BRIDGE_NETFILTER 2396 dst->nf_bridge = src->nf_bridge; 2397 nf_bridge_get(src->nf_bridge); 2398 #endif 2399 } 2400 2401 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src) 2402 { 2403 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2404 nf_conntrack_put(dst->nfct); 2405 #endif 2406 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED 2407 nf_conntrack_put_reasm(dst->nfct_reasm); 2408 #endif 2409 #ifdef CONFIG_BRIDGE_NETFILTER 2410 nf_bridge_put(dst->nf_bridge); 2411 #endif 2412 __nf_copy(dst, src); 2413 } 2414 2415 #ifdef CONFIG_NETWORK_SECMARK 2416 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from) 2417 { 2418 to->secmark = from->secmark; 2419 } 2420 2421 static inline void skb_init_secmark(struct sk_buff *skb) 2422 { 2423 skb->secmark = 0; 2424 } 2425 #else 2426 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from) 2427 { } 2428 2429 static inline void skb_init_secmark(struct sk_buff *skb) 2430 { } 2431 #endif 2432 2433 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping) 2434 { 2435 skb->queue_mapping = queue_mapping; 2436 } 2437 2438 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb) 2439 { 2440 return skb->queue_mapping; 2441 } 2442 2443 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from) 2444 { 2445 to->queue_mapping = from->queue_mapping; 2446 } 2447 2448 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue) 2449 { 2450 skb->queue_mapping = rx_queue + 1; 2451 } 2452 2453 static inline u16 skb_get_rx_queue(const struct sk_buff *skb) 2454 { 2455 return skb->queue_mapping - 1; 2456 } 2457 2458 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb) 2459 { 2460 return skb->queue_mapping != 0; 2461 } 2462 2463 extern u16 __skb_tx_hash(const struct net_device *dev, 2464 const struct sk_buff *skb, 2465 unsigned int num_tx_queues); 2466 2467 #ifdef CONFIG_XFRM 2468 static inline struct sec_path *skb_sec_path(struct sk_buff *skb) 2469 { 2470 return skb->sp; 2471 } 2472 #else 2473 static inline struct sec_path *skb_sec_path(struct sk_buff *skb) 2474 { 2475 return NULL; 2476 } 2477 #endif 2478 2479 static inline bool skb_is_gso(const struct sk_buff *skb) 2480 { 2481 return skb_shinfo(skb)->gso_size; 2482 } 2483 2484 static inline bool skb_is_gso_v6(const struct sk_buff *skb) 2485 { 2486 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; 2487 } 2488 2489 extern void __skb_warn_lro_forwarding(const struct sk_buff *skb); 2490 2491 static inline bool skb_warn_if_lro(const struct sk_buff *skb) 2492 { 2493 /* LRO sets gso_size but not gso_type, whereas if GSO is really 2494 * wanted then gso_type will be set. */ 2495 const struct skb_shared_info *shinfo = skb_shinfo(skb); 2496 2497 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 && 2498 unlikely(shinfo->gso_type == 0)) { 2499 __skb_warn_lro_forwarding(skb); 2500 return true; 2501 } 2502 return false; 2503 } 2504 2505 static inline void skb_forward_csum(struct sk_buff *skb) 2506 { 2507 /* Unfortunately we don't support this one. Any brave souls? */ 2508 if (skb->ip_summed == CHECKSUM_COMPLETE) 2509 skb->ip_summed = CHECKSUM_NONE; 2510 } 2511 2512 /** 2513 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE 2514 * @skb: skb to check 2515 * 2516 * fresh skbs have their ip_summed set to CHECKSUM_NONE. 2517 * Instead of forcing ip_summed to CHECKSUM_NONE, we can 2518 * use this helper, to document places where we make this assertion. 2519 */ 2520 static inline void skb_checksum_none_assert(const struct sk_buff *skb) 2521 { 2522 #ifdef DEBUG 2523 BUG_ON(skb->ip_summed != CHECKSUM_NONE); 2524 #endif 2525 } 2526 2527 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off); 2528 2529 static inline bool skb_is_recycleable(const struct sk_buff *skb, int skb_size) 2530 { 2531 if (irqs_disabled()) 2532 return false; 2533 2534 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) 2535 return false; 2536 2537 if (skb_is_nonlinear(skb) || skb->fclone != SKB_FCLONE_UNAVAILABLE) 2538 return false; 2539 2540 skb_size = SKB_DATA_ALIGN(skb_size + NET_SKB_PAD); 2541 if (skb_end_pointer(skb) - skb->head < skb_size) 2542 return false; 2543 2544 if (skb_shared(skb) || skb_cloned(skb)) 2545 return false; 2546 2547 return true; 2548 } 2549 #endif /* __KERNEL__ */ 2550 #endif /* _LINUX_SKBUFF_H */ 2551