1 #ifndef __LINUX_PERCPU_H 2 #define __LINUX_PERCPU_H 3 4 #include <linux/preempt.h> 5 #include <linux/slab.h> /* For kmalloc() */ 6 #include <linux/smp.h> 7 #include <linux/cpumask.h> 8 #include <linux/pfn.h> 9 10 #include <asm/percpu.h> 11 12 /* enough to cover all DEFINE_PER_CPUs in modules */ 13 #ifdef CONFIG_MODULES 14 #define PERCPU_MODULE_RESERVE (8 << 10) 15 #else 16 #define PERCPU_MODULE_RESERVE 0 17 #endif 18 19 #ifndef PERCPU_ENOUGH_ROOM 20 #define PERCPU_ENOUGH_ROOM \ 21 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \ 22 PERCPU_MODULE_RESERVE) 23 #endif 24 25 /* 26 * Must be an lvalue. Since @var must be a simple identifier, 27 * we force a syntax error here if it isn't. 28 */ 29 #define get_cpu_var(var) (*({ \ 30 extern int simple_identifier_##var(void); \ 31 preempt_disable(); \ 32 &__get_cpu_var(var); })) 33 #define put_cpu_var(var) preempt_enable() 34 35 #ifdef CONFIG_SMP 36 37 /* minimum unit size, also is the maximum supported allocation size */ 38 #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10) 39 40 /* 41 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 42 * back on the first chunk for dynamic percpu allocation if arch is 43 * manually allocating and mapping it for faster access (as a part of 44 * large page mapping for example). 45 * 46 * The following values give between one and two pages of free space 47 * after typical minimal boot (2-way SMP, single disk and NIC) with 48 * both defconfig and a distro config on x86_64 and 32. More 49 * intelligent way to determine this would be nice. 50 */ 51 #if BITS_PER_LONG > 32 52 #define PERCPU_DYNAMIC_RESERVE (20 << 10) 53 #else 54 #define PERCPU_DYNAMIC_RESERVE (12 << 10) 55 #endif 56 57 extern void *pcpu_base_addr; 58 extern const unsigned long *pcpu_unit_offsets; 59 60 struct pcpu_group_info { 61 int nr_units; /* aligned # of units */ 62 unsigned long base_offset; /* base address offset */ 63 unsigned int *cpu_map; /* unit->cpu map, empty 64 * entries contain NR_CPUS */ 65 }; 66 67 struct pcpu_alloc_info { 68 size_t static_size; 69 size_t reserved_size; 70 size_t dyn_size; 71 size_t unit_size; 72 size_t atom_size; 73 size_t alloc_size; 74 size_t __ai_size; /* internal, don't use */ 75 int nr_groups; /* 0 if grouping unnecessary */ 76 struct pcpu_group_info groups[]; 77 }; 78 79 enum pcpu_fc { 80 PCPU_FC_AUTO, 81 PCPU_FC_EMBED, 82 PCPU_FC_PAGE, 83 84 PCPU_FC_NR, 85 }; 86 extern const char *pcpu_fc_names[PCPU_FC_NR]; 87 88 extern enum pcpu_fc pcpu_chosen_fc; 89 90 typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, 91 size_t align); 92 typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); 93 typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); 94 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 95 96 extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, 97 int nr_units); 98 extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai); 99 100 extern struct pcpu_alloc_info * __init pcpu_build_alloc_info( 101 size_t reserved_size, ssize_t dyn_size, 102 size_t atom_size, 103 pcpu_fc_cpu_distance_fn_t cpu_distance_fn); 104 105 extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, 106 void *base_addr); 107 108 #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK 109 extern int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size, 110 size_t atom_size, 111 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 112 pcpu_fc_alloc_fn_t alloc_fn, 113 pcpu_fc_free_fn_t free_fn); 114 #endif 115 116 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 117 extern int __init pcpu_page_first_chunk(size_t reserved_size, 118 pcpu_fc_alloc_fn_t alloc_fn, 119 pcpu_fc_free_fn_t free_fn, 120 pcpu_fc_populate_pte_fn_t populate_pte_fn); 121 #endif 122 123 /* 124 * Use this to get to a cpu's version of the per-cpu object 125 * dynamically allocated. Non-atomic access to the current CPU's 126 * version should probably be combined with get_cpu()/put_cpu(). 127 */ 128 #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) 129 130 extern void *__alloc_reserved_percpu(size_t size, size_t align); 131 extern void *__alloc_percpu(size_t size, size_t align); 132 extern void free_percpu(void *__pdata); 133 extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 134 135 #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA 136 extern void __init setup_per_cpu_areas(void); 137 #endif 138 139 #else /* CONFIG_SMP */ 140 141 #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) 142 143 static inline void *__alloc_percpu(size_t size, size_t align) 144 { 145 /* 146 * Can't easily make larger alignment work with kmalloc. WARN 147 * on it. Larger alignment should only be used for module 148 * percpu sections on SMP for which this path isn't used. 149 */ 150 WARN_ON_ONCE(align > SMP_CACHE_BYTES); 151 return kzalloc(size, GFP_KERNEL); 152 } 153 154 static inline void free_percpu(void *p) 155 { 156 kfree(p); 157 } 158 159 static inline phys_addr_t per_cpu_ptr_to_phys(void *addr) 160 { 161 return __pa(addr); 162 } 163 164 static inline void __init setup_per_cpu_areas(void) { } 165 166 static inline void *pcpu_lpage_remapped(void *kaddr) 167 { 168 return NULL; 169 } 170 171 #endif /* CONFIG_SMP */ 172 173 #define alloc_percpu(type) \ 174 (typeof(type) *)__alloc_percpu(sizeof(type), __alignof__(type)) 175 176 /* 177 * Optional methods for optimized non-lvalue per-cpu variable access. 178 * 179 * @var can be a percpu variable or a field of it and its size should 180 * equal char, int or long. percpu_read() evaluates to a lvalue and 181 * all others to void. 182 * 183 * These operations are guaranteed to be atomic w.r.t. preemption. 184 * The generic versions use plain get/put_cpu_var(). Archs are 185 * encouraged to implement single-instruction alternatives which don't 186 * require preemption protection. 187 */ 188 #ifndef percpu_read 189 # define percpu_read(var) \ 190 ({ \ 191 typeof(per_cpu_var(var)) __tmp_var__; \ 192 __tmp_var__ = get_cpu_var(var); \ 193 put_cpu_var(var); \ 194 __tmp_var__; \ 195 }) 196 #endif 197 198 #define __percpu_generic_to_op(var, val, op) \ 199 do { \ 200 get_cpu_var(var) op val; \ 201 put_cpu_var(var); \ 202 } while (0) 203 204 #ifndef percpu_write 205 # define percpu_write(var, val) __percpu_generic_to_op(var, (val), =) 206 #endif 207 208 #ifndef percpu_add 209 # define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=) 210 #endif 211 212 #ifndef percpu_sub 213 # define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=) 214 #endif 215 216 #ifndef percpu_and 217 # define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=) 218 #endif 219 220 #ifndef percpu_or 221 # define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=) 222 #endif 223 224 #ifndef percpu_xor 225 # define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=) 226 #endif 227 228 /* 229 * Branching function to split up a function into a set of functions that 230 * are called for different scalar sizes of the objects handled. 231 */ 232 233 extern void __bad_size_call_parameter(void); 234 235 #define __pcpu_size_call_return(stem, variable) \ 236 ({ typeof(variable) pscr_ret__; \ 237 switch(sizeof(variable)) { \ 238 case 1: pscr_ret__ = stem##1(variable);break; \ 239 case 2: pscr_ret__ = stem##2(variable);break; \ 240 case 4: pscr_ret__ = stem##4(variable);break; \ 241 case 8: pscr_ret__ = stem##8(variable);break; \ 242 default: \ 243 __bad_size_call_parameter();break; \ 244 } \ 245 pscr_ret__; \ 246 }) 247 248 #define __pcpu_size_call(stem, variable, ...) \ 249 do { \ 250 switch(sizeof(variable)) { \ 251 case 1: stem##1(variable, __VA_ARGS__);break; \ 252 case 2: stem##2(variable, __VA_ARGS__);break; \ 253 case 4: stem##4(variable, __VA_ARGS__);break; \ 254 case 8: stem##8(variable, __VA_ARGS__);break; \ 255 default: \ 256 __bad_size_call_parameter();break; \ 257 } \ 258 } while (0) 259 260 /* 261 * Optimized manipulation for memory allocated through the per cpu 262 * allocator or for addresses of per cpu variables (can be determined 263 * using per_cpu_var(xx). 264 * 265 * These operation guarantee exclusivity of access for other operations 266 * on the *same* processor. The assumption is that per cpu data is only 267 * accessed by a single processor instance (the current one). 268 * 269 * The first group is used for accesses that must be done in a 270 * preemption safe way since we know that the context is not preempt 271 * safe. Interrupts may occur. If the interrupt modifies the variable 272 * too then RMW actions will not be reliable. 273 * 274 * The arch code can provide optimized functions in two ways: 275 * 276 * 1. Override the function completely. F.e. define this_cpu_add(). 277 * The arch must then ensure that the various scalar format passed 278 * are handled correctly. 279 * 280 * 2. Provide functions for certain scalar sizes. F.e. provide 281 * this_cpu_add_2() to provide per cpu atomic operations for 2 byte 282 * sized RMW actions. If arch code does not provide operations for 283 * a scalar size then the fallback in the generic code will be 284 * used. 285 */ 286 287 #define _this_cpu_generic_read(pcp) \ 288 ({ typeof(pcp) ret__; \ 289 preempt_disable(); \ 290 ret__ = *this_cpu_ptr(&(pcp)); \ 291 preempt_enable(); \ 292 ret__; \ 293 }) 294 295 #ifndef this_cpu_read 296 # ifndef this_cpu_read_1 297 # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp) 298 # endif 299 # ifndef this_cpu_read_2 300 # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp) 301 # endif 302 # ifndef this_cpu_read_4 303 # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp) 304 # endif 305 # ifndef this_cpu_read_8 306 # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) 307 # endif 308 # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp)) 309 #endif 310 311 #define _this_cpu_generic_to_op(pcp, val, op) \ 312 do { \ 313 preempt_disable(); \ 314 *__this_cpu_ptr(&pcp) op val; \ 315 preempt_enable(); \ 316 } while (0) 317 318 #ifndef this_cpu_write 319 # ifndef this_cpu_write_1 320 # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 321 # endif 322 # ifndef this_cpu_write_2 323 # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 324 # endif 325 # ifndef this_cpu_write_4 326 # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 327 # endif 328 # ifndef this_cpu_write_8 329 # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 330 # endif 331 # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val)) 332 #endif 333 334 #ifndef this_cpu_add 335 # ifndef this_cpu_add_1 336 # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 337 # endif 338 # ifndef this_cpu_add_2 339 # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 340 # endif 341 # ifndef this_cpu_add_4 342 # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 343 # endif 344 # ifndef this_cpu_add_8 345 # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 346 # endif 347 # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val)) 348 #endif 349 350 #ifndef this_cpu_sub 351 # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val)) 352 #endif 353 354 #ifndef this_cpu_inc 355 # define this_cpu_inc(pcp) this_cpu_add((pcp), 1) 356 #endif 357 358 #ifndef this_cpu_dec 359 # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1) 360 #endif 361 362 #ifndef this_cpu_and 363 # ifndef this_cpu_and_1 364 # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 365 # endif 366 # ifndef this_cpu_and_2 367 # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 368 # endif 369 # ifndef this_cpu_and_4 370 # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 371 # endif 372 # ifndef this_cpu_and_8 373 # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 374 # endif 375 # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val)) 376 #endif 377 378 #ifndef this_cpu_or 379 # ifndef this_cpu_or_1 380 # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 381 # endif 382 # ifndef this_cpu_or_2 383 # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 384 # endif 385 # ifndef this_cpu_or_4 386 # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 387 # endif 388 # ifndef this_cpu_or_8 389 # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 390 # endif 391 # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) 392 #endif 393 394 #ifndef this_cpu_xor 395 # ifndef this_cpu_xor_1 396 # define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) 397 # endif 398 # ifndef this_cpu_xor_2 399 # define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) 400 # endif 401 # ifndef this_cpu_xor_4 402 # define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) 403 # endif 404 # ifndef this_cpu_xor_8 405 # define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) 406 # endif 407 # define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) 408 #endif 409 410 /* 411 * Generic percpu operations that do not require preemption handling. 412 * Either we do not care about races or the caller has the 413 * responsibility of handling preemptions issues. Arch code can still 414 * override these instructions since the arch per cpu code may be more 415 * efficient and may actually get race freeness for free (that is the 416 * case for x86 for example). 417 * 418 * If there is no other protection through preempt disable and/or 419 * disabling interupts then one of these RMW operations can show unexpected 420 * behavior because the execution thread was rescheduled on another processor 421 * or an interrupt occurred and the same percpu variable was modified from 422 * the interrupt context. 423 */ 424 #ifndef __this_cpu_read 425 # ifndef __this_cpu_read_1 426 # define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp))) 427 # endif 428 # ifndef __this_cpu_read_2 429 # define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp))) 430 # endif 431 # ifndef __this_cpu_read_4 432 # define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp))) 433 # endif 434 # ifndef __this_cpu_read_8 435 # define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp))) 436 # endif 437 # define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp)) 438 #endif 439 440 #define __this_cpu_generic_to_op(pcp, val, op) \ 441 do { \ 442 *__this_cpu_ptr(&(pcp)) op val; \ 443 } while (0) 444 445 #ifndef __this_cpu_write 446 # ifndef __this_cpu_write_1 447 # define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 448 # endif 449 # ifndef __this_cpu_write_2 450 # define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 451 # endif 452 # ifndef __this_cpu_write_4 453 # define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 454 # endif 455 # ifndef __this_cpu_write_8 456 # define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 457 # endif 458 # define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val)) 459 #endif 460 461 #ifndef __this_cpu_add 462 # ifndef __this_cpu_add_1 463 # define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 464 # endif 465 # ifndef __this_cpu_add_2 466 # define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 467 # endif 468 # ifndef __this_cpu_add_4 469 # define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 470 # endif 471 # ifndef __this_cpu_add_8 472 # define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 473 # endif 474 # define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val)) 475 #endif 476 477 #ifndef __this_cpu_sub 478 # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val)) 479 #endif 480 481 #ifndef __this_cpu_inc 482 # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1) 483 #endif 484 485 #ifndef __this_cpu_dec 486 # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1) 487 #endif 488 489 #ifndef __this_cpu_and 490 # ifndef __this_cpu_and_1 491 # define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 492 # endif 493 # ifndef __this_cpu_and_2 494 # define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 495 # endif 496 # ifndef __this_cpu_and_4 497 # define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 498 # endif 499 # ifndef __this_cpu_and_8 500 # define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 501 # endif 502 # define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val)) 503 #endif 504 505 #ifndef __this_cpu_or 506 # ifndef __this_cpu_or_1 507 # define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 508 # endif 509 # ifndef __this_cpu_or_2 510 # define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 511 # endif 512 # ifndef __this_cpu_or_4 513 # define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 514 # endif 515 # ifndef __this_cpu_or_8 516 # define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 517 # endif 518 # define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val)) 519 #endif 520 521 #ifndef __this_cpu_xor 522 # ifndef __this_cpu_xor_1 523 # define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) 524 # endif 525 # ifndef __this_cpu_xor_2 526 # define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) 527 # endif 528 # ifndef __this_cpu_xor_4 529 # define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) 530 # endif 531 # ifndef __this_cpu_xor_8 532 # define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) 533 # endif 534 # define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val)) 535 #endif 536 537 /* 538 * IRQ safe versions of the per cpu RMW operations. Note that these operations 539 * are *not* safe against modification of the same variable from another 540 * processors (which one gets when using regular atomic operations) 541 . They are guaranteed to be atomic vs. local interrupts and 542 * preemption only. 543 */ 544 #define irqsafe_cpu_generic_to_op(pcp, val, op) \ 545 do { \ 546 unsigned long flags; \ 547 local_irq_save(flags); \ 548 *__this_cpu_ptr(&(pcp)) op val; \ 549 local_irq_restore(flags); \ 550 } while (0) 551 552 #ifndef irqsafe_cpu_add 553 # ifndef irqsafe_cpu_add_1 554 # define irqsafe_cpu_add_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) 555 # endif 556 # ifndef irqsafe_cpu_add_2 557 # define irqsafe_cpu_add_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) 558 # endif 559 # ifndef irqsafe_cpu_add_4 560 # define irqsafe_cpu_add_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) 561 # endif 562 # ifndef irqsafe_cpu_add_8 563 # define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) 564 # endif 565 # define irqsafe_cpu_add(pcp, val) __pcpu_size_call(irqsafe_cpu_add_, (pcp), (val)) 566 #endif 567 568 #ifndef irqsafe_cpu_sub 569 # define irqsafe_cpu_sub(pcp, val) irqsafe_cpu_add((pcp), -(val)) 570 #endif 571 572 #ifndef irqsafe_cpu_inc 573 # define irqsafe_cpu_inc(pcp) irqsafe_cpu_add((pcp), 1) 574 #endif 575 576 #ifndef irqsafe_cpu_dec 577 # define irqsafe_cpu_dec(pcp) irqsafe_cpu_sub((pcp), 1) 578 #endif 579 580 #ifndef irqsafe_cpu_and 581 # ifndef irqsafe_cpu_and_1 582 # define irqsafe_cpu_and_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) 583 # endif 584 # ifndef irqsafe_cpu_and_2 585 # define irqsafe_cpu_and_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) 586 # endif 587 # ifndef irqsafe_cpu_and_4 588 # define irqsafe_cpu_and_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) 589 # endif 590 # ifndef irqsafe_cpu_and_8 591 # define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) 592 # endif 593 # define irqsafe_cpu_and(pcp, val) __pcpu_size_call(irqsafe_cpu_and_, (val)) 594 #endif 595 596 #ifndef irqsafe_cpu_or 597 # ifndef irqsafe_cpu_or_1 598 # define irqsafe_cpu_or_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) 599 # endif 600 # ifndef irqsafe_cpu_or_2 601 # define irqsafe_cpu_or_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) 602 # endif 603 # ifndef irqsafe_cpu_or_4 604 # define irqsafe_cpu_or_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) 605 # endif 606 # ifndef irqsafe_cpu_or_8 607 # define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) 608 # endif 609 # define irqsafe_cpu_or(pcp, val) __pcpu_size_call(irqsafe_cpu_or_, (val)) 610 #endif 611 612 #ifndef irqsafe_cpu_xor 613 # ifndef irqsafe_cpu_xor_1 614 # define irqsafe_cpu_xor_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) 615 # endif 616 # ifndef irqsafe_cpu_xor_2 617 # define irqsafe_cpu_xor_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) 618 # endif 619 # ifndef irqsafe_cpu_xor_4 620 # define irqsafe_cpu_xor_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) 621 # endif 622 # ifndef irqsafe_cpu_xor_8 623 # define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) 624 # endif 625 # define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val)) 626 #endif 627 628 #endif /* __LINUX_PERCPU_H */ 629