1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_CPUMASK_H 3 #define __LINUX_CPUMASK_H 4 5 /* 6 * Cpumasks provide a bitmap suitable for representing the 7 * set of CPU's in a system, one bit position per CPU number. In general, 8 * only nr_cpu_ids (<= NR_CPUS) bits are valid. 9 */ 10 #include <linux/kernel.h> 11 #include <linux/threads.h> 12 #include <linux/bitmap.h> 13 #include <linux/atomic.h> 14 #include <linux/bug.h> 15 #include <linux/gfp_types.h> 16 #include <linux/numa.h> 17 18 /* Don't assign or return these: may not be this big! */ 19 typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; 20 21 /** 22 * cpumask_bits - get the bits in a cpumask 23 * @maskp: the struct cpumask * 24 * 25 * You should only assume nr_cpu_ids bits of this mask are valid. This is 26 * a macro so it's const-correct. 27 */ 28 #define cpumask_bits(maskp) ((maskp)->bits) 29 30 /** 31 * cpumask_pr_args - printf args to output a cpumask 32 * @maskp: cpumask to be printed 33 * 34 * Can be used to provide arguments for '%*pb[l]' when printing a cpumask. 35 */ 36 #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) 37 38 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 39 #define nr_cpu_ids ((unsigned int)NR_CPUS) 40 #else 41 extern unsigned int nr_cpu_ids; 42 #endif 43 44 static inline void set_nr_cpu_ids(unsigned int nr) 45 { 46 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 47 WARN_ON(nr != nr_cpu_ids); 48 #else 49 nr_cpu_ids = nr; 50 #endif 51 } 52 53 /* Deprecated. Always use nr_cpu_ids. */ 54 #define nr_cpumask_bits nr_cpu_ids 55 56 /* 57 * The following particular system cpumasks and operations manage 58 * possible, present, active and online cpus. 59 * 60 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable 61 * cpu_present_mask - has bit 'cpu' set iff cpu is populated 62 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler 63 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration 64 * 65 * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. 66 * 67 * The cpu_possible_mask is fixed at boot time, as the set of CPU id's 68 * that it is possible might ever be plugged in at anytime during the 69 * life of that system boot. The cpu_present_mask is dynamic(*), 70 * representing which CPUs are currently plugged in. And 71 * cpu_online_mask is the dynamic subset of cpu_present_mask, 72 * indicating those CPUs available for scheduling. 73 * 74 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, 75 * depending on what ACPI reports as currently plugged in, otherwise 76 * cpu_present_mask is just a copy of cpu_possible_mask. 77 * 78 * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not 79 * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot. 80 * 81 * Subtleties: 82 * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode 83 * assumption that their single CPU is online. The UP 84 * cpu_{online,possible,present}_masks are placebos. Changing them 85 * will have no useful affect on the following num_*_cpus() 86 * and cpu_*() macros in the UP case. This ugliness is a UP 87 * optimization - don't waste any instructions or memory references 88 * asking if you're online or how many CPUs there are if there is 89 * only one CPU. 90 */ 91 92 extern struct cpumask __cpu_possible_mask; 93 extern struct cpumask __cpu_online_mask; 94 extern struct cpumask __cpu_present_mask; 95 extern struct cpumask __cpu_active_mask; 96 extern struct cpumask __cpu_dying_mask; 97 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) 98 #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) 99 #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) 100 #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) 101 #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) 102 103 extern atomic_t __num_online_cpus; 104 105 extern cpumask_t cpus_booted_once_mask; 106 107 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) 108 { 109 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 110 WARN_ON_ONCE(cpu >= bits); 111 #endif /* CONFIG_DEBUG_PER_CPU_MAPS */ 112 } 113 114 /* verify cpu argument to cpumask_* operators */ 115 static __always_inline unsigned int cpumask_check(unsigned int cpu) 116 { 117 cpu_max_bits_warn(cpu, nr_cpumask_bits); 118 return cpu; 119 } 120 121 /** 122 * cpumask_first - get the first cpu in a cpumask 123 * @srcp: the cpumask pointer 124 * 125 * Returns >= nr_cpu_ids if no cpus set. 126 */ 127 static inline unsigned int cpumask_first(const struct cpumask *srcp) 128 { 129 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits); 130 } 131 132 /** 133 * cpumask_first_zero - get the first unset cpu in a cpumask 134 * @srcp: the cpumask pointer 135 * 136 * Returns >= nr_cpu_ids if all cpus are set. 137 */ 138 static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) 139 { 140 return find_first_zero_bit(cpumask_bits(srcp), nr_cpumask_bits); 141 } 142 143 /** 144 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 145 * @src1p: the first input 146 * @src2p: the second input 147 * 148 * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). 149 */ 150 static inline 151 unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) 152 { 153 return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); 154 } 155 156 /** 157 * cpumask_last - get the last CPU in a cpumask 158 * @srcp: - the cpumask pointer 159 * 160 * Returns >= nr_cpumask_bits if no CPUs set. 161 */ 162 static inline unsigned int cpumask_last(const struct cpumask *srcp) 163 { 164 return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits); 165 } 166 167 /** 168 * cpumask_next - get the next cpu in a cpumask 169 * @n: the cpu prior to the place to search (ie. return will be > @n) 170 * @srcp: the cpumask pointer 171 * 172 * Returns >= nr_cpu_ids if no further cpus set. 173 */ 174 static inline 175 unsigned int cpumask_next(int n, const struct cpumask *srcp) 176 { 177 /* -1 is a legal arg here. */ 178 if (n != -1) 179 cpumask_check(n); 180 return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); 181 } 182 183 /** 184 * cpumask_next_zero - get the next unset cpu in a cpumask 185 * @n: the cpu prior to the place to search (ie. return will be > @n) 186 * @srcp: the cpumask pointer 187 * 188 * Returns >= nr_cpu_ids if no further cpus unset. 189 */ 190 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) 191 { 192 /* -1 is a legal arg here. */ 193 if (n != -1) 194 cpumask_check(n); 195 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); 196 } 197 198 #if NR_CPUS == 1 199 /* Uniprocessor: there is only one valid CPU */ 200 static inline unsigned int cpumask_local_spread(unsigned int i, int node) 201 { 202 return 0; 203 } 204 205 static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 206 const struct cpumask *src2p) 207 { 208 return cpumask_first_and(src1p, src2p); 209 } 210 211 static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp) 212 { 213 return cpumask_first(srcp); 214 } 215 #else 216 unsigned int cpumask_local_spread(unsigned int i, int node); 217 unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 218 const struct cpumask *src2p); 219 unsigned int cpumask_any_distribute(const struct cpumask *srcp); 220 #endif /* NR_CPUS */ 221 222 /** 223 * cpumask_next_and - get the next cpu in *src1p & *src2p 224 * @n: the cpu prior to the place to search (ie. return will be > @n) 225 * @src1p: the first cpumask pointer 226 * @src2p: the second cpumask pointer 227 * 228 * Returns >= nr_cpu_ids if no further cpus set in both. 229 */ 230 static inline 231 unsigned int cpumask_next_and(int n, const struct cpumask *src1p, 232 const struct cpumask *src2p) 233 { 234 /* -1 is a legal arg here. */ 235 if (n != -1) 236 cpumask_check(n); 237 return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), 238 nr_cpumask_bits, n + 1); 239 } 240 241 /** 242 * for_each_cpu - iterate over every cpu in a mask 243 * @cpu: the (optionally unsigned) integer iterator 244 * @mask: the cpumask pointer 245 * 246 * After the loop, cpu is >= nr_cpu_ids. 247 */ 248 #define for_each_cpu(cpu, mask) \ 249 for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) 250 251 /** 252 * for_each_cpu_not - iterate over every cpu in a complemented mask 253 * @cpu: the (optionally unsigned) integer iterator 254 * @mask: the cpumask pointer 255 * 256 * After the loop, cpu is >= nr_cpu_ids. 257 */ 258 #define for_each_cpu_not(cpu, mask) \ 259 for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) 260 261 #if NR_CPUS == 1 262 static inline 263 unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) 264 { 265 cpumask_check(start); 266 if (n != -1) 267 cpumask_check(n); 268 269 /* 270 * Return the first available CPU when wrapping, or when starting before cpu0, 271 * since there is only one valid option. 272 */ 273 if (wrap && n >= 0) 274 return nr_cpumask_bits; 275 276 return cpumask_first(mask); 277 } 278 #else 279 unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); 280 #endif 281 282 /** 283 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location 284 * @cpu: the (optionally unsigned) integer iterator 285 * @mask: the cpumask pointer 286 * @start: the start location 287 * 288 * The implementation does not assume any bit in @mask is set (including @start). 289 * 290 * After the loop, cpu is >= nr_cpu_ids. 291 */ 292 #define for_each_cpu_wrap(cpu, mask, start) \ 293 for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start) 294 295 /** 296 * for_each_cpu_and - iterate over every cpu in both masks 297 * @cpu: the (optionally unsigned) integer iterator 298 * @mask1: the first cpumask pointer 299 * @mask2: the second cpumask pointer 300 * 301 * This saves a temporary CPU mask in many places. It is equivalent to: 302 * struct cpumask tmp; 303 * cpumask_and(&tmp, &mask1, &mask2); 304 * for_each_cpu(cpu, &tmp) 305 * ... 306 * 307 * After the loop, cpu is >= nr_cpu_ids. 308 */ 309 #define for_each_cpu_and(cpu, mask1, mask2) \ 310 for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) 311 312 /** 313 * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding 314 * those present in another. 315 * @cpu: the (optionally unsigned) integer iterator 316 * @mask1: the first cpumask pointer 317 * @mask2: the second cpumask pointer 318 * 319 * This saves a temporary CPU mask in many places. It is equivalent to: 320 * struct cpumask tmp; 321 * cpumask_andnot(&tmp, &mask1, &mask2); 322 * for_each_cpu(cpu, &tmp) 323 * ... 324 * 325 * After the loop, cpu is >= nr_cpu_ids. 326 */ 327 #define for_each_cpu_andnot(cpu, mask1, mask2) \ 328 for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) 329 330 /** 331 * cpumask_any_but - return a "random" in a cpumask, but not this one. 332 * @mask: the cpumask to search 333 * @cpu: the cpu to ignore. 334 * 335 * Often used to find any cpu but smp_processor_id() in a mask. 336 * Returns >= nr_cpu_ids if no cpus set. 337 */ 338 static inline 339 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) 340 { 341 unsigned int i; 342 343 cpumask_check(cpu); 344 for_each_cpu(i, mask) 345 if (i != cpu) 346 break; 347 return i; 348 } 349 350 /** 351 * cpumask_nth - get the first cpu in a cpumask 352 * @srcp: the cpumask pointer 353 * @cpu: the N'th cpu to find, starting from 0 354 * 355 * Returns >= nr_cpu_ids if such cpu doesn't exist. 356 */ 357 static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) 358 { 359 return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu)); 360 } 361 362 /** 363 * cpumask_nth_and - get the first cpu in 2 cpumasks 364 * @srcp1: the cpumask pointer 365 * @srcp2: the cpumask pointer 366 * @cpu: the N'th cpu to find, starting from 0 367 * 368 * Returns >= nr_cpu_ids if such cpu doesn't exist. 369 */ 370 static inline 371 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, 372 const struct cpumask *srcp2) 373 { 374 return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 375 nr_cpumask_bits, cpumask_check(cpu)); 376 } 377 378 /** 379 * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. 380 * @srcp1: the cpumask pointer 381 * @srcp2: the cpumask pointer 382 * @cpu: the N'th cpu to find, starting from 0 383 * 384 * Returns >= nr_cpu_ids if such cpu doesn't exist. 385 */ 386 static inline 387 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, 388 const struct cpumask *srcp2) 389 { 390 return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 391 nr_cpumask_bits, cpumask_check(cpu)); 392 } 393 394 #define CPU_BITS_NONE \ 395 { \ 396 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 397 } 398 399 #define CPU_BITS_CPU0 \ 400 { \ 401 [0] = 1UL \ 402 } 403 404 /** 405 * cpumask_set_cpu - set a cpu in a cpumask 406 * @cpu: cpu number (< nr_cpu_ids) 407 * @dstp: the cpumask pointer 408 */ 409 static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 410 { 411 set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 412 } 413 414 static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 415 { 416 __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 417 } 418 419 420 /** 421 * cpumask_clear_cpu - clear a cpu in a cpumask 422 * @cpu: cpu number (< nr_cpu_ids) 423 * @dstp: the cpumask pointer 424 */ 425 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) 426 { 427 clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 428 } 429 430 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) 431 { 432 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 433 } 434 435 /** 436 * cpumask_test_cpu - test for a cpu in a cpumask 437 * @cpu: cpu number (< nr_cpu_ids) 438 * @cpumask: the cpumask pointer 439 * 440 * Returns true if @cpu is set in @cpumask, else returns false 441 */ 442 static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) 443 { 444 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 445 } 446 447 /** 448 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask 449 * @cpu: cpu number (< nr_cpu_ids) 450 * @cpumask: the cpumask pointer 451 * 452 * Returns true if @cpu is set in old bitmap of @cpumask, else returns false 453 * 454 * test_and_set_bit wrapper for cpumasks. 455 */ 456 static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) 457 { 458 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 459 } 460 461 /** 462 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask 463 * @cpu: cpu number (< nr_cpu_ids) 464 * @cpumask: the cpumask pointer 465 * 466 * Returns true if @cpu is set in old bitmap of @cpumask, else returns false 467 * 468 * test_and_clear_bit wrapper for cpumasks. 469 */ 470 static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) 471 { 472 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 473 } 474 475 /** 476 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask 477 * @dstp: the cpumask pointer 478 */ 479 static inline void cpumask_setall(struct cpumask *dstp) 480 { 481 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits); 482 } 483 484 /** 485 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask 486 * @dstp: the cpumask pointer 487 */ 488 static inline void cpumask_clear(struct cpumask *dstp) 489 { 490 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits); 491 } 492 493 /** 494 * cpumask_and - *dstp = *src1p & *src2p 495 * @dstp: the cpumask result 496 * @src1p: the first input 497 * @src2p: the second input 498 * 499 * If *@dstp is empty, returns false, else returns true 500 */ 501 static inline bool cpumask_and(struct cpumask *dstp, 502 const struct cpumask *src1p, 503 const struct cpumask *src2p) 504 { 505 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p), 506 cpumask_bits(src2p), nr_cpumask_bits); 507 } 508 509 /** 510 * cpumask_or - *dstp = *src1p | *src2p 511 * @dstp: the cpumask result 512 * @src1p: the first input 513 * @src2p: the second input 514 */ 515 static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, 516 const struct cpumask *src2p) 517 { 518 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p), 519 cpumask_bits(src2p), nr_cpumask_bits); 520 } 521 522 /** 523 * cpumask_xor - *dstp = *src1p ^ *src2p 524 * @dstp: the cpumask result 525 * @src1p: the first input 526 * @src2p: the second input 527 */ 528 static inline void cpumask_xor(struct cpumask *dstp, 529 const struct cpumask *src1p, 530 const struct cpumask *src2p) 531 { 532 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p), 533 cpumask_bits(src2p), nr_cpumask_bits); 534 } 535 536 /** 537 * cpumask_andnot - *dstp = *src1p & ~*src2p 538 * @dstp: the cpumask result 539 * @src1p: the first input 540 * @src2p: the second input 541 * 542 * If *@dstp is empty, returns false, else returns true 543 */ 544 static inline bool cpumask_andnot(struct cpumask *dstp, 545 const struct cpumask *src1p, 546 const struct cpumask *src2p) 547 { 548 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p), 549 cpumask_bits(src2p), nr_cpumask_bits); 550 } 551 552 /** 553 * cpumask_complement - *dstp = ~*srcp 554 * @dstp: the cpumask result 555 * @srcp: the input to invert 556 */ 557 static inline void cpumask_complement(struct cpumask *dstp, 558 const struct cpumask *srcp) 559 { 560 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp), 561 nr_cpumask_bits); 562 } 563 564 /** 565 * cpumask_equal - *src1p == *src2p 566 * @src1p: the first input 567 * @src2p: the second input 568 */ 569 static inline bool cpumask_equal(const struct cpumask *src1p, 570 const struct cpumask *src2p) 571 { 572 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p), 573 nr_cpumask_bits); 574 } 575 576 /** 577 * cpumask_or_equal - *src1p | *src2p == *src3p 578 * @src1p: the first input 579 * @src2p: the second input 580 * @src3p: the third input 581 */ 582 static inline bool cpumask_or_equal(const struct cpumask *src1p, 583 const struct cpumask *src2p, 584 const struct cpumask *src3p) 585 { 586 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), 587 cpumask_bits(src3p), nr_cpumask_bits); 588 } 589 590 /** 591 * cpumask_intersects - (*src1p & *src2p) != 0 592 * @src1p: the first input 593 * @src2p: the second input 594 */ 595 static inline bool cpumask_intersects(const struct cpumask *src1p, 596 const struct cpumask *src2p) 597 { 598 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p), 599 nr_cpumask_bits); 600 } 601 602 /** 603 * cpumask_subset - (*src1p & ~*src2p) == 0 604 * @src1p: the first input 605 * @src2p: the second input 606 * 607 * Returns true if *@src1p is a subset of *@src2p, else returns false 608 */ 609 static inline bool cpumask_subset(const struct cpumask *src1p, 610 const struct cpumask *src2p) 611 { 612 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), 613 nr_cpumask_bits); 614 } 615 616 /** 617 * cpumask_empty - *srcp == 0 618 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear. 619 */ 620 static inline bool cpumask_empty(const struct cpumask *srcp) 621 { 622 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits); 623 } 624 625 /** 626 * cpumask_full - *srcp == 0xFFFFFFFF... 627 * @srcp: the cpumask to that all cpus < nr_cpu_ids are set. 628 */ 629 static inline bool cpumask_full(const struct cpumask *srcp) 630 { 631 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits); 632 } 633 634 /** 635 * cpumask_weight - Count of bits in *srcp 636 * @srcp: the cpumask to count bits (< nr_cpu_ids) in. 637 */ 638 static inline unsigned int cpumask_weight(const struct cpumask *srcp) 639 { 640 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits); 641 } 642 643 /** 644 * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) 645 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. 646 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. 647 */ 648 static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, 649 const struct cpumask *srcp2) 650 { 651 return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); 652 } 653 654 /** 655 * cpumask_shift_right - *dstp = *srcp >> n 656 * @dstp: the cpumask result 657 * @srcp: the input to shift 658 * @n: the number of bits to shift by 659 */ 660 static inline void cpumask_shift_right(struct cpumask *dstp, 661 const struct cpumask *srcp, int n) 662 { 663 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n, 664 nr_cpumask_bits); 665 } 666 667 /** 668 * cpumask_shift_left - *dstp = *srcp << n 669 * @dstp: the cpumask result 670 * @srcp: the input to shift 671 * @n: the number of bits to shift by 672 */ 673 static inline void cpumask_shift_left(struct cpumask *dstp, 674 const struct cpumask *srcp, int n) 675 { 676 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n, 677 nr_cpumask_bits); 678 } 679 680 /** 681 * cpumask_copy - *dstp = *srcp 682 * @dstp: the result 683 * @srcp: the input cpumask 684 */ 685 static inline void cpumask_copy(struct cpumask *dstp, 686 const struct cpumask *srcp) 687 { 688 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits); 689 } 690 691 /** 692 * cpumask_any - pick a "random" cpu from *srcp 693 * @srcp: the input cpumask 694 * 695 * Returns >= nr_cpu_ids if no cpus set. 696 */ 697 #define cpumask_any(srcp) cpumask_first(srcp) 698 699 /** 700 * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2 701 * @mask1: the first input cpumask 702 * @mask2: the second input cpumask 703 * 704 * Returns >= nr_cpu_ids if no cpus set. 705 */ 706 #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2)) 707 708 /** 709 * cpumask_of - the cpumask containing just a given cpu 710 * @cpu: the cpu (<= nr_cpu_ids) 711 */ 712 #define cpumask_of(cpu) (get_cpu_mask(cpu)) 713 714 /** 715 * cpumask_parse_user - extract a cpumask from a user string 716 * @buf: the buffer to extract from 717 * @len: the length of the buffer 718 * @dstp: the cpumask to set. 719 * 720 * Returns -errno, or 0 for success. 721 */ 722 static inline int cpumask_parse_user(const char __user *buf, int len, 723 struct cpumask *dstp) 724 { 725 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); 726 } 727 728 /** 729 * cpumask_parselist_user - extract a cpumask from a user string 730 * @buf: the buffer to extract from 731 * @len: the length of the buffer 732 * @dstp: the cpumask to set. 733 * 734 * Returns -errno, or 0 for success. 735 */ 736 static inline int cpumask_parselist_user(const char __user *buf, int len, 737 struct cpumask *dstp) 738 { 739 return bitmap_parselist_user(buf, len, cpumask_bits(dstp), 740 nr_cpumask_bits); 741 } 742 743 /** 744 * cpumask_parse - extract a cpumask from a string 745 * @buf: the buffer to extract from 746 * @dstp: the cpumask to set. 747 * 748 * Returns -errno, or 0 for success. 749 */ 750 static inline int cpumask_parse(const char *buf, struct cpumask *dstp) 751 { 752 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits); 753 } 754 755 /** 756 * cpulist_parse - extract a cpumask from a user string of ranges 757 * @buf: the buffer to extract from 758 * @dstp: the cpumask to set. 759 * 760 * Returns -errno, or 0 for success. 761 */ 762 static inline int cpulist_parse(const char *buf, struct cpumask *dstp) 763 { 764 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); 765 } 766 767 /** 768 * cpumask_size - size to allocate for a 'struct cpumask' in bytes 769 */ 770 static inline unsigned int cpumask_size(void) 771 { 772 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long); 773 } 774 775 /* 776 * cpumask_var_t: struct cpumask for stack usage. 777 * 778 * Oh, the wicked games we play! In order to make kernel coding a 779 * little more difficult, we typedef cpumask_var_t to an array or a 780 * pointer: doing &mask on an array is a noop, so it still works. 781 * 782 * ie. 783 * cpumask_var_t tmpmask; 784 * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL)) 785 * return -ENOMEM; 786 * 787 * ... use 'tmpmask' like a normal struct cpumask * ... 788 * 789 * free_cpumask_var(tmpmask); 790 * 791 * 792 * However, one notable exception is there. alloc_cpumask_var() allocates 793 * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has 794 * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t. 795 * 796 * cpumask_var_t tmpmask; 797 * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL)) 798 * return -ENOMEM; 799 * 800 * var = *tmpmask; 801 * 802 * This code makes NR_CPUS length memcopy and brings to a memory corruption. 803 * cpumask_copy() provide safe copy functionality. 804 * 805 * Note that there is another evil here: If you define a cpumask_var_t 806 * as a percpu variable then the way to obtain the address of the cpumask 807 * structure differently influences what this_cpu_* operation needs to be 808 * used. Please use this_cpu_cpumask_var_t in those cases. The direct use 809 * of this_cpu_ptr() or this_cpu_read() will lead to failures when the 810 * other type of cpumask_var_t implementation is configured. 811 * 812 * Please also note that __cpumask_var_read_mostly can be used to declare 813 * a cpumask_var_t variable itself (not its content) as read mostly. 814 */ 815 #ifdef CONFIG_CPUMASK_OFFSTACK 816 typedef struct cpumask *cpumask_var_t; 817 818 #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x) 819 #define __cpumask_var_read_mostly __read_mostly 820 821 bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); 822 823 static inline 824 bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) 825 { 826 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); 827 } 828 829 /** 830 * alloc_cpumask_var - allocate a struct cpumask 831 * @mask: pointer to cpumask_var_t where the cpumask is returned 832 * @flags: GFP_ flags 833 * 834 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is 835 * a nop returning a constant 1 (in <linux/cpumask.h>). 836 * 837 * See alloc_cpumask_var_node. 838 */ 839 static inline 840 bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 841 { 842 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); 843 } 844 845 static inline 846 bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 847 { 848 return alloc_cpumask_var(mask, flags | __GFP_ZERO); 849 } 850 851 void alloc_bootmem_cpumask_var(cpumask_var_t *mask); 852 void free_cpumask_var(cpumask_var_t mask); 853 void free_bootmem_cpumask_var(cpumask_var_t mask); 854 855 static inline bool cpumask_available(cpumask_var_t mask) 856 { 857 return mask != NULL; 858 } 859 860 #else 861 typedef struct cpumask cpumask_var_t[1]; 862 863 #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) 864 #define __cpumask_var_read_mostly 865 866 static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 867 { 868 return true; 869 } 870 871 static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 872 int node) 873 { 874 return true; 875 } 876 877 static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 878 { 879 cpumask_clear(*mask); 880 return true; 881 } 882 883 static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 884 int node) 885 { 886 cpumask_clear(*mask); 887 return true; 888 } 889 890 static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) 891 { 892 } 893 894 static inline void free_cpumask_var(cpumask_var_t mask) 895 { 896 } 897 898 static inline void free_bootmem_cpumask_var(cpumask_var_t mask) 899 { 900 } 901 902 static inline bool cpumask_available(cpumask_var_t mask) 903 { 904 return true; 905 } 906 #endif /* CONFIG_CPUMASK_OFFSTACK */ 907 908 /* It's common to want to use cpu_all_mask in struct member initializers, 909 * so it has to refer to an address rather than a pointer. */ 910 extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); 911 #define cpu_all_mask to_cpumask(cpu_all_bits) 912 913 /* First bits of cpu_bit_bitmap are in fact unset. */ 914 #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0]) 915 916 #if NR_CPUS == 1 917 /* Uniprocessor: the possible/online/present masks are always "1" */ 918 #define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 919 #define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 920 #define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 921 #else 922 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) 923 #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) 924 #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) 925 #endif 926 927 /* Wrappers for arch boot code to manipulate normally-constant masks */ 928 void init_cpu_present(const struct cpumask *src); 929 void init_cpu_possible(const struct cpumask *src); 930 void init_cpu_online(const struct cpumask *src); 931 932 static inline void reset_cpu_possible_mask(void) 933 { 934 bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS); 935 } 936 937 static inline void 938 set_cpu_possible(unsigned int cpu, bool possible) 939 { 940 if (possible) 941 cpumask_set_cpu(cpu, &__cpu_possible_mask); 942 else 943 cpumask_clear_cpu(cpu, &__cpu_possible_mask); 944 } 945 946 static inline void 947 set_cpu_present(unsigned int cpu, bool present) 948 { 949 if (present) 950 cpumask_set_cpu(cpu, &__cpu_present_mask); 951 else 952 cpumask_clear_cpu(cpu, &__cpu_present_mask); 953 } 954 955 void set_cpu_online(unsigned int cpu, bool online); 956 957 static inline void 958 set_cpu_active(unsigned int cpu, bool active) 959 { 960 if (active) 961 cpumask_set_cpu(cpu, &__cpu_active_mask); 962 else 963 cpumask_clear_cpu(cpu, &__cpu_active_mask); 964 } 965 966 static inline void 967 set_cpu_dying(unsigned int cpu, bool dying) 968 { 969 if (dying) 970 cpumask_set_cpu(cpu, &__cpu_dying_mask); 971 else 972 cpumask_clear_cpu(cpu, &__cpu_dying_mask); 973 } 974 975 /** 976 * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask * 977 * @bitmap: the bitmap 978 * 979 * There are a few places where cpumask_var_t isn't appropriate and 980 * static cpumasks must be used (eg. very early boot), yet we don't 981 * expose the definition of 'struct cpumask'. 982 * 983 * This does the conversion, and can be used as a constant initializer. 984 */ 985 #define to_cpumask(bitmap) \ 986 ((struct cpumask *)(1 ? (bitmap) \ 987 : (void *)sizeof(__check_is_bitmap(bitmap)))) 988 989 static inline int __check_is_bitmap(const unsigned long *bitmap) 990 { 991 return 1; 992 } 993 994 /* 995 * Special-case data structure for "single bit set only" constant CPU masks. 996 * 997 * We pre-generate all the 64 (or 32) possible bit positions, with enough 998 * padding to the left and the right, and return the constant pointer 999 * appropriately offset. 1000 */ 1001 extern const unsigned long 1002 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; 1003 1004 static inline const struct cpumask *get_cpu_mask(unsigned int cpu) 1005 { 1006 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; 1007 p -= cpu / BITS_PER_LONG; 1008 return to_cpumask(p); 1009 } 1010 1011 #if NR_CPUS > 1 1012 /** 1013 * num_online_cpus() - Read the number of online CPUs 1014 * 1015 * Despite the fact that __num_online_cpus is of type atomic_t, this 1016 * interface gives only a momentary snapshot and is not protected against 1017 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held 1018 * region. 1019 */ 1020 static inline unsigned int num_online_cpus(void) 1021 { 1022 return atomic_read(&__num_online_cpus); 1023 } 1024 #define num_possible_cpus() cpumask_weight(cpu_possible_mask) 1025 #define num_present_cpus() cpumask_weight(cpu_present_mask) 1026 #define num_active_cpus() cpumask_weight(cpu_active_mask) 1027 1028 static inline bool cpu_online(unsigned int cpu) 1029 { 1030 return cpumask_test_cpu(cpu, cpu_online_mask); 1031 } 1032 1033 static inline bool cpu_possible(unsigned int cpu) 1034 { 1035 return cpumask_test_cpu(cpu, cpu_possible_mask); 1036 } 1037 1038 static inline bool cpu_present(unsigned int cpu) 1039 { 1040 return cpumask_test_cpu(cpu, cpu_present_mask); 1041 } 1042 1043 static inline bool cpu_active(unsigned int cpu) 1044 { 1045 return cpumask_test_cpu(cpu, cpu_active_mask); 1046 } 1047 1048 static inline bool cpu_dying(unsigned int cpu) 1049 { 1050 return cpumask_test_cpu(cpu, cpu_dying_mask); 1051 } 1052 1053 #else 1054 1055 #define num_online_cpus() 1U 1056 #define num_possible_cpus() 1U 1057 #define num_present_cpus() 1U 1058 #define num_active_cpus() 1U 1059 1060 static inline bool cpu_online(unsigned int cpu) 1061 { 1062 return cpu == 0; 1063 } 1064 1065 static inline bool cpu_possible(unsigned int cpu) 1066 { 1067 return cpu == 0; 1068 } 1069 1070 static inline bool cpu_present(unsigned int cpu) 1071 { 1072 return cpu == 0; 1073 } 1074 1075 static inline bool cpu_active(unsigned int cpu) 1076 { 1077 return cpu == 0; 1078 } 1079 1080 static inline bool cpu_dying(unsigned int cpu) 1081 { 1082 return false; 1083 } 1084 1085 #endif /* NR_CPUS > 1 */ 1086 1087 #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) 1088 1089 #if NR_CPUS <= BITS_PER_LONG 1090 #define CPU_BITS_ALL \ 1091 { \ 1092 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1093 } 1094 1095 #else /* NR_CPUS > BITS_PER_LONG */ 1096 1097 #define CPU_BITS_ALL \ 1098 { \ 1099 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 1100 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1101 } 1102 #endif /* NR_CPUS > BITS_PER_LONG */ 1103 1104 /** 1105 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either 1106 * as comma-separated list of cpus or hex values of cpumask 1107 * @list: indicates whether the cpumap must be list 1108 * @mask: the cpumask to copy 1109 * @buf: the buffer to copy into 1110 * 1111 * Returns the length of the (null-terminated) @buf string, zero if 1112 * nothing is copied. 1113 */ 1114 static inline ssize_t 1115 cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) 1116 { 1117 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask), 1118 nr_cpu_ids); 1119 } 1120 1121 /** 1122 * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as 1123 * hex values of cpumask 1124 * 1125 * @buf: the buffer to copy into 1126 * @mask: the cpumask to copy 1127 * @off: in the string from which we are copying, we copy to @buf 1128 * @count: the maximum number of bytes to print 1129 * 1130 * The function prints the cpumask into the buffer as hex values of 1131 * cpumask; Typically used by bin_attribute to export cpumask bitmask 1132 * ABI. 1133 * 1134 * Returns the length of how many bytes have been copied, excluding 1135 * terminating '\0'. 1136 */ 1137 static inline ssize_t 1138 cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, 1139 loff_t off, size_t count) 1140 { 1141 return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask), 1142 nr_cpu_ids, off, count) - 1; 1143 } 1144 1145 /** 1146 * cpumap_print_list_to_buf - copies the cpumask into the buffer as 1147 * comma-separated list of cpus 1148 * 1149 * Everything is same with the above cpumap_print_bitmask_to_buf() 1150 * except the print format. 1151 */ 1152 static inline ssize_t 1153 cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, 1154 loff_t off, size_t count) 1155 { 1156 return bitmap_print_list_to_buf(buf, cpumask_bits(mask), 1157 nr_cpu_ids, off, count) - 1; 1158 } 1159 1160 #if NR_CPUS <= BITS_PER_LONG 1161 #define CPU_MASK_ALL \ 1162 (cpumask_t) { { \ 1163 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1164 } } 1165 #else 1166 #define CPU_MASK_ALL \ 1167 (cpumask_t) { { \ 1168 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 1169 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1170 } } 1171 #endif /* NR_CPUS > BITS_PER_LONG */ 1172 1173 #define CPU_MASK_NONE \ 1174 (cpumask_t) { { \ 1175 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 1176 } } 1177 1178 #define CPU_MASK_CPU0 \ 1179 (cpumask_t) { { \ 1180 [0] = 1UL \ 1181 } } 1182 1183 /* 1184 * Provide a valid theoretical max size for cpumap and cpulist sysfs files 1185 * to avoid breaking userspace which may allocate a buffer based on the size 1186 * reported by e.g. fstat. 1187 * 1188 * for cpumap NR_CPUS * 9/32 - 1 should be an exact length. 1189 * 1190 * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up 1191 * to 2 orders of magnitude larger than 8192. And then we divide by 2 to 1192 * cover a worst-case of every other cpu being on one of two nodes for a 1193 * very large NR_CPUS. 1194 * 1195 * Use PAGE_SIZE as a minimum for smaller configurations while avoiding 1196 * unsigned comparison to -1. 1197 */ 1198 #define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \ 1199 ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE) 1200 #define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE) 1201 1202 #endif /* __LINUX_CPUMASK_H */ 1203