1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from: src/sys/alpha/include/atomic.h,v 1.21.2.3 2005/10/06 18:12:05 jhb 29 * $FreeBSD$ 30 */ 31 32 #ifndef _MACHINE_ATOMIC_H_ 33 #define _MACHINE_ATOMIC_H_ 34 35 #ifndef _SYS_CDEFS_H_ 36 #error this file needs sys/cdefs.h as a prerequisite 37 #endif 38 39 /* 40 * Note: All the 64-bit atomic operations are only atomic when running 41 * in 64-bit mode. It is assumed that code compiled for n32 and n64 42 * fits into this definition and no further safeties are needed. 43 * 44 * It is also assumed that the add, subtract and other arithmetic is 45 * done on numbers not pointers. The special rules for n32 pointers 46 * do not have atomic operations defined for them, but generally shouldn't 47 * need atomic operations. 48 */ 49 #ifndef __MIPS_PLATFORM_SYNC_NOPS 50 #define __MIPS_PLATFORM_SYNC_NOPS "" 51 #endif 52 53 static __inline void 54 mips_sync(void) 55 { 56 __asm __volatile (".set noreorder\n" 57 "\tsync\n" 58 __MIPS_PLATFORM_SYNC_NOPS 59 ".set reorder\n" 60 : : : "memory"); 61 } 62 63 #define mb() mips_sync() 64 #define wmb() mips_sync() 65 #define rmb() mips_sync() 66 67 /* 68 * Various simple arithmetic on memory which is atomic in the presence 69 * of interrupts and SMP safe. 70 */ 71 72 void atomic_set_8(__volatile uint8_t *, uint8_t); 73 void atomic_clear_8(__volatile uint8_t *, uint8_t); 74 void atomic_add_8(__volatile uint8_t *, uint8_t); 75 void atomic_subtract_8(__volatile uint8_t *, uint8_t); 76 77 void atomic_set_16(__volatile uint16_t *, uint16_t); 78 void atomic_clear_16(__volatile uint16_t *, uint16_t); 79 void atomic_add_16(__volatile uint16_t *, uint16_t); 80 void atomic_subtract_16(__volatile uint16_t *, uint16_t); 81 82 static __inline void 83 atomic_set_32(__volatile uint32_t *p, uint32_t v) 84 { 85 uint32_t temp; 86 87 __asm __volatile ( 88 "1:\tll %0, %3\n\t" /* load old value */ 89 "or %0, %2, %0\n\t" /* calculate new value */ 90 "sc %0, %1\n\t" /* attempt to store */ 91 "beqz %0, 1b\n\t" /* spin if failed */ 92 : "=&r" (temp), "=m" (*p) 93 : "r" (v), "m" (*p) 94 : "memory"); 95 96 } 97 98 static __inline void 99 atomic_clear_32(__volatile uint32_t *p, uint32_t v) 100 { 101 uint32_t temp; 102 v = ~v; 103 104 __asm __volatile ( 105 "1:\tll %0, %3\n\t" /* load old value */ 106 "and %0, %2, %0\n\t" /* calculate new value */ 107 "sc %0, %1\n\t" /* attempt to store */ 108 "beqz %0, 1b\n\t" /* spin if failed */ 109 : "=&r" (temp), "=m" (*p) 110 : "r" (v), "m" (*p) 111 : "memory"); 112 } 113 114 static __inline void 115 atomic_add_32(__volatile uint32_t *p, uint32_t v) 116 { 117 uint32_t temp; 118 119 __asm __volatile ( 120 "1:\tll %0, %3\n\t" /* load old value */ 121 "addu %0, %2, %0\n\t" /* calculate new value */ 122 "sc %0, %1\n\t" /* attempt to store */ 123 "beqz %0, 1b\n\t" /* spin if failed */ 124 : "=&r" (temp), "=m" (*p) 125 : "r" (v), "m" (*p) 126 : "memory"); 127 } 128 129 static __inline void 130 atomic_subtract_32(__volatile uint32_t *p, uint32_t v) 131 { 132 uint32_t temp; 133 134 __asm __volatile ( 135 "1:\tll %0, %3\n\t" /* load old value */ 136 "subu %0, %2\n\t" /* calculate new value */ 137 "sc %0, %1\n\t" /* attempt to store */ 138 "beqz %0, 1b\n\t" /* spin if failed */ 139 : "=&r" (temp), "=m" (*p) 140 : "r" (v), "m" (*p) 141 : "memory"); 142 } 143 144 static __inline uint32_t 145 atomic_readandclear_32(__volatile uint32_t *addr) 146 { 147 uint32_t result,temp; 148 149 __asm __volatile ( 150 "1:\tll %0,%3\n\t" /* load current value, asserting lock */ 151 "li %1,0\n\t" /* value to store */ 152 "sc %1,%2\n\t" /* attempt to store */ 153 "beqz %1, 1b\n\t" /* if the store failed, spin */ 154 : "=&r"(result), "=&r"(temp), "=m" (*addr) 155 : "m" (*addr) 156 : "memory"); 157 158 return result; 159 } 160 161 static __inline uint32_t 162 atomic_readandset_32(__volatile uint32_t *addr, uint32_t value) 163 { 164 uint32_t result,temp; 165 166 __asm __volatile ( 167 "1:\tll %0,%3\n\t" /* load current value, asserting lock */ 168 "or %1,$0,%4\n\t" 169 "sc %1,%2\n\t" /* attempt to store */ 170 "beqz %1, 1b\n\t" /* if the store failed, spin */ 171 : "=&r"(result), "=&r"(temp), "=m" (*addr) 172 : "m" (*addr), "r" (value) 173 : "memory"); 174 175 return result; 176 } 177 178 #if defined(__mips_n64) || defined(__mips_n32) 179 static __inline void 180 atomic_set_64(__volatile uint64_t *p, uint64_t v) 181 { 182 uint64_t temp; 183 184 __asm __volatile ( 185 "1:\n\t" 186 "lld %0, %3\n\t" /* load old value */ 187 "or %0, %2, %0\n\t" /* calculate new value */ 188 "scd %0, %1\n\t" /* attempt to store */ 189 "beqz %0, 1b\n\t" /* spin if failed */ 190 : "=&r" (temp), "=m" (*p) 191 : "r" (v), "m" (*p) 192 : "memory"); 193 194 } 195 196 static __inline void 197 atomic_clear_64(__volatile uint64_t *p, uint64_t v) 198 { 199 uint64_t temp; 200 v = ~v; 201 202 __asm __volatile ( 203 "1:\n\t" 204 "lld %0, %3\n\t" /* load old value */ 205 "and %0, %2, %0\n\t" /* calculate new value */ 206 "scd %0, %1\n\t" /* attempt to store */ 207 "beqz %0, 1b\n\t" /* spin if failed */ 208 : "=&r" (temp), "=m" (*p) 209 : "r" (v), "m" (*p) 210 : "memory"); 211 } 212 213 static __inline void 214 atomic_add_64(__volatile uint64_t *p, uint64_t v) 215 { 216 uint64_t temp; 217 218 __asm __volatile ( 219 "1:\n\t" 220 "lld %0, %3\n\t" /* load old value */ 221 "daddu %0, %2, %0\n\t" /* calculate new value */ 222 "scd %0, %1\n\t" /* attempt to store */ 223 "beqz %0, 1b\n\t" /* spin if failed */ 224 : "=&r" (temp), "=m" (*p) 225 : "r" (v), "m" (*p) 226 : "memory"); 227 } 228 229 static __inline void 230 atomic_subtract_64(__volatile uint64_t *p, uint64_t v) 231 { 232 uint64_t temp; 233 234 __asm __volatile ( 235 "1:\n\t" 236 "lld %0, %3\n\t" /* load old value */ 237 "dsubu %0, %2\n\t" /* calculate new value */ 238 "scd %0, %1\n\t" /* attempt to store */ 239 "beqz %0, 1b\n\t" /* spin if failed */ 240 : "=&r" (temp), "=m" (*p) 241 : "r" (v), "m" (*p) 242 : "memory"); 243 } 244 245 static __inline uint64_t 246 atomic_readandclear_64(__volatile uint64_t *addr) 247 { 248 uint64_t result,temp; 249 250 __asm __volatile ( 251 "1:\n\t" 252 "lld %0, %3\n\t" /* load old value */ 253 "li %1, 0\n\t" /* value to store */ 254 "scd %1, %2\n\t" /* attempt to store */ 255 "beqz %1, 1b\n\t" /* if the store failed, spin */ 256 : "=&r"(result), "=&r"(temp), "=m" (*addr) 257 : "m" (*addr) 258 : "memory"); 259 260 return result; 261 } 262 263 static __inline uint64_t 264 atomic_readandset_64(__volatile uint64_t *addr, uint64_t value) 265 { 266 uint64_t result,temp; 267 268 __asm __volatile ( 269 "1:\n\t" 270 "lld %0,%3\n\t" /* Load old value*/ 271 "or %1,$0,%4\n\t" 272 "scd %1,%2\n\t" /* attempt to store */ 273 "beqz %1, 1b\n\t" /* if the store failed, spin */ 274 : "=&r"(result), "=&r"(temp), "=m" (*addr) 275 : "m" (*addr), "r" (value) 276 : "memory"); 277 278 return result; 279 } 280 #endif 281 282 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 283 static __inline void \ 284 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 285 { \ 286 atomic_##NAME##_##WIDTH(p, v); \ 287 mips_sync(); \ 288 } \ 289 \ 290 static __inline void \ 291 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 292 { \ 293 mips_sync(); \ 294 atomic_##NAME##_##WIDTH(p, v); \ 295 } 296 297 /* Variants of simple arithmetic with memory barriers. */ 298 ATOMIC_ACQ_REL(set, 8) 299 ATOMIC_ACQ_REL(clear, 8) 300 ATOMIC_ACQ_REL(add, 8) 301 ATOMIC_ACQ_REL(subtract, 8) 302 ATOMIC_ACQ_REL(set, 16) 303 ATOMIC_ACQ_REL(clear, 16) 304 ATOMIC_ACQ_REL(add, 16) 305 ATOMIC_ACQ_REL(subtract, 16) 306 ATOMIC_ACQ_REL(set, 32) 307 ATOMIC_ACQ_REL(clear, 32) 308 ATOMIC_ACQ_REL(add, 32) 309 ATOMIC_ACQ_REL(subtract, 32) 310 #if defined(__mips_n64) || defined(__mips_n32) 311 ATOMIC_ACQ_REL(set, 64) 312 ATOMIC_ACQ_REL(clear, 64) 313 ATOMIC_ACQ_REL(add, 64) 314 ATOMIC_ACQ_REL(subtract, 64) 315 #endif 316 317 #undef ATOMIC_ACQ_REL 318 319 /* 320 * We assume that a = b will do atomic loads and stores. 321 */ 322 #define ATOMIC_STORE_LOAD(WIDTH) \ 323 static __inline uint##WIDTH##_t \ 324 atomic_load_acq_##WIDTH(__volatile uint##WIDTH##_t *p) \ 325 { \ 326 uint##WIDTH##_t v; \ 327 \ 328 v = *p; \ 329 mips_sync(); \ 330 return (v); \ 331 } \ 332 \ 333 static __inline void \ 334 atomic_store_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 335 { \ 336 mips_sync(); \ 337 *p = v; \ 338 } 339 340 ATOMIC_STORE_LOAD(32) 341 ATOMIC_STORE_LOAD(64) 342 #if !defined(__mips_n64) && !defined(__mips_n32) 343 void atomic_store_64(__volatile uint64_t *, uint64_t *); 344 void atomic_load_64(__volatile uint64_t *, uint64_t *); 345 #else 346 static __inline void 347 atomic_store_64(__volatile uint64_t *p, uint64_t *v) 348 { 349 *p = *v; 350 } 351 352 static __inline void 353 atomic_load_64(__volatile uint64_t *p, uint64_t *v) 354 { 355 *v = *p; 356 } 357 #endif 358 359 #undef ATOMIC_STORE_LOAD 360 361 /* 362 * Atomically compare the value stored at *p with cmpval and if the 363 * two values are equal, update the value of *p with newval. Returns 364 * zero if the compare failed, nonzero otherwise. 365 */ 366 static __inline uint32_t 367 atomic_cmpset_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 368 { 369 uint32_t ret; 370 371 __asm __volatile ( 372 "1:\tll %0, %4\n\t" /* load old value */ 373 "bne %0, %2, 2f\n\t" /* compare */ 374 "move %0, %3\n\t" /* value to store */ 375 "sc %0, %1\n\t" /* attempt to store */ 376 "beqz %0, 1b\n\t" /* if it failed, spin */ 377 "j 3f\n\t" 378 "2:\n\t" 379 "li %0, 0\n\t" 380 "3:\n" 381 : "=&r" (ret), "=m" (*p) 382 : "r" (cmpval), "r" (newval), "m" (*p) 383 : "memory"); 384 385 return ret; 386 } 387 388 /* 389 * Atomically compare the value stored at *p with cmpval and if the 390 * two values are equal, update the value of *p with newval. Returns 391 * zero if the compare failed, nonzero otherwise. 392 */ 393 static __inline uint32_t 394 atomic_cmpset_acq_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 395 { 396 int retval; 397 398 retval = atomic_cmpset_32(p, cmpval, newval); 399 mips_sync(); 400 return (retval); 401 } 402 403 static __inline uint32_t 404 atomic_cmpset_rel_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 405 { 406 mips_sync(); 407 return (atomic_cmpset_32(p, cmpval, newval)); 408 } 409 410 static __inline uint32_t 411 atomic_fcmpset_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 412 { 413 uint32_t ret; 414 415 __asm __volatile ( 416 "1:\n\t" 417 "ll %0, %1\n\t" /* load old value */ 418 "bne %0, %4, 2f\n\t" /* compare */ 419 "move %0, %3\n\t" /* value to store */ 420 "sc %0, %1\n\t" /* attempt to store */ 421 "beqz %0, 1b\n\t" /* if it failed, spin */ 422 "j 3f\n\t" 423 "2:\n\t" 424 "sw %0, %2\n\t" /* save old value */ 425 "li %0, 0\n\t" 426 "3:\n" 427 : "=&r" (ret), "+m" (*p), "=m" (*cmpval) 428 : "r" (newval), "r" (*cmpval) 429 : "memory"); 430 return ret; 431 } 432 433 static __inline uint32_t 434 atomic_fcmpset_acq_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 435 { 436 int retval; 437 438 retval = atomic_fcmpset_32(p, cmpval, newval); 439 mips_sync(); 440 return (retval); 441 } 442 443 static __inline uint32_t 444 atomic_fcmpset_rel_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 445 { 446 mips_sync(); 447 return (atomic_fcmpset_32(p, cmpval, newval)); 448 } 449 450 /* 451 * Atomically add the value of v to the integer pointed to by p and return 452 * the previous value of *p. 453 */ 454 static __inline uint32_t 455 atomic_fetchadd_32(__volatile uint32_t *p, uint32_t v) 456 { 457 uint32_t value, temp; 458 459 __asm __volatile ( 460 "1:\tll %0, %1\n\t" /* load old value */ 461 "addu %2, %3, %0\n\t" /* calculate new value */ 462 "sc %2, %1\n\t" /* attempt to store */ 463 "beqz %2, 1b\n\t" /* spin if failed */ 464 : "=&r" (value), "=m" (*p), "=&r" (temp) 465 : "r" (v), "m" (*p)); 466 return (value); 467 } 468 469 #if defined(__mips_n64) || defined(__mips_n32) 470 /* 471 * Atomically compare the value stored at *p with cmpval and if the 472 * two values are equal, update the value of *p with newval. Returns 473 * zero if the compare failed, nonzero otherwise. 474 */ 475 static __inline uint64_t 476 atomic_cmpset_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 477 { 478 uint64_t ret; 479 480 __asm __volatile ( 481 "1:\n\t" 482 "lld %0, %4\n\t" /* load old value */ 483 "bne %0, %2, 2f\n\t" /* compare */ 484 "move %0, %3\n\t" /* value to store */ 485 "scd %0, %1\n\t" /* attempt to store */ 486 "beqz %0, 1b\n\t" /* if it failed, spin */ 487 "j 3f\n\t" 488 "2:\n\t" 489 "li %0, 0\n\t" 490 "3:\n" 491 : "=&r" (ret), "=m" (*p) 492 : "r" (cmpval), "r" (newval), "m" (*p) 493 : "memory"); 494 495 return ret; 496 } 497 498 /* 499 * Atomically compare the value stored at *p with cmpval and if the 500 * two values are equal, update the value of *p with newval. Returns 501 * zero if the compare failed, nonzero otherwise. 502 */ 503 static __inline uint64_t 504 atomic_cmpset_acq_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 505 { 506 int retval; 507 508 retval = atomic_cmpset_64(p, cmpval, newval); 509 mips_sync(); 510 return (retval); 511 } 512 513 static __inline uint64_t 514 atomic_cmpset_rel_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 515 { 516 mips_sync(); 517 return (atomic_cmpset_64(p, cmpval, newval)); 518 } 519 520 static __inline uint32_t 521 atomic_fcmpset_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 522 { 523 uint32_t ret; 524 525 __asm __volatile ( 526 "1:\n\t" 527 "lld %0, %1\n\t" /* load old value */ 528 "bne %0, %4, 2f\n\t" /* compare */ 529 "move %0, %3\n\t" /* value to store */ 530 "scd %0, %1\n\t" /* attempt to store */ 531 "beqz %0, 1b\n\t" /* if it failed, spin */ 532 "j 3f\n\t" 533 "2:\n\t" 534 "sd %0, %2\n\t" /* save old value */ 535 "li %0, 0\n\t" 536 "3:\n" 537 : "=&r" (ret), "+m" (*p), "=m" (*cmpval) 538 : "r" (newval), "r" (*cmpval) 539 : "memory"); 540 541 return ret; 542 } 543 544 static __inline uint64_t 545 atomic_fcmpset_acq_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 546 { 547 int retval; 548 549 retval = atomic_fcmpset_64(p, cmpval, newval); 550 mips_sync(); 551 return (retval); 552 } 553 554 static __inline uint64_t 555 atomic_fcmpset_rel_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 556 { 557 mips_sync(); 558 return (atomic_fcmpset_64(p, cmpval, newval)); 559 } 560 561 /* 562 * Atomically add the value of v to the integer pointed to by p and return 563 * the previous value of *p. 564 */ 565 static __inline uint64_t 566 atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v) 567 { 568 uint64_t value, temp; 569 570 __asm __volatile ( 571 "1:\n\t" 572 "lld %0, %1\n\t" /* load old value */ 573 "daddu %2, %3, %0\n\t" /* calculate new value */ 574 "scd %2, %1\n\t" /* attempt to store */ 575 "beqz %2, 1b\n\t" /* spin if failed */ 576 : "=&r" (value), "=m" (*p), "=&r" (temp) 577 : "r" (v), "m" (*p)); 578 return (value); 579 } 580 #endif 581 582 static __inline void 583 atomic_thread_fence_acq(void) 584 { 585 586 mips_sync(); 587 } 588 589 static __inline void 590 atomic_thread_fence_rel(void) 591 { 592 593 mips_sync(); 594 } 595 596 static __inline void 597 atomic_thread_fence_acq_rel(void) 598 { 599 600 mips_sync(); 601 } 602 603 static __inline void 604 atomic_thread_fence_seq_cst(void) 605 { 606 607 mips_sync(); 608 } 609 610 /* Operations on chars. */ 611 #define atomic_set_char atomic_set_8 612 #define atomic_set_acq_char atomic_set_acq_8 613 #define atomic_set_rel_char atomic_set_rel_8 614 #define atomic_clear_char atomic_clear_8 615 #define atomic_clear_acq_char atomic_clear_acq_8 616 #define atomic_clear_rel_char atomic_clear_rel_8 617 #define atomic_add_char atomic_add_8 618 #define atomic_add_acq_char atomic_add_acq_8 619 #define atomic_add_rel_char atomic_add_rel_8 620 #define atomic_subtract_char atomic_subtract_8 621 #define atomic_subtract_acq_char atomic_subtract_acq_8 622 #define atomic_subtract_rel_char atomic_subtract_rel_8 623 624 /* Operations on shorts. */ 625 #define atomic_set_short atomic_set_16 626 #define atomic_set_acq_short atomic_set_acq_16 627 #define atomic_set_rel_short atomic_set_rel_16 628 #define atomic_clear_short atomic_clear_16 629 #define atomic_clear_acq_short atomic_clear_acq_16 630 #define atomic_clear_rel_short atomic_clear_rel_16 631 #define atomic_add_short atomic_add_16 632 #define atomic_add_acq_short atomic_add_acq_16 633 #define atomic_add_rel_short atomic_add_rel_16 634 #define atomic_subtract_short atomic_subtract_16 635 #define atomic_subtract_acq_short atomic_subtract_acq_16 636 #define atomic_subtract_rel_short atomic_subtract_rel_16 637 638 /* Operations on ints. */ 639 #define atomic_set_int atomic_set_32 640 #define atomic_set_acq_int atomic_set_acq_32 641 #define atomic_set_rel_int atomic_set_rel_32 642 #define atomic_clear_int atomic_clear_32 643 #define atomic_clear_acq_int atomic_clear_acq_32 644 #define atomic_clear_rel_int atomic_clear_rel_32 645 #define atomic_add_int atomic_add_32 646 #define atomic_add_acq_int atomic_add_acq_32 647 #define atomic_add_rel_int atomic_add_rel_32 648 #define atomic_subtract_int atomic_subtract_32 649 #define atomic_subtract_acq_int atomic_subtract_acq_32 650 #define atomic_subtract_rel_int atomic_subtract_rel_32 651 #define atomic_cmpset_int atomic_cmpset_32 652 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 653 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 654 #define atomic_fcmpset_int atomic_fcmpset_32 655 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 656 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 657 #define atomic_load_acq_int atomic_load_acq_32 658 #define atomic_store_rel_int atomic_store_rel_32 659 #define atomic_readandclear_int atomic_readandclear_32 660 #define atomic_readandset_int atomic_readandset_32 661 #define atomic_fetchadd_int atomic_fetchadd_32 662 663 /* 664 * I think the following is right, even for n32. For n32 the pointers 665 * are still 32-bits, so we need to operate on them as 32-bit quantities, 666 * even though they are sign extended in operation. For longs, there's 667 * no question because they are always 32-bits. 668 */ 669 #ifdef __mips_n64 670 /* Operations on longs. */ 671 #define atomic_set_long atomic_set_64 672 #define atomic_set_acq_long atomic_set_acq_64 673 #define atomic_set_rel_long atomic_set_rel_64 674 #define atomic_clear_long atomic_clear_64 675 #define atomic_clear_acq_long atomic_clear_acq_64 676 #define atomic_clear_rel_long atomic_clear_rel_64 677 #define atomic_add_long atomic_add_64 678 #define atomic_add_acq_long atomic_add_acq_64 679 #define atomic_add_rel_long atomic_add_rel_64 680 #define atomic_subtract_long atomic_subtract_64 681 #define atomic_subtract_acq_long atomic_subtract_acq_64 682 #define atomic_subtract_rel_long atomic_subtract_rel_64 683 #define atomic_cmpset_long atomic_cmpset_64 684 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 685 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 686 #define atomic_fcmpset_long atomic_fcmpset_64 687 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 688 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 689 #define atomic_load_acq_long atomic_load_acq_64 690 #define atomic_store_rel_long atomic_store_rel_64 691 #define atomic_fetchadd_long atomic_fetchadd_64 692 #define atomic_readandclear_long atomic_readandclear_64 693 694 #else /* !__mips_n64 */ 695 696 /* Operations on longs. */ 697 #define atomic_set_long(p, v) \ 698 atomic_set_32((volatile u_int *)(p), (u_int)(v)) 699 #define atomic_set_acq_long(p, v) \ 700 atomic_set_acq_32((volatile u_int *)(p), (u_int)(v)) 701 #define atomic_set_rel_long(p, v) \ 702 atomic_set_rel_32((volatile u_int *)(p), (u_int)(v)) 703 #define atomic_clear_long(p, v) \ 704 atomic_clear_32((volatile u_int *)(p), (u_int)(v)) 705 #define atomic_clear_acq_long(p, v) \ 706 atomic_clear_acq_32((volatile u_int *)(p), (u_int)(v)) 707 #define atomic_clear_rel_long(p, v) \ 708 atomic_clear_rel_32((volatile u_int *)(p), (u_int)(v)) 709 #define atomic_add_long(p, v) \ 710 atomic_add_32((volatile u_int *)(p), (u_int)(v)) 711 #define atomic_add_acq_long(p, v) \ 712 atomic_add_32((volatile u_int *)(p), (u_int)(v)) 713 #define atomic_add_rel_long(p, v) \ 714 atomic_add_32((volatile u_int *)(p), (u_int)(v)) 715 #define atomic_subtract_long(p, v) \ 716 atomic_subtract_32((volatile u_int *)(p), (u_int)(v)) 717 #define atomic_subtract_acq_long(p, v) \ 718 atomic_subtract_acq_32((volatile u_int *)(p), (u_int)(v)) 719 #define atomic_subtract_rel_long(p, v) \ 720 atomic_subtract_rel_32((volatile u_int *)(p), (u_int)(v)) 721 #define atomic_cmpset_long(p, cmpval, newval) \ 722 atomic_cmpset_32((volatile u_int *)(p), (u_int)(cmpval), \ 723 (u_int)(newval)) 724 #define atomic_cmpset_acq_long(p, cmpval, newval) \ 725 atomic_cmpset_acq_32((volatile u_int *)(p), (u_int)(cmpval), \ 726 (u_int)(newval)) 727 #define atomic_cmpset_rel_long(p, cmpval, newval) \ 728 atomic_cmpset_rel_32((volatile u_int *)(p), (u_int)(cmpval), \ 729 (u_int)(newval)) 730 #define atomic_fcmpset_long(p, cmpval, newval) \ 731 atomic_fcmpset_32((volatile u_int *)(p), (u_int *)(cmpval), \ 732 (u_int)(newval)) 733 #define atomic_fcmpset_acq_long(p, cmpval, newval) \ 734 atomic_fcmpset_acq_32((volatile u_int *)(p), (u_int *)(cmpval), \ 735 (u_int)(newval)) 736 #define atomic_fcmpset_rel_long(p, cmpval, newval) \ 737 atomic_fcmpset_rel_32((volatile u_int *)(p), (u_int *)(cmpval), \ 738 (u_int)(newval)) 739 #define atomic_load_acq_long(p) \ 740 (u_long)atomic_load_acq_32((volatile u_int *)(p)) 741 #define atomic_store_rel_long(p, v) \ 742 atomic_store_rel_32((volatile u_int *)(p), (u_int)(v)) 743 #define atomic_fetchadd_long(p, v) \ 744 atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) 745 #define atomic_readandclear_long(p) \ 746 atomic_readandclear_32((volatile u_int *)(p)) 747 748 #endif /* __mips_n64 */ 749 750 /* Operations on pointers. */ 751 #define atomic_set_ptr atomic_set_long 752 #define atomic_set_acq_ptr atomic_set_acq_long 753 #define atomic_set_rel_ptr atomic_set_rel_long 754 #define atomic_clear_ptr atomic_clear_long 755 #define atomic_clear_acq_ptr atomic_clear_acq_long 756 #define atomic_clear_rel_ptr atomic_clear_rel_long 757 #define atomic_add_ptr atomic_add_long 758 #define atomic_add_acq_ptr atomic_add_acq_long 759 #define atomic_add_rel_ptr atomic_add_rel_long 760 #define atomic_subtract_ptr atomic_subtract_long 761 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 762 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 763 #define atomic_cmpset_ptr atomic_cmpset_long 764 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 765 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 766 #define atomic_fcmpset_ptr atomic_fcmpset_long 767 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 768 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 769 #define atomic_load_acq_ptr atomic_load_acq_long 770 #define atomic_store_rel_ptr atomic_store_rel_long 771 #define atomic_readandclear_ptr atomic_readandclear_long 772 773 #endif /* ! _MACHINE_ATOMIC_H_ */ 774