1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 * Copyright(c) 2019 Arm Limited 4 */ 5 6 #include <stdio.h> 7 #include <stdint.h> 8 #include <unistd.h> 9 #include <sys/queue.h> 10 11 #include <rte_memory.h> 12 #include <rte_per_lcore.h> 13 #include <rte_launch.h> 14 #include <rte_atomic.h> 15 #include <rte_eal.h> 16 #include <rte_lcore.h> 17 18 #include "test.h" 19 20 /* 21 * Atomic Variables 22 * ================ 23 * 24 * - The main test function performs four subtests. The first test 25 * checks that the usual inc/dec/add/sub functions are working 26 * correctly: 27 * 28 * - Initialize 16-bit, 32-bit and 64-bit atomic variables to specific 29 * values. 30 * 31 * - These variables are incremented and decremented on each core at 32 * the same time in ``test_atomic_usual()``. 33 * 34 * - The function checks that once all lcores finish their function, 35 * the value of the atomic variables are still the same. 36 * 37 * - The second test verifies the behavior of "test and set" functions. 38 * 39 * - Initialize 16-bit, 32-bit and 64-bit atomic variables to zero. 40 * 41 * - Invoke ``test_atomic_tas()`` on each lcore: before doing anything 42 * else. The cores are waiting a synchro using ``while 43 * (rte_atomic32_read(&val) == 0)`` which is triggered by the main test 44 * function. Then all cores do a 45 * ``rte_atomicXX_test_and_set()`` at the same time. If it is successful, 46 * it increments another atomic counter. 47 * 48 * - The main function checks that the atomic counter was incremented 49 * twice only (one for 16-bit, one for 32-bit and one for 64-bit values). 50 * 51 * - Test "add/sub and return" 52 * 53 * - Initialize 16-bit, 32-bit and 64-bit atomic variables to zero. 54 * 55 * - Invoke ``test_atomic_addsub_return()`` on each lcore. Before doing 56 * anything else, the cores are waiting a synchro. Each lcore does 57 * this operation several times:: 58 * 59 * tmp = rte_atomicXX_add_return(&a, 1); 60 * atomic_add(&count, tmp); 61 * tmp = rte_atomicXX_sub_return(&a, 1); 62 * atomic_sub(&count, tmp+1); 63 * 64 * - At the end of the test, the *count* value must be 0. 65 * 66 * - Test "128-bit compare and swap" (aarch64 and x86_64 only) 67 * 68 * - Initialize 128-bit atomic variables to zero. 69 * 70 * - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing 71 * anything else, the cores are waiting a synchro. Each lcore does 72 * these compare and swap (CAS) operations several times:: 73 * 74 * Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; 75 * Released CAS update counter.val[0] + 2; counter.val[1] + 1; 76 * Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; 77 * Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; 78 * 79 * - At the end of the test, the *count128* first 64-bit value and 80 * second 64-bit value differ by the total iterations. 81 */ 82 83 #define NUM_ATOMIC_TYPES 3 84 85 #define N 1000000 86 87 static rte_atomic16_t a16; 88 static rte_atomic32_t a32; 89 static rte_atomic64_t a64; 90 static rte_atomic64_t count; 91 static rte_atomic32_t synchro; 92 93 static int 94 test_atomic_usual(__attribute__((unused)) void *arg) 95 { 96 unsigned i; 97 98 while (rte_atomic32_read(&synchro) == 0) 99 ; 100 101 for (i = 0; i < N; i++) 102 rte_atomic16_inc(&a16); 103 for (i = 0; i < N; i++) 104 rte_atomic16_dec(&a16); 105 for (i = 0; i < (N / 5); i++) 106 rte_atomic16_add(&a16, 5); 107 for (i = 0; i < (N / 5); i++) 108 rte_atomic16_sub(&a16, 5); 109 110 for (i = 0; i < N; i++) 111 rte_atomic32_inc(&a32); 112 for (i = 0; i < N; i++) 113 rte_atomic32_dec(&a32); 114 for (i = 0; i < (N / 5); i++) 115 rte_atomic32_add(&a32, 5); 116 for (i = 0; i < (N / 5); i++) 117 rte_atomic32_sub(&a32, 5); 118 119 for (i = 0; i < N; i++) 120 rte_atomic64_inc(&a64); 121 for (i = 0; i < N; i++) 122 rte_atomic64_dec(&a64); 123 for (i = 0; i < (N / 5); i++) 124 rte_atomic64_add(&a64, 5); 125 for (i = 0; i < (N / 5); i++) 126 rte_atomic64_sub(&a64, 5); 127 128 return 0; 129 } 130 131 static int 132 test_atomic_tas(__attribute__((unused)) void *arg) 133 { 134 while (rte_atomic32_read(&synchro) == 0) 135 ; 136 137 if (rte_atomic16_test_and_set(&a16)) 138 rte_atomic64_inc(&count); 139 if (rte_atomic32_test_and_set(&a32)) 140 rte_atomic64_inc(&count); 141 if (rte_atomic64_test_and_set(&a64)) 142 rte_atomic64_inc(&count); 143 144 return 0; 145 } 146 147 static int 148 test_atomic_addsub_and_return(__attribute__((unused)) void *arg) 149 { 150 uint32_t tmp16; 151 uint32_t tmp32; 152 uint64_t tmp64; 153 unsigned i; 154 155 while (rte_atomic32_read(&synchro) == 0) 156 ; 157 158 for (i = 0; i < N; i++) { 159 tmp16 = rte_atomic16_add_return(&a16, 1); 160 rte_atomic64_add(&count, tmp16); 161 162 tmp16 = rte_atomic16_sub_return(&a16, 1); 163 rte_atomic64_sub(&count, tmp16+1); 164 165 tmp32 = rte_atomic32_add_return(&a32, 1); 166 rte_atomic64_add(&count, tmp32); 167 168 tmp32 = rte_atomic32_sub_return(&a32, 1); 169 rte_atomic64_sub(&count, tmp32+1); 170 171 tmp64 = rte_atomic64_add_return(&a64, 1); 172 rte_atomic64_add(&count, tmp64); 173 174 tmp64 = rte_atomic64_sub_return(&a64, 1); 175 rte_atomic64_sub(&count, tmp64+1); 176 } 177 178 return 0; 179 } 180 181 /* 182 * rte_atomic32_inc_and_test() would increase a 32 bits counter by one and then 183 * test if that counter is equal to 0. It would return true if the counter is 0 184 * and false if the counter is not 0. rte_atomic64_inc_and_test() could do the 185 * same thing but for a 64 bits counter. 186 * Here checks that if the 32/64 bits counter is equal to 0 after being atomically 187 * increased by one. If it is, increase the variable of "count" by one which would 188 * be checked as the result later. 189 * 190 */ 191 static int 192 test_atomic_inc_and_test(__attribute__((unused)) void *arg) 193 { 194 while (rte_atomic32_read(&synchro) == 0) 195 ; 196 197 if (rte_atomic16_inc_and_test(&a16)) { 198 rte_atomic64_inc(&count); 199 } 200 if (rte_atomic32_inc_and_test(&a32)) { 201 rte_atomic64_inc(&count); 202 } 203 if (rte_atomic64_inc_and_test(&a64)) { 204 rte_atomic64_inc(&count); 205 } 206 207 return 0; 208 } 209 210 /* 211 * rte_atomicXX_dec_and_test() should decrease a 32 bits counter by one and then 212 * test if that counter is equal to 0. It should return true if the counter is 0 213 * and false if the counter is not 0. 214 * This test checks if the counter is equal to 0 after being atomically 215 * decreased by one. If it is, increase the value of "count" by one which is to 216 * be checked as the result later. 217 */ 218 static int 219 test_atomic_dec_and_test(__attribute__((unused)) void *arg) 220 { 221 while (rte_atomic32_read(&synchro) == 0) 222 ; 223 224 if (rte_atomic16_dec_and_test(&a16)) 225 rte_atomic64_inc(&count); 226 227 if (rte_atomic32_dec_and_test(&a32)) 228 rte_atomic64_inc(&count); 229 230 if (rte_atomic64_dec_and_test(&a64)) 231 rte_atomic64_inc(&count); 232 233 return 0; 234 } 235 236 #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) 237 static rte_int128_t count128; 238 239 /* 240 * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 241 * bits by 2 and the second 64 bits by 1 in this test. It should return true 242 * if the compare exchange operation is successful. 243 * This test repeats 128 bits compare and swap operations N rounds. In each 244 * iteration it runs compare and swap operation with different memory models. 245 */ 246 static int 247 test_atomic128_cmp_exchange(__attribute__((unused)) void *arg) 248 { 249 rte_int128_t expected; 250 int success; 251 unsigned int i; 252 253 while (rte_atomic32_read(&synchro) == 0) 254 ; 255 256 expected = count128; 257 258 for (i = 0; i < N; i++) { 259 do { 260 rte_int128_t desired; 261 262 desired.val[0] = expected.val[0] + 2; 263 desired.val[1] = expected.val[1] + 1; 264 265 success = rte_atomic128_cmp_exchange(&count128, 266 &expected, &desired, 1, 267 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); 268 } while (success == 0); 269 270 do { 271 rte_int128_t desired; 272 273 desired.val[0] = expected.val[0] + 2; 274 desired.val[1] = expected.val[1] + 1; 275 276 success = rte_atomic128_cmp_exchange(&count128, 277 &expected, &desired, 1, 278 __ATOMIC_RELEASE, __ATOMIC_RELAXED); 279 } while (success == 0); 280 281 do { 282 rte_int128_t desired; 283 284 desired.val[0] = expected.val[0] + 2; 285 desired.val[1] = expected.val[1] + 1; 286 287 success = rte_atomic128_cmp_exchange(&count128, 288 &expected, &desired, 1, 289 __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); 290 } while (success == 0); 291 292 do { 293 rte_int128_t desired; 294 295 desired.val[0] = expected.val[0] + 2; 296 desired.val[1] = expected.val[1] + 1; 297 298 success = rte_atomic128_cmp_exchange(&count128, 299 &expected, &desired, 1, 300 __ATOMIC_RELAXED, __ATOMIC_RELAXED); 301 } while (success == 0); 302 } 303 304 return 0; 305 } 306 #endif 307 308 static int 309 test_atomic(void) 310 { 311 rte_atomic16_init(&a16); 312 rte_atomic32_init(&a32); 313 rte_atomic64_init(&a64); 314 rte_atomic64_init(&count); 315 rte_atomic32_init(&synchro); 316 317 rte_atomic16_set(&a16, 1UL << 10); 318 rte_atomic32_set(&a32, 1UL << 10); 319 rte_atomic64_set(&a64, 1ULL << 33); 320 321 printf("usual inc/dec/add/sub functions\n"); 322 323 rte_eal_mp_remote_launch(test_atomic_usual, NULL, SKIP_MASTER); 324 rte_atomic32_set(&synchro, 1); 325 rte_eal_mp_wait_lcore(); 326 rte_atomic32_set(&synchro, 0); 327 328 if (rte_atomic16_read(&a16) != 1UL << 10) { 329 printf("Atomic16 usual functions failed\n"); 330 return -1; 331 } 332 333 if (rte_atomic32_read(&a32) != 1UL << 10) { 334 printf("Atomic32 usual functions failed\n"); 335 return -1; 336 } 337 338 if (rte_atomic64_read(&a64) != 1ULL << 33) { 339 printf("Atomic64 usual functions failed\n"); 340 return -1; 341 } 342 343 printf("test and set\n"); 344 345 rte_atomic64_set(&a64, 0); 346 rte_atomic32_set(&a32, 0); 347 rte_atomic16_set(&a16, 0); 348 rte_atomic64_set(&count, 0); 349 rte_eal_mp_remote_launch(test_atomic_tas, NULL, SKIP_MASTER); 350 rte_atomic32_set(&synchro, 1); 351 rte_eal_mp_wait_lcore(); 352 rte_atomic32_set(&synchro, 0); 353 354 if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) { 355 printf("Atomic test and set failed\n"); 356 return -1; 357 } 358 359 printf("add/sub and return\n"); 360 361 rte_atomic64_set(&a64, 0); 362 rte_atomic32_set(&a32, 0); 363 rte_atomic16_set(&a16, 0); 364 rte_atomic64_set(&count, 0); 365 rte_eal_mp_remote_launch(test_atomic_addsub_and_return, NULL, 366 SKIP_MASTER); 367 rte_atomic32_set(&synchro, 1); 368 rte_eal_mp_wait_lcore(); 369 rte_atomic32_set(&synchro, 0); 370 371 if (rte_atomic64_read(&count) != 0) { 372 printf("Atomic add/sub+return failed\n"); 373 return -1; 374 } 375 376 /* 377 * Set a64, a32 and a16 with the same value of minus "number of slave 378 * lcores", launch all slave lcores to atomically increase by one and 379 * test them respectively. 380 * Each lcore should have only one chance to increase a64 by one and 381 * then check if it is equal to 0, but there should be only one lcore 382 * that finds that it is 0. It is similar for a32 and a16. 383 * Then a variable of "count", initialized to zero, is increased by 384 * one if a64, a32 or a16 is 0 after being increased and tested 385 * atomically. 386 * We can check if "count" is finally equal to 3 to see if all slave 387 * lcores performed "atomic inc and test" right. 388 */ 389 printf("inc and test\n"); 390 391 rte_atomic64_clear(&a64); 392 rte_atomic32_clear(&a32); 393 rte_atomic16_clear(&a16); 394 rte_atomic32_clear(&synchro); 395 rte_atomic64_clear(&count); 396 397 rte_atomic64_set(&a64, (int64_t)(1 - (int64_t)rte_lcore_count())); 398 rte_atomic32_set(&a32, (int32_t)(1 - (int32_t)rte_lcore_count())); 399 rte_atomic16_set(&a16, (int16_t)(1 - (int16_t)rte_lcore_count())); 400 rte_eal_mp_remote_launch(test_atomic_inc_and_test, NULL, SKIP_MASTER); 401 rte_atomic32_set(&synchro, 1); 402 rte_eal_mp_wait_lcore(); 403 rte_atomic32_clear(&synchro); 404 405 if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) { 406 printf("Atomic inc and test failed %d\n", (int)count.cnt); 407 return -1; 408 } 409 410 /* 411 * Same as above, but this time we set the values to "number of slave 412 * lcores", and decrement instead of increment. 413 */ 414 printf("dec and test\n"); 415 416 rte_atomic32_clear(&synchro); 417 rte_atomic64_clear(&count); 418 419 rte_atomic64_set(&a64, (int64_t)(rte_lcore_count() - 1)); 420 rte_atomic32_set(&a32, (int32_t)(rte_lcore_count() - 1)); 421 rte_atomic16_set(&a16, (int16_t)(rte_lcore_count() - 1)); 422 rte_eal_mp_remote_launch(test_atomic_dec_and_test, NULL, SKIP_MASTER); 423 rte_atomic32_set(&synchro, 1); 424 rte_eal_mp_wait_lcore(); 425 rte_atomic32_clear(&synchro); 426 427 if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) { 428 printf("Atomic dec and test failed\n"); 429 return -1; 430 } 431 432 #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) 433 /* 434 * This case tests the functionality of rte_atomic128_cmp_exchange 435 * API. It calls rte_atomic128_cmp_exchange with four kinds of memory 436 * models successively on each slave core. Once each 128-bit atomic 437 * compare and swap operation is successful, it updates the global 438 * 128-bit counter by 2 for the first 64-bit and 1 for the second 439 * 64-bit. Each slave core iterates this test N times. 440 * At the end of test, verify whether the first 64-bits of the 128-bit 441 * counter and the second 64bits is differ by the total iterations. If 442 * it is, the test passes. 443 */ 444 printf("128-bit compare and swap test\n"); 445 uint64_t iterations = 0; 446 447 rte_atomic32_clear(&synchro); 448 count128.val[0] = 0; 449 count128.val[1] = 0; 450 451 rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, 452 SKIP_MASTER); 453 rte_atomic32_set(&synchro, 1); 454 rte_eal_mp_wait_lcore(); 455 rte_atomic32_clear(&synchro); 456 457 iterations = count128.val[0] - count128.val[1]; 458 if (iterations != 4*N*(rte_lcore_count()-1)) { 459 printf("128-bit compare and swap failed\n"); 460 return -1; 461 } 462 #endif 463 464 return 0; 465 } 466 467 REGISTER_TEST_COMMAND(atomic_autotest, test_atomic); 468