1 #ifndef __LINUX_SPINLOCK_API_SMP_H 2 #define __LINUX_SPINLOCK_API_SMP_H 3 4 #ifndef __LINUX_SPINLOCK_H 5 # error "please don't include this file directly" 6 #endif 7 8 /* 9 * include/linux/spinlock_api_smp.h 10 * 11 * spinlock API declarations on SMP (and debug) 12 * (implemented in kernel/spinlock.c) 13 * 14 * portions Copyright 2005, Red Hat, Inc., Ingo Molnar 15 * Released under the General Public License (GPL). 16 */ 17 18 int in_lock_functions(unsigned long addr); 19 20 #define assert_spin_locked(x) BUG_ON(!spin_is_locked(x)) 21 22 void __lockfunc _spin_lock(spinlock_t *lock) __acquires(lock); 23 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) 24 __acquires(lock); 25 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *map) 26 __acquires(lock); 27 void __lockfunc _read_lock(rwlock_t *lock) __acquires(lock); 28 void __lockfunc _write_lock(rwlock_t *lock) __acquires(lock); 29 void __lockfunc _spin_lock_bh(spinlock_t *lock) __acquires(lock); 30 void __lockfunc _read_lock_bh(rwlock_t *lock) __acquires(lock); 31 void __lockfunc _write_lock_bh(rwlock_t *lock) __acquires(lock); 32 void __lockfunc _spin_lock_irq(spinlock_t *lock) __acquires(lock); 33 void __lockfunc _read_lock_irq(rwlock_t *lock) __acquires(lock); 34 void __lockfunc _write_lock_irq(rwlock_t *lock) __acquires(lock); 35 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) 36 __acquires(lock); 37 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass) 38 __acquires(lock); 39 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) 40 __acquires(lock); 41 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) 42 __acquires(lock); 43 int __lockfunc _spin_trylock(spinlock_t *lock); 44 int __lockfunc _read_trylock(rwlock_t *lock); 45 int __lockfunc _write_trylock(rwlock_t *lock); 46 int __lockfunc _spin_trylock_bh(spinlock_t *lock); 47 void __lockfunc _spin_unlock(spinlock_t *lock) __releases(lock); 48 void __lockfunc _read_unlock(rwlock_t *lock) __releases(lock); 49 void __lockfunc _write_unlock(rwlock_t *lock) __releases(lock); 50 void __lockfunc _spin_unlock_bh(spinlock_t *lock) __releases(lock); 51 void __lockfunc _read_unlock_bh(rwlock_t *lock) __releases(lock); 52 void __lockfunc _write_unlock_bh(rwlock_t *lock) __releases(lock); 53 void __lockfunc _spin_unlock_irq(spinlock_t *lock) __releases(lock); 54 void __lockfunc _read_unlock_irq(rwlock_t *lock) __releases(lock); 55 void __lockfunc _write_unlock_irq(rwlock_t *lock) __releases(lock); 56 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) 57 __releases(lock); 58 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 59 __releases(lock); 60 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 61 __releases(lock); 62 63 #ifndef CONFIG_DEBUG_SPINLOCK 64 #ifndef CONFIG_GENERIC_LOCKBREAK 65 66 #ifdef __always_inline__spin_lock 67 #define _spin_lock(lock) __spin_lock(lock) 68 #endif 69 70 #ifdef __always_inline__read_lock 71 #define _read_lock(lock) __read_lock(lock) 72 #endif 73 74 #ifdef __always_inline__write_lock 75 #define _write_lock(lock) __write_lock(lock) 76 #endif 77 78 #ifdef __always_inline__spin_lock_bh 79 #define _spin_lock_bh(lock) __spin_lock_bh(lock) 80 #endif 81 82 #ifdef __always_inline__read_lock_bh 83 #define _read_lock_bh(lock) __read_lock_bh(lock) 84 #endif 85 86 #ifdef __always_inline__write_lock_bh 87 #define _write_lock_bh(lock) __write_lock_bh(lock) 88 #endif 89 90 #ifdef __always_inline__spin_lock_irq 91 #define _spin_lock_irq(lock) __spin_lock_irq(lock) 92 #endif 93 94 #ifdef __always_inline__read_lock_irq 95 #define _read_lock_irq(lock) __read_lock_irq(lock) 96 #endif 97 98 #ifdef __always_inline__write_lock_irq 99 #define _write_lock_irq(lock) __write_lock_irq(lock) 100 #endif 101 102 #ifdef __always_inline__spin_lock_irqsave 103 #define _spin_lock_irqsave(lock) __spin_lock_irqsave(lock) 104 #endif 105 106 #ifdef __always_inline__read_lock_irqsave 107 #define _read_lock_irqsave(lock) __read_lock_irqsave(lock) 108 #endif 109 110 #ifdef __always_inline__write_lock_irqsave 111 #define _write_lock_irqsave(lock) __write_lock_irqsave(lock) 112 #endif 113 114 #endif /* !CONFIG_GENERIC_LOCKBREAK */ 115 116 #ifdef __always_inline__spin_trylock 117 #define _spin_trylock(lock) __spin_trylock(lock) 118 #endif 119 120 #ifdef __always_inline__read_trylock 121 #define _read_trylock(lock) __read_trylock(lock) 122 #endif 123 124 #ifdef __always_inline__write_trylock 125 #define _write_trylock(lock) __write_trylock(lock) 126 #endif 127 128 #ifdef __always_inline__spin_trylock_bh 129 #define _spin_trylock_bh(lock) __spin_trylock_bh(lock) 130 #endif 131 132 #ifdef __always_inline__spin_unlock 133 #define _spin_unlock(lock) __spin_unlock(lock) 134 #endif 135 136 #ifdef __always_inline__read_unlock 137 #define _read_unlock(lock) __read_unlock(lock) 138 #endif 139 140 #ifdef __always_inline__write_unlock 141 #define _write_unlock(lock) __write_unlock(lock) 142 #endif 143 144 #ifdef __always_inline__spin_unlock_bh 145 #define _spin_unlock_bh(lock) __spin_unlock_bh(lock) 146 #endif 147 148 #ifdef __always_inline__read_unlock_bh 149 #define _read_unlock_bh(lock) __read_unlock_bh(lock) 150 #endif 151 152 #ifdef __always_inline__write_unlock_bh 153 #define _write_unlock_bh(lock) __write_unlock_bh(lock) 154 #endif 155 156 #ifdef __always_inline__spin_unlock_irq 157 #define _spin_unlock_irq(lock) __spin_unlock_irq(lock) 158 #endif 159 160 #ifdef __always_inline__read_unlock_irq 161 #define _read_unlock_irq(lock) __read_unlock_irq(lock) 162 #endif 163 164 #ifdef __always_inline__write_unlock_irq 165 #define _write_unlock_irq(lock) __write_unlock_irq(lock) 166 #endif 167 168 #ifdef __always_inline__spin_unlock_irqrestore 169 #define _spin_unlock_irqrestore(lock, flags) __spin_unlock_irqrestore(lock, flags) 170 #endif 171 172 #ifdef __always_inline__read_unlock_irqrestore 173 #define _read_unlock_irqrestore(lock, flags) __read_unlock_irqrestore(lock, flags) 174 #endif 175 176 #ifdef __always_inline__write_unlock_irqrestore 177 #define _write_unlock_irqrestore(lock, flags) __write_unlock_irqrestore(lock, flags) 178 #endif 179 180 #endif /* CONFIG_DEBUG_SPINLOCK */ 181 182 static inline int __spin_trylock(spinlock_t *lock) 183 { 184 preempt_disable(); 185 if (_raw_spin_trylock(lock)) { 186 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); 187 return 1; 188 } 189 preempt_enable(); 190 return 0; 191 } 192 193 static inline int __read_trylock(rwlock_t *lock) 194 { 195 preempt_disable(); 196 if (_raw_read_trylock(lock)) { 197 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_); 198 return 1; 199 } 200 preempt_enable(); 201 return 0; 202 } 203 204 static inline int __write_trylock(rwlock_t *lock) 205 { 206 preempt_disable(); 207 if (_raw_write_trylock(lock)) { 208 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_); 209 return 1; 210 } 211 preempt_enable(); 212 return 0; 213 } 214 215 /* 216 * If lockdep is enabled then we use the non-preemption spin-ops 217 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are 218 * not re-enabled during lock-acquire (which the preempt-spin-ops do): 219 */ 220 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) 221 222 static inline void __read_lock(rwlock_t *lock) 223 { 224 preempt_disable(); 225 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); 226 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock); 227 } 228 229 static inline unsigned long __spin_lock_irqsave(spinlock_t *lock) 230 { 231 unsigned long flags; 232 233 local_irq_save(flags); 234 preempt_disable(); 235 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 236 /* 237 * On lockdep we dont want the hand-coded irq-enable of 238 * _raw_spin_lock_flags() code, because lockdep assumes 239 * that interrupts are not re-enabled during lock-acquire: 240 */ 241 #ifdef CONFIG_LOCKDEP 242 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); 243 #else 244 _raw_spin_lock_flags(lock, &flags); 245 #endif 246 return flags; 247 } 248 249 static inline void __spin_lock_irq(spinlock_t *lock) 250 { 251 local_irq_disable(); 252 preempt_disable(); 253 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 254 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); 255 } 256 257 static inline void __spin_lock_bh(spinlock_t *lock) 258 { 259 local_bh_disable(); 260 preempt_disable(); 261 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 262 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); 263 } 264 265 static inline unsigned long __read_lock_irqsave(rwlock_t *lock) 266 { 267 unsigned long flags; 268 269 local_irq_save(flags); 270 preempt_disable(); 271 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); 272 LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock, 273 _raw_read_lock_flags, &flags); 274 return flags; 275 } 276 277 static inline void __read_lock_irq(rwlock_t *lock) 278 { 279 local_irq_disable(); 280 preempt_disable(); 281 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); 282 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock); 283 } 284 285 static inline void __read_lock_bh(rwlock_t *lock) 286 { 287 local_bh_disable(); 288 preempt_disable(); 289 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); 290 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock); 291 } 292 293 static inline unsigned long __write_lock_irqsave(rwlock_t *lock) 294 { 295 unsigned long flags; 296 297 local_irq_save(flags); 298 preempt_disable(); 299 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); 300 LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock, 301 _raw_write_lock_flags, &flags); 302 return flags; 303 } 304 305 static inline void __write_lock_irq(rwlock_t *lock) 306 { 307 local_irq_disable(); 308 preempt_disable(); 309 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); 310 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock); 311 } 312 313 static inline void __write_lock_bh(rwlock_t *lock) 314 { 315 local_bh_disable(); 316 preempt_disable(); 317 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); 318 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock); 319 } 320 321 static inline void __spin_lock(spinlock_t *lock) 322 { 323 preempt_disable(); 324 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 325 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); 326 } 327 328 static inline void __write_lock(rwlock_t *lock) 329 { 330 preempt_disable(); 331 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); 332 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock); 333 } 334 335 #endif /* CONFIG_PREEMPT */ 336 337 static inline void __spin_unlock(spinlock_t *lock) 338 { 339 spin_release(&lock->dep_map, 1, _RET_IP_); 340 _raw_spin_unlock(lock); 341 preempt_enable(); 342 } 343 344 static inline void __write_unlock(rwlock_t *lock) 345 { 346 rwlock_release(&lock->dep_map, 1, _RET_IP_); 347 _raw_write_unlock(lock); 348 preempt_enable(); 349 } 350 351 static inline void __read_unlock(rwlock_t *lock) 352 { 353 rwlock_release(&lock->dep_map, 1, _RET_IP_); 354 _raw_read_unlock(lock); 355 preempt_enable(); 356 } 357 358 static inline void __spin_unlock_irqrestore(spinlock_t *lock, 359 unsigned long flags) 360 { 361 spin_release(&lock->dep_map, 1, _RET_IP_); 362 _raw_spin_unlock(lock); 363 local_irq_restore(flags); 364 preempt_enable(); 365 } 366 367 static inline void __spin_unlock_irq(spinlock_t *lock) 368 { 369 spin_release(&lock->dep_map, 1, _RET_IP_); 370 _raw_spin_unlock(lock); 371 local_irq_enable(); 372 preempt_enable(); 373 } 374 375 static inline void __spin_unlock_bh(spinlock_t *lock) 376 { 377 spin_release(&lock->dep_map, 1, _RET_IP_); 378 _raw_spin_unlock(lock); 379 preempt_enable_no_resched(); 380 local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 381 } 382 383 static inline void __read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 384 { 385 rwlock_release(&lock->dep_map, 1, _RET_IP_); 386 _raw_read_unlock(lock); 387 local_irq_restore(flags); 388 preempt_enable(); 389 } 390 391 static inline void __read_unlock_irq(rwlock_t *lock) 392 { 393 rwlock_release(&lock->dep_map, 1, _RET_IP_); 394 _raw_read_unlock(lock); 395 local_irq_enable(); 396 preempt_enable(); 397 } 398 399 static inline void __read_unlock_bh(rwlock_t *lock) 400 { 401 rwlock_release(&lock->dep_map, 1, _RET_IP_); 402 _raw_read_unlock(lock); 403 preempt_enable_no_resched(); 404 local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 405 } 406 407 static inline void __write_unlock_irqrestore(rwlock_t *lock, 408 unsigned long flags) 409 { 410 rwlock_release(&lock->dep_map, 1, _RET_IP_); 411 _raw_write_unlock(lock); 412 local_irq_restore(flags); 413 preempt_enable(); 414 } 415 416 static inline void __write_unlock_irq(rwlock_t *lock) 417 { 418 rwlock_release(&lock->dep_map, 1, _RET_IP_); 419 _raw_write_unlock(lock); 420 local_irq_enable(); 421 preempt_enable(); 422 } 423 424 static inline void __write_unlock_bh(rwlock_t *lock) 425 { 426 rwlock_release(&lock->dep_map, 1, _RET_IP_); 427 _raw_write_unlock(lock); 428 preempt_enable_no_resched(); 429 local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 430 } 431 432 static inline int __spin_trylock_bh(spinlock_t *lock) 433 { 434 local_bh_disable(); 435 preempt_disable(); 436 if (_raw_spin_trylock(lock)) { 437 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); 438 return 1; 439 } 440 preempt_enable_no_resched(); 441 local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 442 return 0; 443 } 444 445 #endif /* __LINUX_SPINLOCK_API_SMP_H */ 446