48 static inline void percpu_down_read(struct percpu_rw_semaphore *sem)  in percpu_down_read()  argument
52 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);  in percpu_down_read()
63 	if (likely(rcu_sync_is_idle(&sem->rss)))  in percpu_down_read()
64 		this_cpu_inc(*sem->read_count);  in percpu_down_read()
66 		__percpu_down_read(sem, false); /* Unconditional memory barrier */  in percpu_down_read()
74 static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem)  in percpu_down_read_trylock()  argument
82 	if (likely(rcu_sync_is_idle(&sem->rss)))  in percpu_down_read_trylock()
83 		this_cpu_inc(*sem->read_count);  in percpu_down_read_trylock()
85 		ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */  in percpu_down_read_trylock()
93 		rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);  in percpu_down_read_trylock()
98 static inline void percpu_up_read(struct percpu_rw_semaphore *sem)  in percpu_up_read()  argument
100 	rwsem_release(&sem->dep_map, _RET_IP_);  in percpu_up_read()
106 	if (likely(rcu_sync_is_idle(&sem->rss))) {  in percpu_up_read()
107 		this_cpu_dec(*sem->read_count);  in percpu_up_read()
119 		this_cpu_dec(*sem->read_count);  in percpu_up_read()
120 		rcuwait_wake_up(&sem->writer);  in percpu_up_read()
136 static inline bool percpu_is_write_locked(struct percpu_rw_semaphore *sem)  in DEFINE_GUARD()
138 	return atomic_read(&sem->block);  in DEFINE_GUARD()
146 #define percpu_init_rwsem(sem)					\  argument
149 	__percpu_init_rwsem(sem, #sem, &rwsem_key);		\
152 #define percpu_rwsem_is_held(sem)	lockdep_is_held(sem)  argument
153 #define percpu_rwsem_assert_held(sem)	lockdep_assert_held(sem)  argument
155 static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,  in percpu_rwsem_release()  argument
158 	lock_release(&sem->dep_map, ip);  in percpu_rwsem_release()
161 static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,  in percpu_rwsem_acquire()  argument
164 	lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip);  in percpu_rwsem_acquire()