42 static inline bool lock_wqueue(struct watch_queue *wqueue)  in lock_wqueue()  argument
44 	spin_lock_bh(&wqueue->lock);  in lock_wqueue()
45 	if (unlikely(!wqueue->pipe)) {  in lock_wqueue()
46 		spin_unlock_bh(&wqueue->lock);  in lock_wqueue()
52 static inline void unlock_wqueue(struct watch_queue *wqueue)  in unlock_wqueue()  argument
54 	spin_unlock_bh(&wqueue->lock);  in unlock_wqueue()
60 	struct watch_queue *wqueue = (struct watch_queue *)buf->private;  in watch_queue_pipe_buf_release()  local
76 	set_bit(bit, wqueue->notes_bitmap);  in watch_queue_pipe_buf_release()
97 static bool post_one_notification(struct watch_queue *wqueue,  in post_one_notification()  argument
101 	struct pipe_inode_info *pipe = wqueue->pipe;  in post_one_notification()
114 	note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);  in post_one_notification()
115 	if (note >= wqueue->nr_notes)  in post_one_notification()
118 	page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];  in post_one_notification()
128 	buf->private = (unsigned long)wqueue;  in post_one_notification()
135 	if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {  in post_one_notification()
199 	struct watch_queue *wqueue;  in __post_watch_notification()  local
215 		wqueue = rcu_dereference(watch->queue);  in __post_watch_notification()
216 		wf = rcu_dereference(wqueue->filter);  in __post_watch_notification()
223 		if (lock_wqueue(wqueue)) {  in __post_watch_notification()
224 			post_one_notification(wqueue, n);  in __post_watch_notification()
225 			unlock_wqueue(wqueue);  in __post_watch_notification()
239 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_size()  local
245 	if (!wqueue)  in watch_queue_set_size()
247 	if (wqueue->notes)  in watch_queue_set_size()
297 	wqueue->notes = pages;  in watch_queue_set_size()
298 	wqueue->notes_bitmap = bitmap;  in watch_queue_set_size()
299 	wqueue->nr_pages = nr_pages;  in watch_queue_set_size()
300 	wqueue->nr_notes = nr_notes;  in watch_queue_set_size()
322 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_filter()  local
325 	if (!wqueue)  in watch_queue_set_filter()
382 	wfilter = rcu_replace_pointer(wqueue->filter, wfilter,  in watch_queue_set_filter()
396 	struct watch_queue *wqueue =  in __put_watch_queue()  local
401 	for (i = 0; i < wqueue->nr_pages; i++)  in __put_watch_queue()
402 		__free_page(wqueue->notes[i]);  in __put_watch_queue()
403 	kfree(wqueue->notes);  in __put_watch_queue()
404 	bitmap_free(wqueue->notes_bitmap);  in __put_watch_queue()
406 	wfilter = rcu_access_pointer(wqueue->filter);  in __put_watch_queue()
409 	kfree_rcu(wqueue, rcu);  in __put_watch_queue()
416 void put_watch_queue(struct watch_queue *wqueue)  in put_watch_queue()  argument
418 	kref_put(&wqueue->usage, __put_watch_queue);  in put_watch_queue()
454 void init_watch(struct watch *watch, struct watch_queue *wqueue)  in init_watch()  argument
459 	rcu_assign_pointer(watch->queue, wqueue);  in init_watch()
462 static int add_one_watch(struct watch *watch, struct watch_list *wlist, struct watch_queue *wqueue)  in add_one_watch()  argument
469 		if (wqueue == wq && watch->id == w->id)  in add_one_watch()
482 	kref_get(&wqueue->usage);  in add_one_watch()
484 	hlist_add_head(&watch->queue_node, &wqueue->watches);  in add_one_watch()
503 	struct watch_queue *wqueue;  in add_watch_to_object()  local
508 	wqueue = rcu_access_pointer(watch->queue);  in add_watch_to_object()
509 	if (lock_wqueue(wqueue)) {  in add_watch_to_object()
511 		ret = add_one_watch(watch, wlist, wqueue);  in add_watch_to_object()
513 		unlock_wqueue(wqueue);  in add_watch_to_object()
535 	struct watch_queue *wqueue;  in remove_watch_from_object()  local
566 	wqueue = rcu_dereference(watch->queue);  in remove_watch_from_object()
568 	if (lock_wqueue(wqueue)) {  in remove_watch_from_object()
569 		post_one_notification(wqueue, &n.watch);  in remove_watch_from_object()
576 		unlock_wqueue(wqueue);  in remove_watch_from_object()
602 void watch_queue_clear(struct watch_queue *wqueue)  in watch_queue_clear()  argument
609 	spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
615 	wqueue->pipe = NULL;  in watch_queue_clear()
617 	while (!hlist_empty(&wqueue->watches)) {  in watch_queue_clear()
618 		watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);  in watch_queue_clear()
621 		spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
659 		spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
662 	spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
673 	struct watch_queue *wqueue = ERR_PTR(-EINVAL);  in get_watch_queue()  local
679 			wqueue = pipe->watch_queue;  in get_watch_queue()
680 			kref_get(&wqueue->usage);  in get_watch_queue()
684 	return wqueue;  in get_watch_queue()
693 	struct watch_queue *wqueue;  in watch_queue_init()  local
695 	wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);  in watch_queue_init()
696 	if (!wqueue)  in watch_queue_init()
699 	wqueue->pipe = pipe;  in watch_queue_init()
700 	kref_init(&wqueue->usage);  in watch_queue_init()
701 	spin_lock_init(&wqueue->lock);  in watch_queue_init()
702 	INIT_HLIST_HEAD(&wqueue->watches);  in watch_queue_init()
704 	pipe->watch_queue = wqueue;  in watch_queue_init()