xref: /linux-6.15/include/linux/list.h (revision 886cf190)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_LIST_H
3 #define _LINUX_LIST_H
4 
5 #include <linux/types.h>
6 #include <linux/stddef.h>
7 #include <linux/poison.h>
8 #include <linux/const.h>
9 #include <linux/kernel.h>
10 
11 /*
12  * Simple doubly linked list implementation.
13  *
14  * Some of the internal functions ("__xxx") are useful when
15  * manipulating whole lists rather than single entries, as
16  * sometimes we already know the next/prev entries and we can
17  * generate better code by using them directly rather than
18  * using the generic single-entry routines.
19  */
20 
21 #define LIST_HEAD_INIT(name) { &(name), &(name) }
22 
23 #define LIST_HEAD(name) \
24 	struct list_head name = LIST_HEAD_INIT(name)
25 
26 static inline void INIT_LIST_HEAD(struct list_head *list)
27 {
28 	WRITE_ONCE(list->next, list);
29 	list->prev = list;
30 }
31 
32 #ifdef CONFIG_DEBUG_LIST
33 extern bool __list_add_valid(struct list_head *new,
34 			      struct list_head *prev,
35 			      struct list_head *next);
36 extern bool __list_del_entry_valid(struct list_head *entry);
37 #else
38 static inline bool __list_add_valid(struct list_head *new,
39 				struct list_head *prev,
40 				struct list_head *next)
41 {
42 	return true;
43 }
44 static inline bool __list_del_entry_valid(struct list_head *entry)
45 {
46 	return true;
47 }
48 #endif
49 
50 /*
51  * Insert a new entry between two known consecutive entries.
52  *
53  * This is only for internal list manipulation where we know
54  * the prev/next entries already!
55  */
56 static inline void __list_add(struct list_head *new,
57 			      struct list_head *prev,
58 			      struct list_head *next)
59 {
60 	if (!__list_add_valid(new, prev, next))
61 		return;
62 
63 	next->prev = new;
64 	new->next = next;
65 	new->prev = prev;
66 	WRITE_ONCE(prev->next, new);
67 }
68 
69 /**
70  * list_add - add a new entry
71  * @new: new entry to be added
72  * @head: list head to add it after
73  *
74  * Insert a new entry after the specified head.
75  * This is good for implementing stacks.
76  */
77 static inline void list_add(struct list_head *new, struct list_head *head)
78 {
79 	__list_add(new, head, head->next);
80 }
81 
82 
83 /**
84  * list_add_tail - add a new entry
85  * @new: new entry to be added
86  * @head: list head to add it before
87  *
88  * Insert a new entry before the specified head.
89  * This is useful for implementing queues.
90  */
91 static inline void list_add_tail(struct list_head *new, struct list_head *head)
92 {
93 	__list_add(new, head->prev, head);
94 }
95 
96 /*
97  * Delete a list entry by making the prev/next entries
98  * point to each other.
99  *
100  * This is only for internal list manipulation where we know
101  * the prev/next entries already!
102  */
103 static inline void __list_del(struct list_head * prev, struct list_head * next)
104 {
105 	next->prev = prev;
106 	WRITE_ONCE(prev->next, next);
107 }
108 
109 /**
110  * list_del - deletes entry from list.
111  * @entry: the element to delete from the list.
112  * Note: list_empty() on entry does not return true after this, the entry is
113  * in an undefined state.
114  */
115 static inline void __list_del_entry(struct list_head *entry)
116 {
117 	if (!__list_del_entry_valid(entry))
118 		return;
119 
120 	__list_del(entry->prev, entry->next);
121 }
122 
123 static inline void list_del(struct list_head *entry)
124 {
125 	__list_del_entry(entry);
126 	entry->next = LIST_POISON1;
127 	entry->prev = LIST_POISON2;
128 }
129 
130 /**
131  * list_replace - replace old entry by new one
132  * @old : the element to be replaced
133  * @new : the new element to insert
134  *
135  * If @old was empty, it will be overwritten.
136  */
137 static inline void list_replace(struct list_head *old,
138 				struct list_head *new)
139 {
140 	new->next = old->next;
141 	new->next->prev = new;
142 	new->prev = old->prev;
143 	new->prev->next = new;
144 }
145 
146 static inline void list_replace_init(struct list_head *old,
147 					struct list_head *new)
148 {
149 	list_replace(old, new);
150 	INIT_LIST_HEAD(old);
151 }
152 
153 /**
154  * list_del_init - deletes entry from list and reinitialize it.
155  * @entry: the element to delete from the list.
156  */
157 static inline void list_del_init(struct list_head *entry)
158 {
159 	__list_del_entry(entry);
160 	INIT_LIST_HEAD(entry);
161 }
162 
163 /**
164  * list_move - delete from one list and add as another's head
165  * @list: the entry to move
166  * @head: the head that will precede our entry
167  */
168 static inline void list_move(struct list_head *list, struct list_head *head)
169 {
170 	__list_del_entry(list);
171 	list_add(list, head);
172 }
173 
174 /**
175  * list_move_tail - delete from one list and add as another's tail
176  * @list: the entry to move
177  * @head: the head that will follow our entry
178  */
179 static inline void list_move_tail(struct list_head *list,
180 				  struct list_head *head)
181 {
182 	__list_del_entry(list);
183 	list_add_tail(list, head);
184 }
185 
186 /**
187  * list_bulk_move_tail - move a subsection of a list to its tail
188  * @head: the head that will follow our entry
189  * @first: first entry to move
190  * @last: last entry to move, can be the same as first
191  *
192  * Move all entries between @first and including @last before @head.
193  * All three entries must belong to the same linked list.
194  */
195 static inline void list_bulk_move_tail(struct list_head *head,
196 				       struct list_head *first,
197 				       struct list_head *last)
198 {
199 	first->prev->next = last->next;
200 	last->next->prev = first->prev;
201 
202 	head->prev->next = first;
203 	first->prev = head->prev;
204 
205 	last->next = head;
206 	head->prev = last;
207 }
208 
209 /**
210  * list_is_first -- tests whether @list is the first entry in list @head
211  * @list: the entry to test
212  * @head: the head of the list
213  */
214 static inline int list_is_first(const struct list_head *list,
215 					const struct list_head *head)
216 {
217 	return list->prev == head;
218 }
219 
220 /**
221  * list_is_last - tests whether @list is the last entry in list @head
222  * @list: the entry to test
223  * @head: the head of the list
224  */
225 static inline int list_is_last(const struct list_head *list,
226 				const struct list_head *head)
227 {
228 	return list->next == head;
229 }
230 
231 /**
232  * list_empty - tests whether a list is empty
233  * @head: the list to test.
234  */
235 static inline int list_empty(const struct list_head *head)
236 {
237 	return READ_ONCE(head->next) == head;
238 }
239 
240 /**
241  * list_empty_careful - tests whether a list is empty and not being modified
242  * @head: the list to test
243  *
244  * Description:
245  * tests whether a list is empty _and_ checks that no other CPU might be
246  * in the process of modifying either member (next or prev)
247  *
248  * NOTE: using list_empty_careful() without synchronization
249  * can only be safe if the only activity that can happen
250  * to the list entry is list_del_init(). Eg. it cannot be used
251  * if another CPU could re-list_add() it.
252  */
253 static inline int list_empty_careful(const struct list_head *head)
254 {
255 	struct list_head *next = head->next;
256 	return (next == head) && (next == head->prev);
257 }
258 
259 /**
260  * list_rotate_left - rotate the list to the left
261  * @head: the head of the list
262  */
263 static inline void list_rotate_left(struct list_head *head)
264 {
265 	struct list_head *first;
266 
267 	if (!list_empty(head)) {
268 		first = head->next;
269 		list_move_tail(first, head);
270 	}
271 }
272 
273 /**
274  * list_rotate_to_front() - Rotate list to specific item.
275  * @list: The desired new front of the list.
276  * @head: The head of the list.
277  *
278  * Rotates list so that @list becomes the new front of the list.
279  */
280 static inline void list_rotate_to_front(struct list_head *list,
281 					struct list_head *head)
282 {
283 	/*
284 	 * Deletes the list head from the list denoted by @head and
285 	 * places it as the tail of @list, this effectively rotates the
286 	 * list so that @list is at the front.
287 	 */
288 	list_move_tail(head, list);
289 }
290 
291 /**
292  * list_is_singular - tests whether a list has just one entry.
293  * @head: the list to test.
294  */
295 static inline int list_is_singular(const struct list_head *head)
296 {
297 	return !list_empty(head) && (head->next == head->prev);
298 }
299 
300 static inline void __list_cut_position(struct list_head *list,
301 		struct list_head *head, struct list_head *entry)
302 {
303 	struct list_head *new_first = entry->next;
304 	list->next = head->next;
305 	list->next->prev = list;
306 	list->prev = entry;
307 	entry->next = list;
308 	head->next = new_first;
309 	new_first->prev = head;
310 }
311 
312 /**
313  * list_cut_position - cut a list into two
314  * @list: a new list to add all removed entries
315  * @head: a list with entries
316  * @entry: an entry within head, could be the head itself
317  *	and if so we won't cut the list
318  *
319  * This helper moves the initial part of @head, up to and
320  * including @entry, from @head to @list. You should
321  * pass on @entry an element you know is on @head. @list
322  * should be an empty list or a list you do not care about
323  * losing its data.
324  *
325  */
326 static inline void list_cut_position(struct list_head *list,
327 		struct list_head *head, struct list_head *entry)
328 {
329 	if (list_empty(head))
330 		return;
331 	if (list_is_singular(head) &&
332 		(head->next != entry && head != entry))
333 		return;
334 	if (entry == head)
335 		INIT_LIST_HEAD(list);
336 	else
337 		__list_cut_position(list, head, entry);
338 }
339 
340 /**
341  * list_cut_before - cut a list into two, before given entry
342  * @list: a new list to add all removed entries
343  * @head: a list with entries
344  * @entry: an entry within head, could be the head itself
345  *
346  * This helper moves the initial part of @head, up to but
347  * excluding @entry, from @head to @list.  You should pass
348  * in @entry an element you know is on @head.  @list should
349  * be an empty list or a list you do not care about losing
350  * its data.
351  * If @entry == @head, all entries on @head are moved to
352  * @list.
353  */
354 static inline void list_cut_before(struct list_head *list,
355 				   struct list_head *head,
356 				   struct list_head *entry)
357 {
358 	if (head->next == entry) {
359 		INIT_LIST_HEAD(list);
360 		return;
361 	}
362 	list->next = head->next;
363 	list->next->prev = list;
364 	list->prev = entry->prev;
365 	list->prev->next = list;
366 	head->next = entry;
367 	entry->prev = head;
368 }
369 
370 static inline void __list_splice(const struct list_head *list,
371 				 struct list_head *prev,
372 				 struct list_head *next)
373 {
374 	struct list_head *first = list->next;
375 	struct list_head *last = list->prev;
376 
377 	first->prev = prev;
378 	prev->next = first;
379 
380 	last->next = next;
381 	next->prev = last;
382 }
383 
384 /**
385  * list_splice - join two lists, this is designed for stacks
386  * @list: the new list to add.
387  * @head: the place to add it in the first list.
388  */
389 static inline void list_splice(const struct list_head *list,
390 				struct list_head *head)
391 {
392 	if (!list_empty(list))
393 		__list_splice(list, head, head->next);
394 }
395 
396 /**
397  * list_splice_tail - join two lists, each list being a queue
398  * @list: the new list to add.
399  * @head: the place to add it in the first list.
400  */
401 static inline void list_splice_tail(struct list_head *list,
402 				struct list_head *head)
403 {
404 	if (!list_empty(list))
405 		__list_splice(list, head->prev, head);
406 }
407 
408 /**
409  * list_splice_init - join two lists and reinitialise the emptied list.
410  * @list: the new list to add.
411  * @head: the place to add it in the first list.
412  *
413  * The list at @list is reinitialised
414  */
415 static inline void list_splice_init(struct list_head *list,
416 				    struct list_head *head)
417 {
418 	if (!list_empty(list)) {
419 		__list_splice(list, head, head->next);
420 		INIT_LIST_HEAD(list);
421 	}
422 }
423 
424 /**
425  * list_splice_tail_init - join two lists and reinitialise the emptied list
426  * @list: the new list to add.
427  * @head: the place to add it in the first list.
428  *
429  * Each of the lists is a queue.
430  * The list at @list is reinitialised
431  */
432 static inline void list_splice_tail_init(struct list_head *list,
433 					 struct list_head *head)
434 {
435 	if (!list_empty(list)) {
436 		__list_splice(list, head->prev, head);
437 		INIT_LIST_HEAD(list);
438 	}
439 }
440 
441 /**
442  * list_entry - get the struct for this entry
443  * @ptr:	the &struct list_head pointer.
444  * @type:	the type of the struct this is embedded in.
445  * @member:	the name of the list_head within the struct.
446  */
447 #define list_entry(ptr, type, member) \
448 	container_of(ptr, type, member)
449 
450 /**
451  * list_first_entry - get the first element from a list
452  * @ptr:	the list head to take the element from.
453  * @type:	the type of the struct this is embedded in.
454  * @member:	the name of the list_head within the struct.
455  *
456  * Note, that list is expected to be not empty.
457  */
458 #define list_first_entry(ptr, type, member) \
459 	list_entry((ptr)->next, type, member)
460 
461 /**
462  * list_last_entry - get the last element from a list
463  * @ptr:	the list head to take the element from.
464  * @type:	the type of the struct this is embedded in.
465  * @member:	the name of the list_head within the struct.
466  *
467  * Note, that list is expected to be not empty.
468  */
469 #define list_last_entry(ptr, type, member) \
470 	list_entry((ptr)->prev, type, member)
471 
472 /**
473  * list_first_entry_or_null - get the first element from a list
474  * @ptr:	the list head to take the element from.
475  * @type:	the type of the struct this is embedded in.
476  * @member:	the name of the list_head within the struct.
477  *
478  * Note that if the list is empty, it returns NULL.
479  */
480 #define list_first_entry_or_null(ptr, type, member) ({ \
481 	struct list_head *head__ = (ptr); \
482 	struct list_head *pos__ = READ_ONCE(head__->next); \
483 	pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
484 })
485 
486 /**
487  * list_next_entry - get the next element in list
488  * @pos:	the type * to cursor
489  * @member:	the name of the list_head within the struct.
490  */
491 #define list_next_entry(pos, member) \
492 	list_entry((pos)->member.next, typeof(*(pos)), member)
493 
494 /**
495  * list_prev_entry - get the prev element in list
496  * @pos:	the type * to cursor
497  * @member:	the name of the list_head within the struct.
498  */
499 #define list_prev_entry(pos, member) \
500 	list_entry((pos)->member.prev, typeof(*(pos)), member)
501 
502 /**
503  * list_for_each	-	iterate over a list
504  * @pos:	the &struct list_head to use as a loop cursor.
505  * @head:	the head for your list.
506  */
507 #define list_for_each(pos, head) \
508 	for (pos = (head)->next; pos != (head); pos = pos->next)
509 
510 /**
511  * list_for_each_prev	-	iterate over a list backwards
512  * @pos:	the &struct list_head to use as a loop cursor.
513  * @head:	the head for your list.
514  */
515 #define list_for_each_prev(pos, head) \
516 	for (pos = (head)->prev; pos != (head); pos = pos->prev)
517 
518 /**
519  * list_for_each_safe - iterate over a list safe against removal of list entry
520  * @pos:	the &struct list_head to use as a loop cursor.
521  * @n:		another &struct list_head to use as temporary storage
522  * @head:	the head for your list.
523  */
524 #define list_for_each_safe(pos, n, head) \
525 	for (pos = (head)->next, n = pos->next; pos != (head); \
526 		pos = n, n = pos->next)
527 
528 /**
529  * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry
530  * @pos:	the &struct list_head to use as a loop cursor.
531  * @n:		another &struct list_head to use as temporary storage
532  * @head:	the head for your list.
533  */
534 #define list_for_each_prev_safe(pos, n, head) \
535 	for (pos = (head)->prev, n = pos->prev; \
536 	     pos != (head); \
537 	     pos = n, n = pos->prev)
538 
539 /**
540  * list_for_each_entry	-	iterate over list of given type
541  * @pos:	the type * to use as a loop cursor.
542  * @head:	the head for your list.
543  * @member:	the name of the list_head within the struct.
544  */
545 #define list_for_each_entry(pos, head, member)				\
546 	for (pos = list_first_entry(head, typeof(*pos), member);	\
547 	     &pos->member != (head);					\
548 	     pos = list_next_entry(pos, member))
549 
550 /**
551  * list_for_each_entry_reverse - iterate backwards over list of given type.
552  * @pos:	the type * to use as a loop cursor.
553  * @head:	the head for your list.
554  * @member:	the name of the list_head within the struct.
555  */
556 #define list_for_each_entry_reverse(pos, head, member)			\
557 	for (pos = list_last_entry(head, typeof(*pos), member);		\
558 	     &pos->member != (head); 					\
559 	     pos = list_prev_entry(pos, member))
560 
561 /**
562  * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue()
563  * @pos:	the type * to use as a start point
564  * @head:	the head of the list
565  * @member:	the name of the list_head within the struct.
566  *
567  * Prepares a pos entry for use as a start point in list_for_each_entry_continue().
568  */
569 #define list_prepare_entry(pos, head, member) \
570 	((pos) ? : list_entry(head, typeof(*pos), member))
571 
572 /**
573  * list_for_each_entry_continue - continue iteration over list of given type
574  * @pos:	the type * to use as a loop cursor.
575  * @head:	the head for your list.
576  * @member:	the name of the list_head within the struct.
577  *
578  * Continue to iterate over list of given type, continuing after
579  * the current position.
580  */
581 #define list_for_each_entry_continue(pos, head, member) 		\
582 	for (pos = list_next_entry(pos, member);			\
583 	     &pos->member != (head);					\
584 	     pos = list_next_entry(pos, member))
585 
586 /**
587  * list_for_each_entry_continue_reverse - iterate backwards from the given point
588  * @pos:	the type * to use as a loop cursor.
589  * @head:	the head for your list.
590  * @member:	the name of the list_head within the struct.
591  *
592  * Start to iterate over list of given type backwards, continuing after
593  * the current position.
594  */
595 #define list_for_each_entry_continue_reverse(pos, head, member)		\
596 	for (pos = list_prev_entry(pos, member);			\
597 	     &pos->member != (head);					\
598 	     pos = list_prev_entry(pos, member))
599 
600 /**
601  * list_for_each_entry_from - iterate over list of given type from the current point
602  * @pos:	the type * to use as a loop cursor.
603  * @head:	the head for your list.
604  * @member:	the name of the list_head within the struct.
605  *
606  * Iterate over list of given type, continuing from current position.
607  */
608 #define list_for_each_entry_from(pos, head, member) 			\
609 	for (; &pos->member != (head);					\
610 	     pos = list_next_entry(pos, member))
611 
612 /**
613  * list_for_each_entry_from_reverse - iterate backwards over list of given type
614  *                                    from the current point
615  * @pos:	the type * to use as a loop cursor.
616  * @head:	the head for your list.
617  * @member:	the name of the list_head within the struct.
618  *
619  * Iterate backwards over list of given type, continuing from current position.
620  */
621 #define list_for_each_entry_from_reverse(pos, head, member)		\
622 	for (; &pos->member != (head);					\
623 	     pos = list_prev_entry(pos, member))
624 
625 /**
626  * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
627  * @pos:	the type * to use as a loop cursor.
628  * @n:		another type * to use as temporary storage
629  * @head:	the head for your list.
630  * @member:	the name of the list_head within the struct.
631  */
632 #define list_for_each_entry_safe(pos, n, head, member)			\
633 	for (pos = list_first_entry(head, typeof(*pos), member),	\
634 		n = list_next_entry(pos, member);			\
635 	     &pos->member != (head); 					\
636 	     pos = n, n = list_next_entry(n, member))
637 
638 /**
639  * list_for_each_entry_safe_continue - continue list iteration safe against removal
640  * @pos:	the type * to use as a loop cursor.
641  * @n:		another type * to use as temporary storage
642  * @head:	the head for your list.
643  * @member:	the name of the list_head within the struct.
644  *
645  * Iterate over list of given type, continuing after current point,
646  * safe against removal of list entry.
647  */
648 #define list_for_each_entry_safe_continue(pos, n, head, member) 		\
649 	for (pos = list_next_entry(pos, member), 				\
650 		n = list_next_entry(pos, member);				\
651 	     &pos->member != (head);						\
652 	     pos = n, n = list_next_entry(n, member))
653 
654 /**
655  * list_for_each_entry_safe_from - iterate over list from current point safe against removal
656  * @pos:	the type * to use as a loop cursor.
657  * @n:		another type * to use as temporary storage
658  * @head:	the head for your list.
659  * @member:	the name of the list_head within the struct.
660  *
661  * Iterate over list of given type from current point, safe against
662  * removal of list entry.
663  */
664 #define list_for_each_entry_safe_from(pos, n, head, member) 			\
665 	for (n = list_next_entry(pos, member);					\
666 	     &pos->member != (head);						\
667 	     pos = n, n = list_next_entry(n, member))
668 
669 /**
670  * list_for_each_entry_safe_reverse - iterate backwards over list safe against removal
671  * @pos:	the type * to use as a loop cursor.
672  * @n:		another type * to use as temporary storage
673  * @head:	the head for your list.
674  * @member:	the name of the list_head within the struct.
675  *
676  * Iterate backwards over list of given type, safe against removal
677  * of list entry.
678  */
679 #define list_for_each_entry_safe_reverse(pos, n, head, member)		\
680 	for (pos = list_last_entry(head, typeof(*pos), member),		\
681 		n = list_prev_entry(pos, member);			\
682 	     &pos->member != (head); 					\
683 	     pos = n, n = list_prev_entry(n, member))
684 
685 /**
686  * list_safe_reset_next - reset a stale list_for_each_entry_safe loop
687  * @pos:	the loop cursor used in the list_for_each_entry_safe loop
688  * @n:		temporary storage used in list_for_each_entry_safe
689  * @member:	the name of the list_head within the struct.
690  *
691  * list_safe_reset_next is not safe to use in general if the list may be
692  * modified concurrently (eg. the lock is dropped in the loop body). An
693  * exception to this is if the cursor element (pos) is pinned in the list,
694  * and list_safe_reset_next is called after re-taking the lock and before
695  * completing the current iteration of the loop body.
696  */
697 #define list_safe_reset_next(pos, n, member)				\
698 	n = list_next_entry(pos, member)
699 
700 /*
701  * Double linked lists with a single pointer list head.
702  * Mostly useful for hash tables where the two pointer list head is
703  * too wasteful.
704  * You lose the ability to access the tail in O(1).
705  */
706 
707 #define HLIST_HEAD_INIT { .first = NULL }
708 #define HLIST_HEAD(name) struct hlist_head name = {  .first = NULL }
709 #define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
710 static inline void INIT_HLIST_NODE(struct hlist_node *h)
711 {
712 	h->next = NULL;
713 	h->pprev = NULL;
714 }
715 
716 static inline int hlist_unhashed(const struct hlist_node *h)
717 {
718 	return !h->pprev;
719 }
720 
721 static inline int hlist_empty(const struct hlist_head *h)
722 {
723 	return !READ_ONCE(h->first);
724 }
725 
726 static inline void __hlist_del(struct hlist_node *n)
727 {
728 	struct hlist_node *next = n->next;
729 	struct hlist_node **pprev = n->pprev;
730 
731 	WRITE_ONCE(*pprev, next);
732 	if (next)
733 		next->pprev = pprev;
734 }
735 
736 static inline void hlist_del(struct hlist_node *n)
737 {
738 	__hlist_del(n);
739 	n->next = LIST_POISON1;
740 	n->pprev = LIST_POISON2;
741 }
742 
743 static inline void hlist_del_init(struct hlist_node *n)
744 {
745 	if (!hlist_unhashed(n)) {
746 		__hlist_del(n);
747 		INIT_HLIST_NODE(n);
748 	}
749 }
750 
751 static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
752 {
753 	struct hlist_node *first = h->first;
754 	n->next = first;
755 	if (first)
756 		first->pprev = &n->next;
757 	WRITE_ONCE(h->first, n);
758 	n->pprev = &h->first;
759 }
760 
761 /* next must be != NULL */
762 static inline void hlist_add_before(struct hlist_node *n,
763 					struct hlist_node *next)
764 {
765 	n->pprev = next->pprev;
766 	n->next = next;
767 	next->pprev = &n->next;
768 	WRITE_ONCE(*(n->pprev), n);
769 }
770 
771 static inline void hlist_add_behind(struct hlist_node *n,
772 				    struct hlist_node *prev)
773 {
774 	n->next = prev->next;
775 	WRITE_ONCE(prev->next, n);
776 	n->pprev = &prev->next;
777 
778 	if (n->next)
779 		n->next->pprev  = &n->next;
780 }
781 
782 /* after that we'll appear to be on some hlist and hlist_del will work */
783 static inline void hlist_add_fake(struct hlist_node *n)
784 {
785 	n->pprev = &n->next;
786 }
787 
788 static inline bool hlist_fake(struct hlist_node *h)
789 {
790 	return h->pprev == &h->next;
791 }
792 
793 /*
794  * Check whether the node is the only node of the head without
795  * accessing head:
796  */
797 static inline bool
798 hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h)
799 {
800 	return !n->next && n->pprev == &h->first;
801 }
802 
803 /*
804  * Move a list from one list head to another. Fixup the pprev
805  * reference of the first entry if it exists.
806  */
807 static inline void hlist_move_list(struct hlist_head *old,
808 				   struct hlist_head *new)
809 {
810 	new->first = old->first;
811 	if (new->first)
812 		new->first->pprev = &new->first;
813 	old->first = NULL;
814 }
815 
816 #define hlist_entry(ptr, type, member) container_of(ptr,type,member)
817 
818 #define hlist_for_each(pos, head) \
819 	for (pos = (head)->first; pos ; pos = pos->next)
820 
821 #define hlist_for_each_safe(pos, n, head) \
822 	for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
823 	     pos = n)
824 
825 #define hlist_entry_safe(ptr, type, member) \
826 	({ typeof(ptr) ____ptr = (ptr); \
827 	   ____ptr ? hlist_entry(____ptr, type, member) : NULL; \
828 	})
829 
830 /**
831  * hlist_for_each_entry	- iterate over list of given type
832  * @pos:	the type * to use as a loop cursor.
833  * @head:	the head for your list.
834  * @member:	the name of the hlist_node within the struct.
835  */
836 #define hlist_for_each_entry(pos, head, member)				\
837 	for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\
838 	     pos;							\
839 	     pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
840 
841 /**
842  * hlist_for_each_entry_continue - iterate over a hlist continuing after current point
843  * @pos:	the type * to use as a loop cursor.
844  * @member:	the name of the hlist_node within the struct.
845  */
846 #define hlist_for_each_entry_continue(pos, member)			\
847 	for (pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member);\
848 	     pos;							\
849 	     pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
850 
851 /**
852  * hlist_for_each_entry_from - iterate over a hlist continuing from current point
853  * @pos:	the type * to use as a loop cursor.
854  * @member:	the name of the hlist_node within the struct.
855  */
856 #define hlist_for_each_entry_from(pos, member)				\
857 	for (; pos;							\
858 	     pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
859 
860 /**
861  * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
862  * @pos:	the type * to use as a loop cursor.
863  * @n:		another &struct hlist_node to use as temporary storage
864  * @head:	the head for your list.
865  * @member:	the name of the hlist_node within the struct.
866  */
867 #define hlist_for_each_entry_safe(pos, n, head, member) 		\
868 	for (pos = hlist_entry_safe((head)->first, typeof(*pos), member);\
869 	     pos && ({ n = pos->member.next; 1; });			\
870 	     pos = hlist_entry_safe(n, typeof(*pos), member))
871 
872 #endif
873