1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Queue of folios definitions 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells ([email protected]) 6 */ 7 8 #ifndef _LINUX_FOLIO_QUEUE_H 9 #define _LINUX_FOLIO_QUEUE_H 10 11 #include <linux/pagevec.h> 12 13 /* 14 * Segment in a queue of running buffers. Each segment can hold a number of 15 * folios and a portion of the queue can be referenced with the ITER_FOLIOQ 16 * iterator. The possibility exists of inserting non-folio elements into the 17 * queue (such as gaps). 18 * 19 * Explicit prev and next pointers are used instead of a list_head to make it 20 * easier to add segments to tail and remove them from the head without the 21 * need for a lock. 22 */ 23 struct folio_queue { 24 struct folio_batch vec; /* Folios in the queue segment */ 25 u8 orders[PAGEVEC_SIZE]; /* Order of each folio */ 26 struct folio_queue *next; /* Next queue segment or NULL */ 27 struct folio_queue *prev; /* Previous queue segment of NULL */ 28 unsigned long marks; /* 1-bit mark per folio */ 29 unsigned long marks2; /* Second 1-bit mark per folio */ 30 #if PAGEVEC_SIZE > BITS_PER_LONG 31 #error marks is not big enough 32 #endif 33 }; 34 35 static inline void folioq_init(struct folio_queue *folioq) 36 { 37 folio_batch_init(&folioq->vec); 38 folioq->next = NULL; 39 folioq->prev = NULL; 40 folioq->marks = 0; 41 folioq->marks2 = 0; 42 } 43 44 static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq) 45 { 46 return PAGEVEC_SIZE; 47 } 48 49 static inline unsigned int folioq_count(struct folio_queue *folioq) 50 { 51 return folio_batch_count(&folioq->vec); 52 } 53 54 static inline bool folioq_full(struct folio_queue *folioq) 55 { 56 //return !folio_batch_space(&folioq->vec); 57 return folioq_count(folioq) >= folioq_nr_slots(folioq); 58 } 59 60 static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot) 61 { 62 return test_bit(slot, &folioq->marks); 63 } 64 65 static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot) 66 { 67 set_bit(slot, &folioq->marks); 68 } 69 70 static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot) 71 { 72 clear_bit(slot, &folioq->marks); 73 } 74 75 static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot) 76 { 77 return test_bit(slot, &folioq->marks2); 78 } 79 80 static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot) 81 { 82 set_bit(slot, &folioq->marks2); 83 } 84 85 static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot) 86 { 87 clear_bit(slot, &folioq->marks2); 88 } 89 90 static inline unsigned int __folio_order(struct folio *folio) 91 { 92 if (!folio_test_large(folio)) 93 return 0; 94 return folio->_flags_1 & 0xff; 95 } 96 97 static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio) 98 { 99 unsigned int slot = folioq->vec.nr++; 100 101 folioq->vec.folios[slot] = folio; 102 folioq->orders[slot] = __folio_order(folio); 103 return slot; 104 } 105 106 static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio) 107 { 108 unsigned int slot = folioq->vec.nr++; 109 110 folioq->vec.folios[slot] = folio; 111 folioq->orders[slot] = __folio_order(folio); 112 folioq_mark(folioq, slot); 113 return slot; 114 } 115 116 static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot) 117 { 118 return folioq->vec.folios[slot]; 119 } 120 121 static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot) 122 { 123 return folioq->orders[slot]; 124 } 125 126 static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot) 127 { 128 return PAGE_SIZE << folioq_folio_order(folioq, slot); 129 } 130 131 static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot) 132 { 133 folioq->vec.folios[slot] = NULL; 134 folioq_unmark(folioq, slot); 135 folioq_unmark2(folioq, slot); 136 } 137 138 #endif /* _LINUX_FOLIO_QUEUE_H */ 139