1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Queue of folios definitions 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells ([email protected]) 6 */ 7 8 #ifndef _LINUX_FOLIO_QUEUE_H 9 #define _LINUX_FOLIO_QUEUE_H 10 11 #include <linux/pagevec.h> 12 13 /* 14 * Segment in a queue of running buffers. Each segment can hold a number of 15 * folios and a portion of the queue can be referenced with the ITER_FOLIOQ 16 * iterator. The possibility exists of inserting non-folio elements into the 17 * queue (such as gaps). 18 * 19 * Explicit prev and next pointers are used instead of a list_head to make it 20 * easier to add segments to tail and remove them from the head without the 21 * need for a lock. 22 */ 23 struct folio_queue { 24 struct folio_batch vec; /* Folios in the queue segment */ 25 u8 orders[PAGEVEC_SIZE]; /* Order of each folio */ 26 struct folio_queue *next; /* Next queue segment or NULL */ 27 struct folio_queue *prev; /* Previous queue segment of NULL */ 28 unsigned long marks; /* 1-bit mark per folio */ 29 unsigned long marks2; /* Second 1-bit mark per folio */ 30 unsigned long marks3; /* Third 1-bit mark per folio */ 31 #if PAGEVEC_SIZE > BITS_PER_LONG 32 #error marks is not big enough 33 #endif 34 }; 35 36 static inline void folioq_init(struct folio_queue *folioq) 37 { 38 folio_batch_init(&folioq->vec); 39 folioq->next = NULL; 40 folioq->prev = NULL; 41 folioq->marks = 0; 42 folioq->marks2 = 0; 43 folioq->marks3 = 0; 44 } 45 46 static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq) 47 { 48 return PAGEVEC_SIZE; 49 } 50 51 static inline unsigned int folioq_count(struct folio_queue *folioq) 52 { 53 return folio_batch_count(&folioq->vec); 54 } 55 56 static inline bool folioq_full(struct folio_queue *folioq) 57 { 58 //return !folio_batch_space(&folioq->vec); 59 return folioq_count(folioq) >= folioq_nr_slots(folioq); 60 } 61 62 static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot) 63 { 64 return test_bit(slot, &folioq->marks); 65 } 66 67 static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot) 68 { 69 set_bit(slot, &folioq->marks); 70 } 71 72 static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot) 73 { 74 clear_bit(slot, &folioq->marks); 75 } 76 77 static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot) 78 { 79 return test_bit(slot, &folioq->marks2); 80 } 81 82 static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot) 83 { 84 set_bit(slot, &folioq->marks2); 85 } 86 87 static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot) 88 { 89 clear_bit(slot, &folioq->marks2); 90 } 91 92 static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot) 93 { 94 return test_bit(slot, &folioq->marks3); 95 } 96 97 static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot) 98 { 99 set_bit(slot, &folioq->marks3); 100 } 101 102 static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot) 103 { 104 clear_bit(slot, &folioq->marks3); 105 } 106 107 static inline unsigned int __folio_order(struct folio *folio) 108 { 109 if (!folio_test_large(folio)) 110 return 0; 111 return folio->_flags_1 & 0xff; 112 } 113 114 static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio) 115 { 116 unsigned int slot = folioq->vec.nr++; 117 118 folioq->vec.folios[slot] = folio; 119 folioq->orders[slot] = __folio_order(folio); 120 return slot; 121 } 122 123 static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio) 124 { 125 unsigned int slot = folioq->vec.nr++; 126 127 folioq->vec.folios[slot] = folio; 128 folioq->orders[slot] = __folio_order(folio); 129 folioq_mark(folioq, slot); 130 return slot; 131 } 132 133 static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot) 134 { 135 return folioq->vec.folios[slot]; 136 } 137 138 static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot) 139 { 140 return folioq->orders[slot]; 141 } 142 143 static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot) 144 { 145 return PAGE_SIZE << folioq_folio_order(folioq, slot); 146 } 147 148 static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot) 149 { 150 folioq->vec.folios[slot] = NULL; 151 folioq_unmark(folioq, slot); 152 folioq_unmark2(folioq, slot); 153 folioq_unmark3(folioq, slot); 154 } 155 156 #endif /* _LINUX_FOLIO_QUEUE_H */ 157