1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PIPE_FS_I_H 3 #define _LINUX_PIPE_FS_I_H 4 5 #define PIPE_DEF_BUFFERS 16 6 7 #define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */ 8 #define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */ 9 #define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */ 10 #define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */ 11 #define PIPE_BUF_FLAG_CAN_MERGE 0x10 /* can merge buffers */ 12 #define PIPE_BUF_FLAG_WHOLE 0x20 /* read() must return entire buffer or error */ 13 #ifdef CONFIG_WATCH_QUEUE 14 #define PIPE_BUF_FLAG_LOSS 0x40 /* Message loss happened after this buffer */ 15 #endif 16 17 /** 18 * struct pipe_buffer - a linux kernel pipe buffer 19 * @page: the page containing the data for the pipe buffer 20 * @offset: offset of data inside the @page 21 * @len: length of data inside the @page 22 * @ops: operations associated with this buffer. See @pipe_buf_operations. 23 * @flags: pipe buffer flags. See above. 24 * @private: private data owned by the ops. 25 **/ 26 struct pipe_buffer { 27 struct page *page; 28 unsigned int offset, len; 29 const struct pipe_buf_operations *ops; 30 unsigned int flags; 31 unsigned long private; 32 }; 33 34 /* 35 * Really only alpha needs 32-bit fields, but 36 * might as well do it for 64-bit architectures 37 * since that's what we've historically done, 38 * and it makes 'head_tail' always be a simple 39 * 'unsigned long'. 40 */ 41 #ifdef CONFIG_64BIT 42 typedef unsigned int pipe_index_t; 43 #else 44 typedef unsigned short pipe_index_t; 45 #endif 46 47 /* 48 * We have to declare this outside 'struct pipe_inode_info', 49 * but then we can't use 'union pipe_index' for an anonymous 50 * union, so we end up having to duplicate this declaration 51 * below. Annoying. 52 */ 53 union pipe_index { 54 unsigned long head_tail; 55 struct { 56 pipe_index_t head; 57 pipe_index_t tail; 58 }; 59 }; 60 61 /** 62 * struct pipe_inode_info - a linux kernel pipe 63 * @mutex: mutex protecting the whole thing 64 * @rd_wait: reader wait point in case of empty pipe 65 * @wr_wait: writer wait point in case of full pipe 66 * @head: The point of buffer production 67 * @tail: The point of buffer consumption 68 * @head_tail: unsigned long union of @head and @tail 69 * @note_loss: The next read() should insert a data-lost message 70 * @max_usage: The maximum number of slots that may be used in the ring 71 * @ring_size: total number of buffers (should be a power of 2) 72 * @nr_accounted: The amount this pipe accounts for in user->pipe_bufs 73 * @tmp_page: cached released page 74 * @readers: number of current readers of this pipe 75 * @writers: number of current writers of this pipe 76 * @files: number of struct file referring this pipe (protected by ->i_lock) 77 * @r_counter: reader counter 78 * @w_counter: writer counter 79 * @poll_usage: is this pipe used for epoll, which has crazy wakeups? 80 * @fasync_readers: reader side fasync 81 * @fasync_writers: writer side fasync 82 * @bufs: the circular array of pipe buffers 83 * @user: the user who created this pipe 84 * @watch_queue: If this pipe is a watch_queue, this is the stuff for that 85 **/ 86 struct pipe_inode_info { 87 struct mutex mutex; 88 wait_queue_head_t rd_wait, wr_wait; 89 90 /* This has to match the 'union pipe_index' above */ 91 union { 92 unsigned long head_tail; 93 struct { 94 pipe_index_t head; 95 pipe_index_t tail; 96 }; 97 }; 98 99 unsigned int max_usage; 100 unsigned int ring_size; 101 unsigned int nr_accounted; 102 unsigned int readers; 103 unsigned int writers; 104 unsigned int files; 105 unsigned int r_counter; 106 unsigned int w_counter; 107 bool poll_usage; 108 #ifdef CONFIG_WATCH_QUEUE 109 bool note_loss; 110 #endif 111 struct page *tmp_page; 112 struct fasync_struct *fasync_readers; 113 struct fasync_struct *fasync_writers; 114 struct pipe_buffer *bufs; 115 struct user_struct *user; 116 #ifdef CONFIG_WATCH_QUEUE 117 struct watch_queue *watch_queue; 118 #endif 119 }; 120 121 /* 122 * Note on the nesting of these functions: 123 * 124 * ->confirm() 125 * ->try_steal() 126 * 127 * That is, ->try_steal() must be called on a confirmed buffer. See below for 128 * the meaning of each operation. Also see the kerneldoc in fs/pipe.c for the 129 * pipe and generic variants of these hooks. 130 */ 131 struct pipe_buf_operations { 132 /* 133 * ->confirm() verifies that the data in the pipe buffer is there 134 * and that the contents are good. If the pages in the pipe belong 135 * to a file system, we may need to wait for IO completion in this 136 * hook. Returns 0 for good, or a negative error value in case of 137 * error. If not present all pages are considered good. 138 */ 139 int (*confirm)(struct pipe_inode_info *, struct pipe_buffer *); 140 141 /* 142 * When the contents of this pipe buffer has been completely 143 * consumed by a reader, ->release() is called. 144 */ 145 void (*release)(struct pipe_inode_info *, struct pipe_buffer *); 146 147 /* 148 * Attempt to take ownership of the pipe buffer and its contents. 149 * ->try_steal() returns %true for success, in which case the contents 150 * of the pipe (the buf->page) is locked and now completely owned by the 151 * caller. The page may then be transferred to a different mapping, the 152 * most often used case is insertion into different file address space 153 * cache. 154 */ 155 bool (*try_steal)(struct pipe_inode_info *, struct pipe_buffer *); 156 157 /* 158 * Get a reference to the pipe buffer. 159 */ 160 bool (*get)(struct pipe_inode_info *, struct pipe_buffer *); 161 }; 162 163 /** 164 * pipe_has_watch_queue - Check whether the pipe is a watch_queue, 165 * i.e. it was created with O_NOTIFICATION_PIPE 166 * @pipe: The pipe to check 167 * 168 * Return: true if pipe is a watch queue, false otherwise. 169 */ 170 static inline bool pipe_has_watch_queue(const struct pipe_inode_info *pipe) 171 { 172 #ifdef CONFIG_WATCH_QUEUE 173 return pipe->watch_queue != NULL; 174 #else 175 return false; 176 #endif 177 } 178 179 /** 180 * pipe_occupancy - Return number of slots used in the pipe 181 * @head: The pipe ring head pointer 182 * @tail: The pipe ring tail pointer 183 */ 184 static inline unsigned int pipe_occupancy(unsigned int head, unsigned int tail) 185 { 186 return (pipe_index_t)(head - tail); 187 } 188 189 /** 190 * pipe_empty - Return true if the pipe is empty 191 * @head: The pipe ring head pointer 192 * @tail: The pipe ring tail pointer 193 */ 194 static inline bool pipe_empty(unsigned int head, unsigned int tail) 195 { 196 return !pipe_occupancy(head, tail); 197 } 198 199 /** 200 * pipe_full - Return true if the pipe is full 201 * @head: The pipe ring head pointer 202 * @tail: The pipe ring tail pointer 203 * @limit: The maximum amount of slots available. 204 */ 205 static inline bool pipe_full(unsigned int head, unsigned int tail, 206 unsigned int limit) 207 { 208 return pipe_occupancy(head, tail) >= limit; 209 } 210 211 /** 212 * pipe_buf - Return the pipe buffer for the specified slot in the pipe ring 213 * @pipe: The pipe to access 214 * @slot: The slot of interest 215 */ 216 static inline struct pipe_buffer *pipe_buf(const struct pipe_inode_info *pipe, 217 unsigned int slot) 218 { 219 return &pipe->bufs[slot & (pipe->ring_size - 1)]; 220 } 221 222 /** 223 * pipe_head_buf - Return the pipe buffer at the head of the pipe ring 224 * @pipe: The pipe to access 225 */ 226 static inline struct pipe_buffer *pipe_head_buf(const struct pipe_inode_info *pipe) 227 { 228 return pipe_buf(pipe, pipe->head); 229 } 230 231 /** 232 * pipe_buf_get - get a reference to a pipe_buffer 233 * @pipe: the pipe that the buffer belongs to 234 * @buf: the buffer to get a reference to 235 * 236 * Return: %true if the reference was successfully obtained. 237 */ 238 static inline __must_check bool pipe_buf_get(struct pipe_inode_info *pipe, 239 struct pipe_buffer *buf) 240 { 241 return buf->ops->get(pipe, buf); 242 } 243 244 /** 245 * pipe_buf_release - put a reference to a pipe_buffer 246 * @pipe: the pipe that the buffer belongs to 247 * @buf: the buffer to put a reference to 248 */ 249 static inline void pipe_buf_release(struct pipe_inode_info *pipe, 250 struct pipe_buffer *buf) 251 { 252 const struct pipe_buf_operations *ops = buf->ops; 253 254 buf->ops = NULL; 255 ops->release(pipe, buf); 256 } 257 258 /** 259 * pipe_buf_confirm - verify contents of the pipe buffer 260 * @pipe: the pipe that the buffer belongs to 261 * @buf: the buffer to confirm 262 */ 263 static inline int pipe_buf_confirm(struct pipe_inode_info *pipe, 264 struct pipe_buffer *buf) 265 { 266 if (!buf->ops->confirm) 267 return 0; 268 return buf->ops->confirm(pipe, buf); 269 } 270 271 /** 272 * pipe_buf_try_steal - attempt to take ownership of a pipe_buffer 273 * @pipe: the pipe that the buffer belongs to 274 * @buf: the buffer to attempt to steal 275 */ 276 static inline bool pipe_buf_try_steal(struct pipe_inode_info *pipe, 277 struct pipe_buffer *buf) 278 { 279 if (!buf->ops->try_steal) 280 return false; 281 return buf->ops->try_steal(pipe, buf); 282 } 283 284 /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual 285 memory allocation, whereas PIPE_BUF makes atomicity guarantees. */ 286 #define PIPE_SIZE PAGE_SIZE 287 288 /* Pipe lock and unlock operations */ 289 void pipe_lock(struct pipe_inode_info *); 290 void pipe_unlock(struct pipe_inode_info *); 291 void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *); 292 293 /* Wait for a pipe to be readable/writable while dropping the pipe lock */ 294 void pipe_wait_readable(struct pipe_inode_info *); 295 void pipe_wait_writable(struct pipe_inode_info *); 296 297 struct pipe_inode_info *alloc_pipe_info(void); 298 void free_pipe_info(struct pipe_inode_info *); 299 300 /* Generic pipe buffer ops functions */ 301 bool generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *); 302 bool generic_pipe_buf_try_steal(struct pipe_inode_info *, struct pipe_buffer *); 303 void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *); 304 305 extern const struct pipe_buf_operations nosteal_pipe_buf_ops; 306 307 unsigned long account_pipe_buffers(struct user_struct *user, 308 unsigned long old, unsigned long new); 309 bool too_many_pipe_buffers_soft(unsigned long user_bufs); 310 bool too_many_pipe_buffers_hard(unsigned long user_bufs); 311 bool pipe_is_unprivileged_user(void); 312 313 /* for F_SETPIPE_SZ and F_GETPIPE_SZ */ 314 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots); 315 long pipe_fcntl(struct file *, unsigned int, unsigned int arg); 316 struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice); 317 318 int create_pipe_files(struct file **, int); 319 unsigned int round_pipe_size(unsigned int size); 320 321 #endif 322