1 /* 2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License as published by the Free 6 * Software Foundation; either version 2 of the License, or (at your option) 7 * any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 59 16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * The full GNU General Public License is included in this distribution in the 19 * file called COPYING. 20 */ 21 #ifndef DMAENGINE_H 22 #define DMAENGINE_H 23 24 #ifdef CONFIG_DMA_ENGINE 25 26 #include <linux/device.h> 27 #include <linux/uio.h> 28 #include <linux/kref.h> 29 #include <linux/completion.h> 30 #include <linux/rcupdate.h> 31 32 /** 33 * enum dma_event - resource PNP/power managment events 34 * @DMA_RESOURCE_SUSPEND: DMA device going into low power state 35 * @DMA_RESOURCE_RESUME: DMA device returning to full power 36 * @DMA_RESOURCE_ADDED: DMA device added to the system 37 * @DMA_RESOURCE_REMOVED: DMA device removed from the system 38 */ 39 enum dma_event { 40 DMA_RESOURCE_SUSPEND, 41 DMA_RESOURCE_RESUME, 42 DMA_RESOURCE_ADDED, 43 DMA_RESOURCE_REMOVED, 44 }; 45 46 /** 47 * typedef dma_cookie_t 48 * 49 * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code 50 */ 51 typedef s32 dma_cookie_t; 52 53 #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0) 54 55 /** 56 * enum dma_status - DMA transaction status 57 * @DMA_SUCCESS: transaction completed successfully 58 * @DMA_IN_PROGRESS: transaction not yet processed 59 * @DMA_ERROR: transaction failed 60 */ 61 enum dma_status { 62 DMA_SUCCESS, 63 DMA_IN_PROGRESS, 64 DMA_ERROR, 65 }; 66 67 /** 68 * struct dma_chan_percpu - the per-CPU part of struct dma_chan 69 * @refcount: local_t used for open-coded "bigref" counting 70 * @memcpy_count: transaction counter 71 * @bytes_transferred: byte counter 72 */ 73 74 struct dma_chan_percpu { 75 local_t refcount; 76 /* stats */ 77 unsigned long memcpy_count; 78 unsigned long bytes_transferred; 79 }; 80 81 /** 82 * struct dma_chan - devices supply DMA channels, clients use them 83 * @client: ptr to the client user of this chan, will be NULL when unused 84 * @device: ptr to the dma device who supplies this channel, always !NULL 85 * @cookie: last cookie value returned to client 86 * @chan_id: 87 * @class_dev: 88 * @refcount: kref, used in "bigref" slow-mode 89 * @slow_ref: 90 * @rcu: 91 * @client_node: used to add this to the client chan list 92 * @device_node: used to add this to the device chan list 93 * @local: per-cpu pointer to a struct dma_chan_percpu 94 */ 95 struct dma_chan { 96 struct dma_client *client; 97 struct dma_device *device; 98 dma_cookie_t cookie; 99 100 /* sysfs */ 101 int chan_id; 102 struct class_device class_dev; 103 104 struct kref refcount; 105 int slow_ref; 106 struct rcu_head rcu; 107 108 struct list_head client_node; 109 struct list_head device_node; 110 struct dma_chan_percpu *local; 111 }; 112 113 void dma_chan_cleanup(struct kref *kref); 114 115 static inline void dma_chan_get(struct dma_chan *chan) 116 { 117 if (unlikely(chan->slow_ref)) 118 kref_get(&chan->refcount); 119 else { 120 local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount)); 121 put_cpu(); 122 } 123 } 124 125 static inline void dma_chan_put(struct dma_chan *chan) 126 { 127 if (unlikely(chan->slow_ref)) 128 kref_put(&chan->refcount, dma_chan_cleanup); 129 else { 130 local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount)); 131 put_cpu(); 132 } 133 } 134 135 /* 136 * typedef dma_event_callback - function pointer to a DMA event callback 137 */ 138 typedef void (*dma_event_callback) (struct dma_client *client, 139 struct dma_chan *chan, enum dma_event event); 140 141 /** 142 * struct dma_client - info on the entity making use of DMA services 143 * @event_callback: func ptr to call when something happens 144 * @chan_count: number of chans allocated 145 * @chans_desired: number of chans requested. Can be +/- chan_count 146 * @lock: protects access to the channels list 147 * @channels: the list of DMA channels allocated 148 * @global_node: list_head for global dma_client_list 149 */ 150 struct dma_client { 151 dma_event_callback event_callback; 152 unsigned int chan_count; 153 unsigned int chans_desired; 154 155 spinlock_t lock; 156 struct list_head channels; 157 struct list_head global_node; 158 }; 159 160 /** 161 * struct dma_device - info on the entity supplying DMA services 162 * @chancnt: how many DMA channels are supported 163 * @channels: the list of struct dma_chan 164 * @global_node: list_head for global dma_device_list 165 * @refcount: 166 * @done: 167 * @dev_id: 168 * Other func ptrs: used to make use of this device's capabilities 169 */ 170 struct dma_device { 171 172 unsigned int chancnt; 173 struct list_head channels; 174 struct list_head global_node; 175 176 struct kref refcount; 177 struct completion done; 178 179 int dev_id; 180 181 int (*device_alloc_chan_resources)(struct dma_chan *chan); 182 void (*device_free_chan_resources)(struct dma_chan *chan); 183 dma_cookie_t (*device_memcpy_buf_to_buf)(struct dma_chan *chan, 184 void *dest, void *src, size_t len); 185 dma_cookie_t (*device_memcpy_buf_to_pg)(struct dma_chan *chan, 186 struct page *page, unsigned int offset, void *kdata, 187 size_t len); 188 dma_cookie_t (*device_memcpy_pg_to_pg)(struct dma_chan *chan, 189 struct page *dest_pg, unsigned int dest_off, 190 struct page *src_pg, unsigned int src_off, size_t len); 191 enum dma_status (*device_memcpy_complete)(struct dma_chan *chan, 192 dma_cookie_t cookie, dma_cookie_t *last, 193 dma_cookie_t *used); 194 void (*device_memcpy_issue_pending)(struct dma_chan *chan); 195 }; 196 197 /* --- public DMA engine API --- */ 198 199 struct dma_client *dma_async_client_register(dma_event_callback event_callback); 200 void dma_async_client_unregister(struct dma_client *client); 201 void dma_async_client_chan_request(struct dma_client *client, 202 unsigned int number); 203 204 /** 205 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses 206 * @chan: DMA channel to offload copy to 207 * @dest: destination address (virtual) 208 * @src: source address (virtual) 209 * @len: length 210 * 211 * Both @dest and @src must be mappable to a bus address according to the 212 * DMA mapping API rules for streaming mappings. 213 * Both @dest and @src must stay memory resident (kernel memory or locked 214 * user space pages) 215 */ 216 static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan, 217 void *dest, void *src, size_t len) 218 { 219 int cpu = get_cpu(); 220 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 221 per_cpu_ptr(chan->local, cpu)->memcpy_count++; 222 put_cpu(); 223 224 return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len); 225 } 226 227 /** 228 * dma_async_memcpy_buf_to_pg - offloaded copy 229 * @chan: DMA channel to offload copy to 230 * @page: destination page 231 * @offset: offset in page to copy to 232 * @kdata: source address (virtual) 233 * @len: length 234 * 235 * Both @page/@offset and @kdata must be mappable to a bus address according 236 * to the DMA mapping API rules for streaming mappings. 237 * Both @page/@offset and @kdata must stay memory resident (kernel memory or 238 * locked user space pages) 239 */ 240 static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan, 241 struct page *page, unsigned int offset, void *kdata, size_t len) 242 { 243 int cpu = get_cpu(); 244 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 245 per_cpu_ptr(chan->local, cpu)->memcpy_count++; 246 put_cpu(); 247 248 return chan->device->device_memcpy_buf_to_pg(chan, page, offset, 249 kdata, len); 250 } 251 252 /** 253 * dma_async_memcpy_buf_to_pg - offloaded copy 254 * @chan: DMA channel to offload copy to 255 * @dest_page: destination page 256 * @dest_off: offset in page to copy to 257 * @src_page: source page 258 * @src_off: offset in page to copy from 259 * @len: length 260 * 261 * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus 262 * address according to the DMA mapping API rules for streaming mappings. 263 * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident 264 * (kernel memory or locked user space pages) 265 */ 266 static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan, 267 struct page *dest_pg, unsigned int dest_off, struct page *src_pg, 268 unsigned int src_off, size_t len) 269 { 270 int cpu = get_cpu(); 271 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 272 per_cpu_ptr(chan->local, cpu)->memcpy_count++; 273 put_cpu(); 274 275 return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off, 276 src_pg, src_off, len); 277 } 278 279 /** 280 * dma_async_memcpy_issue_pending - flush pending copies to HW 281 * @chan: 282 * 283 * This allows drivers to push copies to HW in batches, 284 * reducing MMIO writes where possible. 285 */ 286 static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan) 287 { 288 return chan->device->device_memcpy_issue_pending(chan); 289 } 290 291 /** 292 * dma_async_memcpy_complete - poll for transaction completion 293 * @chan: DMA channel 294 * @cookie: transaction identifier to check status of 295 * @last: returns last completed cookie, can be NULL 296 * @used: returns last issued cookie, can be NULL 297 * 298 * If @last and @used are passed in, upon return they reflect the driver 299 * internal state and can be used with dma_async_is_complete() to check 300 * the status of multiple cookies without re-checking hardware state. 301 */ 302 static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan, 303 dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used) 304 { 305 return chan->device->device_memcpy_complete(chan, cookie, last, used); 306 } 307 308 /** 309 * dma_async_is_complete - test a cookie against chan state 310 * @cookie: transaction identifier to test status of 311 * @last_complete: last know completed transaction 312 * @last_used: last cookie value handed out 313 * 314 * dma_async_is_complete() is used in dma_async_memcpy_complete() 315 * the test logic is seperated for lightweight testing of multiple cookies 316 */ 317 static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie, 318 dma_cookie_t last_complete, dma_cookie_t last_used) 319 { 320 if (last_complete <= last_used) { 321 if ((cookie <= last_complete) || (cookie > last_used)) 322 return DMA_SUCCESS; 323 } else { 324 if ((cookie <= last_complete) && (cookie > last_used)) 325 return DMA_SUCCESS; 326 } 327 return DMA_IN_PROGRESS; 328 } 329 330 331 /* --- DMA device --- */ 332 333 int dma_async_device_register(struct dma_device *device); 334 void dma_async_device_unregister(struct dma_device *device); 335 336 /* --- Helper iov-locking functions --- */ 337 338 struct dma_page_list { 339 char *base_address; 340 int nr_pages; 341 struct page **pages; 342 }; 343 344 struct dma_pinned_list { 345 int nr_iovecs; 346 struct dma_page_list page_list[0]; 347 }; 348 349 struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len); 350 void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list); 351 352 dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, 353 struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len); 354 dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, 355 struct dma_pinned_list *pinned_list, struct page *page, 356 unsigned int offset, size_t len); 357 358 #endif /* CONFIG_DMA_ENGINE */ 359 #endif /* DMAENGINE_H */ 360