1 /* 2 * core routines for the asynchronous memory transfer/transform api 3 * 4 * Copyright © 2006, Intel Corporation. 5 * 6 * Dan Williams <[email protected]> 7 * 8 * with architecture considerations by: 9 * Neil Brown <[email protected]> 10 * Jeff Garzik <[email protected]> 11 * 12 * This program is free software; you can redistribute it and/or modify it 13 * under the terms and conditions of the GNU General Public License, 14 * version 2, as published by the Free Software Foundation. 15 * 16 * This program is distributed in the hope it will be useful, but WITHOUT 17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 19 * more details. 20 * 21 * You should have received a copy of the GNU General Public License along with 22 * this program; if not, write to the Free Software Foundation, Inc., 23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 24 * 25 */ 26 #include <linux/rculist.h> 27 #include <linux/kernel.h> 28 #include <linux/async_tx.h> 29 30 #ifdef CONFIG_DMA_ENGINE 31 static enum dma_state_client 32 dma_channel_add_remove(struct dma_client *client, 33 struct dma_chan *chan, enum dma_state state); 34 35 static struct dma_client async_tx_dma = { 36 .event_callback = dma_channel_add_remove, 37 /* .cap_mask == 0 defaults to all channels */ 38 }; 39 40 /** 41 * async_tx_lock - protect modification of async_tx_master_list and serialize 42 * rebalance operations 43 */ 44 static DEFINE_SPINLOCK(async_tx_lock); 45 46 static LIST_HEAD(async_tx_master_list); 47 48 static void 49 free_dma_chan_ref(struct rcu_head *rcu) 50 { 51 struct dma_chan_ref *ref; 52 ref = container_of(rcu, struct dma_chan_ref, rcu); 53 kfree(ref); 54 } 55 56 static void 57 init_dma_chan_ref(struct dma_chan_ref *ref, struct dma_chan *chan) 58 { 59 INIT_LIST_HEAD(&ref->node); 60 INIT_RCU_HEAD(&ref->rcu); 61 ref->chan = chan; 62 atomic_set(&ref->count, 0); 63 } 64 65 static enum dma_state_client 66 dma_channel_add_remove(struct dma_client *client, 67 struct dma_chan *chan, enum dma_state state) 68 { 69 unsigned long found, flags; 70 struct dma_chan_ref *master_ref, *ref; 71 enum dma_state_client ack = DMA_DUP; /* default: take no action */ 72 73 switch (state) { 74 case DMA_RESOURCE_AVAILABLE: 75 found = 0; 76 rcu_read_lock(); 77 list_for_each_entry_rcu(ref, &async_tx_master_list, node) 78 if (ref->chan == chan) { 79 found = 1; 80 break; 81 } 82 rcu_read_unlock(); 83 84 pr_debug("async_tx: dma resource available [%s]\n", 85 found ? "old" : "new"); 86 87 if (!found) 88 ack = DMA_ACK; 89 else 90 break; 91 92 /* add the channel to the generic management list */ 93 master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL); 94 if (master_ref) { 95 init_dma_chan_ref(master_ref, chan); 96 spin_lock_irqsave(&async_tx_lock, flags); 97 list_add_tail_rcu(&master_ref->node, 98 &async_tx_master_list); 99 spin_unlock_irqrestore(&async_tx_lock, 100 flags); 101 } else { 102 printk(KERN_WARNING "async_tx: unable to create" 103 " new master entry in response to" 104 " a DMA_RESOURCE_ADDED event" 105 " (-ENOMEM)\n"); 106 return 0; 107 } 108 break; 109 case DMA_RESOURCE_REMOVED: 110 found = 0; 111 spin_lock_irqsave(&async_tx_lock, flags); 112 list_for_each_entry(ref, &async_tx_master_list, node) 113 if (ref->chan == chan) { 114 list_del_rcu(&ref->node); 115 call_rcu(&ref->rcu, free_dma_chan_ref); 116 found = 1; 117 break; 118 } 119 spin_unlock_irqrestore(&async_tx_lock, flags); 120 121 pr_debug("async_tx: dma resource removed [%s]\n", 122 found ? "ours" : "not ours"); 123 124 if (found) 125 ack = DMA_ACK; 126 else 127 break; 128 break; 129 case DMA_RESOURCE_SUSPEND: 130 case DMA_RESOURCE_RESUME: 131 printk(KERN_WARNING "async_tx: does not support dma channel" 132 " suspend/resume\n"); 133 break; 134 default: 135 BUG(); 136 } 137 138 return ack; 139 } 140 141 static int __init async_tx_init(void) 142 { 143 dma_async_client_register(&async_tx_dma); 144 dma_async_client_chan_request(&async_tx_dma); 145 146 printk(KERN_INFO "async_tx: api initialized (async)\n"); 147 148 return 0; 149 } 150 151 static void __exit async_tx_exit(void) 152 { 153 dma_async_client_unregister(&async_tx_dma); 154 } 155 156 /** 157 * __async_tx_find_channel - find a channel to carry out the operation or let 158 * the transaction execute synchronously 159 * @depend_tx: transaction dependency 160 * @tx_type: transaction type 161 */ 162 struct dma_chan * 163 __async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx, 164 enum dma_transaction_type tx_type) 165 { 166 /* see if we can keep the chain on one channel */ 167 if (depend_tx && 168 dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) 169 return depend_tx->chan; 170 return dma_find_channel(tx_type); 171 } 172 EXPORT_SYMBOL_GPL(__async_tx_find_channel); 173 #else 174 static int __init async_tx_init(void) 175 { 176 printk(KERN_INFO "async_tx: api initialized (sync-only)\n"); 177 return 0; 178 } 179 180 static void __exit async_tx_exit(void) 181 { 182 do { } while (0); 183 } 184 #endif 185 186 187 /** 188 * async_tx_channel_switch - queue an interrupt descriptor with a dependency 189 * pre-attached. 190 * @depend_tx: the operation that must finish before the new operation runs 191 * @tx: the new operation 192 */ 193 static void 194 async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, 195 struct dma_async_tx_descriptor *tx) 196 { 197 struct dma_chan *chan; 198 struct dma_device *device; 199 struct dma_async_tx_descriptor *intr_tx = (void *) ~0; 200 201 /* first check to see if we can still append to depend_tx */ 202 spin_lock_bh(&depend_tx->lock); 203 if (depend_tx->parent && depend_tx->chan == tx->chan) { 204 tx->parent = depend_tx; 205 depend_tx->next = tx; 206 intr_tx = NULL; 207 } 208 spin_unlock_bh(&depend_tx->lock); 209 210 if (!intr_tx) 211 return; 212 213 chan = depend_tx->chan; 214 device = chan->device; 215 216 /* see if we can schedule an interrupt 217 * otherwise poll for completion 218 */ 219 if (dma_has_cap(DMA_INTERRUPT, device->cap_mask)) 220 intr_tx = device->device_prep_dma_interrupt(chan, 0); 221 else 222 intr_tx = NULL; 223 224 if (intr_tx) { 225 intr_tx->callback = NULL; 226 intr_tx->callback_param = NULL; 227 tx->parent = intr_tx; 228 /* safe to set ->next outside the lock since we know we are 229 * not submitted yet 230 */ 231 intr_tx->next = tx; 232 233 /* check if we need to append */ 234 spin_lock_bh(&depend_tx->lock); 235 if (depend_tx->parent) { 236 intr_tx->parent = depend_tx; 237 depend_tx->next = intr_tx; 238 async_tx_ack(intr_tx); 239 intr_tx = NULL; 240 } 241 spin_unlock_bh(&depend_tx->lock); 242 243 if (intr_tx) { 244 intr_tx->parent = NULL; 245 intr_tx->tx_submit(intr_tx); 246 async_tx_ack(intr_tx); 247 } 248 } else { 249 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) 250 panic("%s: DMA_ERROR waiting for depend_tx\n", 251 __func__); 252 tx->tx_submit(tx); 253 } 254 } 255 256 257 /** 258 * submit_disposition - while holding depend_tx->lock we must avoid submitting 259 * new operations to prevent a circular locking dependency with 260 * drivers that already hold a channel lock when calling 261 * async_tx_run_dependencies. 262 * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock 263 * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch 264 * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly 265 */ 266 enum submit_disposition { 267 ASYNC_TX_SUBMITTED, 268 ASYNC_TX_CHANNEL_SWITCH, 269 ASYNC_TX_DIRECT_SUBMIT, 270 }; 271 272 void 273 async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, 274 enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, 275 dma_async_tx_callback cb_fn, void *cb_param) 276 { 277 tx->callback = cb_fn; 278 tx->callback_param = cb_param; 279 280 if (depend_tx) { 281 enum submit_disposition s; 282 283 /* sanity check the dependency chain: 284 * 1/ if ack is already set then we cannot be sure 285 * we are referring to the correct operation 286 * 2/ dependencies are 1:1 i.e. two transactions can 287 * not depend on the same parent 288 */ 289 BUG_ON(async_tx_test_ack(depend_tx) || depend_tx->next || 290 tx->parent); 291 292 /* the lock prevents async_tx_run_dependencies from missing 293 * the setting of ->next when ->parent != NULL 294 */ 295 spin_lock_bh(&depend_tx->lock); 296 if (depend_tx->parent) { 297 /* we have a parent so we can not submit directly 298 * if we are staying on the same channel: append 299 * else: channel switch 300 */ 301 if (depend_tx->chan == chan) { 302 tx->parent = depend_tx; 303 depend_tx->next = tx; 304 s = ASYNC_TX_SUBMITTED; 305 } else 306 s = ASYNC_TX_CHANNEL_SWITCH; 307 } else { 308 /* we do not have a parent so we may be able to submit 309 * directly if we are staying on the same channel 310 */ 311 if (depend_tx->chan == chan) 312 s = ASYNC_TX_DIRECT_SUBMIT; 313 else 314 s = ASYNC_TX_CHANNEL_SWITCH; 315 } 316 spin_unlock_bh(&depend_tx->lock); 317 318 switch (s) { 319 case ASYNC_TX_SUBMITTED: 320 break; 321 case ASYNC_TX_CHANNEL_SWITCH: 322 async_tx_channel_switch(depend_tx, tx); 323 break; 324 case ASYNC_TX_DIRECT_SUBMIT: 325 tx->parent = NULL; 326 tx->tx_submit(tx); 327 break; 328 } 329 } else { 330 tx->parent = NULL; 331 tx->tx_submit(tx); 332 } 333 334 if (flags & ASYNC_TX_ACK) 335 async_tx_ack(tx); 336 337 if (depend_tx && (flags & ASYNC_TX_DEP_ACK)) 338 async_tx_ack(depend_tx); 339 } 340 EXPORT_SYMBOL_GPL(async_tx_submit); 341 342 /** 343 * async_trigger_callback - schedules the callback function to be run after 344 * any dependent operations have been completed. 345 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK 346 * @depend_tx: 'callback' requires the completion of this transaction 347 * @cb_fn: function to call after depend_tx completes 348 * @cb_param: parameter to pass to the callback routine 349 */ 350 struct dma_async_tx_descriptor * 351 async_trigger_callback(enum async_tx_flags flags, 352 struct dma_async_tx_descriptor *depend_tx, 353 dma_async_tx_callback cb_fn, void *cb_param) 354 { 355 struct dma_chan *chan; 356 struct dma_device *device; 357 struct dma_async_tx_descriptor *tx; 358 359 if (depend_tx) { 360 chan = depend_tx->chan; 361 device = chan->device; 362 363 /* see if we can schedule an interrupt 364 * otherwise poll for completion 365 */ 366 if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask)) 367 device = NULL; 368 369 tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL; 370 } else 371 tx = NULL; 372 373 if (tx) { 374 pr_debug("%s: (async)\n", __func__); 375 376 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); 377 } else { 378 pr_debug("%s: (sync)\n", __func__); 379 380 /* wait for any prerequisite operations */ 381 async_tx_quiesce(&depend_tx); 382 383 async_tx_sync_epilog(cb_fn, cb_param); 384 } 385 386 return tx; 387 } 388 EXPORT_SYMBOL_GPL(async_trigger_callback); 389 390 /** 391 * async_tx_quiesce - ensure tx is complete and freeable upon return 392 * @tx - transaction to quiesce 393 */ 394 void async_tx_quiesce(struct dma_async_tx_descriptor **tx) 395 { 396 if (*tx) { 397 /* if ack is already set then we cannot be sure 398 * we are referring to the correct operation 399 */ 400 BUG_ON(async_tx_test_ack(*tx)); 401 if (dma_wait_for_async_tx(*tx) == DMA_ERROR) 402 panic("DMA_ERROR waiting for transaction\n"); 403 async_tx_ack(*tx); 404 *tx = NULL; 405 } 406 } 407 EXPORT_SYMBOL_GPL(async_tx_quiesce); 408 409 module_init(async_tx_init); 410 module_exit(async_tx_exit); 411 412 MODULE_AUTHOR("Intel Corporation"); 413 MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API"); 414 MODULE_LICENSE("GPL"); 415