1 /* 2 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> 3 * 4 * Provides a framework for enqueueing and running callbacks from hardirq 5 * context. The enqueueing is NMI-safe. 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/irq_work.h> 11 #include <linux/hardirq.h> 12 13 /* 14 * An entry can be in one of four states: 15 * 16 * free NULL, 0 -> {claimed} : free to be used 17 * claimed NULL, 3 -> {pending} : claimed to be enqueued 18 * pending next, 3 -> {busy} : queued, pending callback 19 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed 20 */ 21 22 #define IRQ_WORK_PENDING 1UL 23 #define IRQ_WORK_BUSY 2UL 24 #define IRQ_WORK_FLAGS 3UL 25 26 static DEFINE_PER_CPU(struct llist_head, irq_work_list); 27 28 /* 29 * Claim the entry so that no one else will poke at it. 30 */ 31 static bool irq_work_claim(struct irq_work *work) 32 { 33 unsigned long flags, nflags; 34 35 for (;;) { 36 flags = work->flags; 37 if (flags & IRQ_WORK_PENDING) 38 return false; 39 nflags = flags | IRQ_WORK_FLAGS; 40 if (cmpxchg(&work->flags, flags, nflags) == flags) 41 break; 42 cpu_relax(); 43 } 44 45 return true; 46 } 47 48 void __weak arch_irq_work_raise(void) 49 { 50 /* 51 * Lame architectures will get the timer tick callback 52 */ 53 } 54 55 /* 56 * Queue the entry and raise the IPI if needed. 57 */ 58 static void __irq_work_queue(struct irq_work *work) 59 { 60 bool empty; 61 62 preempt_disable(); 63 64 empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list)); 65 /* The list was empty, raise self-interrupt to start processing. */ 66 if (empty) 67 arch_irq_work_raise(); 68 69 preempt_enable(); 70 } 71 72 /* 73 * Enqueue the irq_work @entry, returns true on success, failure when the 74 * @entry was already enqueued by someone else. 75 * 76 * Can be re-enqueued while the callback is still in progress. 77 */ 78 bool irq_work_queue(struct irq_work *work) 79 { 80 if (!irq_work_claim(work)) { 81 /* 82 * Already enqueued, can't do! 83 */ 84 return false; 85 } 86 87 __irq_work_queue(work); 88 return true; 89 } 90 EXPORT_SYMBOL_GPL(irq_work_queue); 91 92 /* 93 * Run the irq_work entries on this cpu. Requires to be ran from hardirq 94 * context with local IRQs disabled. 95 */ 96 void irq_work_run(void) 97 { 98 struct irq_work *work; 99 struct llist_head *this_list; 100 struct llist_node *llnode; 101 102 this_list = &__get_cpu_var(irq_work_list); 103 if (llist_empty(this_list)) 104 return; 105 106 BUG_ON(!in_irq()); 107 BUG_ON(!irqs_disabled()); 108 109 llnode = llist_del_all(this_list); 110 while (llnode != NULL) { 111 work = llist_entry(llnode, struct irq_work, llnode); 112 113 llnode = llist_next(llnode); 114 115 /* 116 * Clear the PENDING bit, after this point the @work 117 * can be re-used. 118 */ 119 work->flags = IRQ_WORK_BUSY; 120 work->func(work); 121 /* 122 * Clear the BUSY bit and return to the free state if 123 * no-one else claimed it meanwhile. 124 */ 125 (void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0); 126 } 127 } 128 EXPORT_SYMBOL_GPL(irq_work_run); 129 130 /* 131 * Synchronize against the irq_work @entry, ensures the entry is not 132 * currently in use. 133 */ 134 void irq_work_sync(struct irq_work *work) 135 { 136 WARN_ON_ONCE(irqs_disabled()); 137 138 while (work->flags & IRQ_WORK_BUSY) 139 cpu_relax(); 140 } 141 EXPORT_SYMBOL_GPL(irq_work_sync); 142