1 // SPDX-License-Identifier: GPL-2.0 2 #include "bcachefs.h" 3 #include "clock.h" 4 5 #include <linux/freezer.h> 6 #include <linux/kthread.h> 7 #include <linux/preempt.h> 8 9 static inline bool io_timer_cmp(const void *l, const void *r, void __always_unused *args) 10 { 11 struct io_timer **_l = (struct io_timer **)l; 12 struct io_timer **_r = (struct io_timer **)r; 13 14 return (*_l)->expire < (*_r)->expire; 15 } 16 17 static inline void io_timer_swp(void *l, void *r, void __always_unused *args) 18 { 19 struct io_timer **_l = (struct io_timer **)l; 20 struct io_timer **_r = (struct io_timer **)r; 21 22 swap(*_l, *_r); 23 } 24 25 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer) 26 { 27 size_t i; 28 const struct min_heap_callbacks callbacks = { 29 .less = io_timer_cmp, 30 .swp = io_timer_swp, 31 }; 32 33 spin_lock(&clock->timer_lock); 34 35 if (time_after_eq((unsigned long) atomic64_read(&clock->now), 36 timer->expire)) { 37 spin_unlock(&clock->timer_lock); 38 timer->fn(timer); 39 return; 40 } 41 42 for (i = 0; i < clock->timers.nr; i++) 43 if (clock->timers.data[i] == timer) 44 goto out; 45 46 BUG_ON(!min_heap_push(&clock->timers, &timer, &callbacks, NULL)); 47 out: 48 spin_unlock(&clock->timer_lock); 49 } 50 51 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer) 52 { 53 size_t i; 54 const struct min_heap_callbacks callbacks = { 55 .less = io_timer_cmp, 56 .swp = io_timer_swp, 57 }; 58 59 spin_lock(&clock->timer_lock); 60 61 for (i = 0; i < clock->timers.nr; i++) 62 if (clock->timers.data[i] == timer) { 63 min_heap_del(&clock->timers, i, &callbacks, NULL); 64 break; 65 } 66 67 spin_unlock(&clock->timer_lock); 68 } 69 70 struct io_clock_wait { 71 struct io_timer io_timer; 72 struct timer_list cpu_timer; 73 struct task_struct *task; 74 int expired; 75 }; 76 77 static void io_clock_wait_fn(struct io_timer *timer) 78 { 79 struct io_clock_wait *wait = container_of(timer, 80 struct io_clock_wait, io_timer); 81 82 wait->expired = 1; 83 wake_up_process(wait->task); 84 } 85 86 static void io_clock_cpu_timeout(struct timer_list *timer) 87 { 88 struct io_clock_wait *wait = container_of(timer, 89 struct io_clock_wait, cpu_timer); 90 91 wait->expired = 1; 92 wake_up_process(wait->task); 93 } 94 95 void bch2_io_clock_schedule_timeout(struct io_clock *clock, unsigned long until) 96 { 97 struct io_clock_wait wait; 98 99 /* XXX: calculate sleep time rigorously */ 100 wait.io_timer.expire = until; 101 wait.io_timer.fn = io_clock_wait_fn; 102 wait.task = current; 103 wait.expired = 0; 104 bch2_io_timer_add(clock, &wait.io_timer); 105 106 schedule(); 107 108 bch2_io_timer_del(clock, &wait.io_timer); 109 } 110 111 void bch2_kthread_io_clock_wait(struct io_clock *clock, 112 unsigned long io_until, 113 unsigned long cpu_timeout) 114 { 115 bool kthread = (current->flags & PF_KTHREAD) != 0; 116 struct io_clock_wait wait; 117 118 wait.io_timer.expire = io_until; 119 wait.io_timer.fn = io_clock_wait_fn; 120 wait.task = current; 121 wait.expired = 0; 122 bch2_io_timer_add(clock, &wait.io_timer); 123 124 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0); 125 126 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT) 127 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies); 128 129 do { 130 set_current_state(TASK_INTERRUPTIBLE); 131 if (kthread && kthread_should_stop()) 132 break; 133 134 if (wait.expired) 135 break; 136 137 schedule(); 138 try_to_freeze(); 139 } while (0); 140 141 __set_current_state(TASK_RUNNING); 142 del_timer_sync(&wait.cpu_timer); 143 destroy_timer_on_stack(&wait.cpu_timer); 144 bch2_io_timer_del(clock, &wait.io_timer); 145 } 146 147 static struct io_timer *get_expired_timer(struct io_clock *clock, 148 unsigned long now) 149 { 150 struct io_timer *ret = NULL; 151 const struct min_heap_callbacks callbacks = { 152 .less = io_timer_cmp, 153 .swp = io_timer_swp, 154 }; 155 156 spin_lock(&clock->timer_lock); 157 158 if (clock->timers.nr && 159 time_after_eq(now, clock->timers.data[0]->expire)) { 160 ret = *min_heap_peek(&clock->timers); 161 min_heap_pop(&clock->timers, &callbacks, NULL); 162 } 163 164 spin_unlock(&clock->timer_lock); 165 166 return ret; 167 } 168 169 void __bch2_increment_clock(struct io_clock *clock, unsigned sectors) 170 { 171 struct io_timer *timer; 172 unsigned long now = atomic64_add_return(sectors, &clock->now); 173 174 while ((timer = get_expired_timer(clock, now))) 175 timer->fn(timer); 176 } 177 178 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock) 179 { 180 unsigned long now; 181 unsigned i; 182 183 out->atomic++; 184 spin_lock(&clock->timer_lock); 185 now = atomic64_read(&clock->now); 186 187 for (i = 0; i < clock->timers.nr; i++) 188 prt_printf(out, "%ps:\t%li\n", 189 clock->timers.data[i]->fn, 190 clock->timers.data[i]->expire - now); 191 spin_unlock(&clock->timer_lock); 192 --out->atomic; 193 } 194 195 void bch2_io_clock_exit(struct io_clock *clock) 196 { 197 free_heap(&clock->timers); 198 free_percpu(clock->pcpu_buf); 199 } 200 201 int bch2_io_clock_init(struct io_clock *clock) 202 { 203 atomic64_set(&clock->now, 0); 204 spin_lock_init(&clock->timer_lock); 205 206 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus(); 207 208 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf); 209 if (!clock->pcpu_buf) 210 return -BCH_ERR_ENOMEM_io_clock_init; 211 212 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL)) 213 return -BCH_ERR_ENOMEM_io_clock_init; 214 215 return 0; 216 } 217