1 // SPDX-License-Identifier: GPL-2.0 2 #include "bcachefs.h" 3 #include "clock.h" 4 5 #include <linux/freezer.h> 6 #include <linux/kthread.h> 7 #include <linux/preempt.h> 8 9 static inline bool io_timer_cmp(const void *l, const void *r, void __always_unused *args) 10 { 11 struct io_timer **_l = (struct io_timer **)l; 12 struct io_timer **_r = (struct io_timer **)r; 13 14 return (*_l)->expire < (*_r)->expire; 15 } 16 17 static inline void io_timer_swp(void *l, void *r, void __always_unused *args) 18 { 19 struct io_timer **_l = (struct io_timer **)l; 20 struct io_timer **_r = (struct io_timer **)r; 21 22 swap(*_l, *_r); 23 } 24 25 static const struct min_heap_callbacks callbacks = { 26 .less = io_timer_cmp, 27 .swp = io_timer_swp, 28 }; 29 30 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer) 31 { 32 spin_lock(&clock->timer_lock); 33 34 if (time_after_eq64((u64) atomic64_read(&clock->now), timer->expire)) { 35 spin_unlock(&clock->timer_lock); 36 timer->fn(timer); 37 return; 38 } 39 40 for (size_t i = 0; i < clock->timers.nr; i++) 41 if (clock->timers.data[i] == timer) 42 goto out; 43 44 BUG_ON(!min_heap_push(&clock->timers, &timer, &callbacks, NULL)); 45 out: 46 spin_unlock(&clock->timer_lock); 47 } 48 49 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer) 50 { 51 spin_lock(&clock->timer_lock); 52 53 for (size_t i = 0; i < clock->timers.nr; i++) 54 if (clock->timers.data[i] == timer) { 55 min_heap_del(&clock->timers, i, &callbacks, NULL); 56 break; 57 } 58 59 spin_unlock(&clock->timer_lock); 60 } 61 62 struct io_clock_wait { 63 struct io_timer io_timer; 64 struct timer_list cpu_timer; 65 struct task_struct *task; 66 int expired; 67 }; 68 69 static void io_clock_wait_fn(struct io_timer *timer) 70 { 71 struct io_clock_wait *wait = container_of(timer, 72 struct io_clock_wait, io_timer); 73 74 wait->expired = 1; 75 wake_up_process(wait->task); 76 } 77 78 static void io_clock_cpu_timeout(struct timer_list *timer) 79 { 80 struct io_clock_wait *wait = container_of(timer, 81 struct io_clock_wait, cpu_timer); 82 83 wait->expired = 1; 84 wake_up_process(wait->task); 85 } 86 87 void bch2_io_clock_schedule_timeout(struct io_clock *clock, u64 until) 88 { 89 struct io_clock_wait wait = { 90 .io_timer.expire = until, 91 .io_timer.fn = io_clock_wait_fn, 92 .io_timer.fn2 = (void *) _RET_IP_, 93 .task = current, 94 }; 95 96 bch2_io_timer_add(clock, &wait.io_timer); 97 schedule(); 98 bch2_io_timer_del(clock, &wait.io_timer); 99 } 100 101 void bch2_kthread_io_clock_wait(struct io_clock *clock, 102 u64 io_until, unsigned long cpu_timeout) 103 { 104 bool kthread = (current->flags & PF_KTHREAD) != 0; 105 struct io_clock_wait wait = { 106 .io_timer.expire = io_until, 107 .io_timer.fn = io_clock_wait_fn, 108 .io_timer.fn2 = (void *) _RET_IP_, 109 .task = current, 110 }; 111 112 bch2_io_timer_add(clock, &wait.io_timer); 113 114 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0); 115 116 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT) 117 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies); 118 119 do { 120 set_current_state(TASK_INTERRUPTIBLE); 121 if (kthread && kthread_should_stop()) 122 break; 123 124 if (wait.expired) 125 break; 126 127 schedule(); 128 try_to_freeze(); 129 } while (0); 130 131 __set_current_state(TASK_RUNNING); 132 del_timer_sync(&wait.cpu_timer); 133 destroy_timer_on_stack(&wait.cpu_timer); 134 bch2_io_timer_del(clock, &wait.io_timer); 135 } 136 137 static struct io_timer *get_expired_timer(struct io_clock *clock, u64 now) 138 { 139 struct io_timer *ret = NULL; 140 141 if (clock->timers.nr && 142 time_after_eq64(now, clock->timers.data[0]->expire)) { 143 ret = *min_heap_peek(&clock->timers); 144 min_heap_pop(&clock->timers, &callbacks, NULL); 145 } 146 147 return ret; 148 } 149 150 void __bch2_increment_clock(struct io_clock *clock, u64 sectors) 151 { 152 struct io_timer *timer; 153 u64 now = atomic64_add_return(sectors, &clock->now); 154 155 spin_lock(&clock->timer_lock); 156 while ((timer = get_expired_timer(clock, now))) 157 timer->fn(timer); 158 spin_unlock(&clock->timer_lock); 159 } 160 161 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock) 162 { 163 out->atomic++; 164 spin_lock(&clock->timer_lock); 165 u64 now = atomic64_read(&clock->now); 166 167 printbuf_tabstop_push(out, 40); 168 prt_printf(out, "current time:\t%llu\n", now); 169 170 for (unsigned i = 0; i < clock->timers.nr; i++) 171 prt_printf(out, "%ps %ps:\t%llu\n", 172 clock->timers.data[i]->fn, 173 clock->timers.data[i]->fn2, 174 clock->timers.data[i]->expire); 175 spin_unlock(&clock->timer_lock); 176 --out->atomic; 177 } 178 179 void bch2_io_clock_exit(struct io_clock *clock) 180 { 181 free_heap(&clock->timers); 182 free_percpu(clock->pcpu_buf); 183 } 184 185 int bch2_io_clock_init(struct io_clock *clock) 186 { 187 atomic64_set(&clock->now, 0); 188 spin_lock_init(&clock->timer_lock); 189 190 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus(); 191 192 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf); 193 if (!clock->pcpu_buf) 194 return -BCH_ERR_ENOMEM_io_clock_init; 195 196 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL)) 197 return -BCH_ERR_ENOMEM_io_clock_init; 198 199 return 0; 200 } 201