1 // SPDX-License-Identifier: GPL-2.0 2 #include "bcachefs.h" 3 #include "clock.h" 4 5 #include <linux/freezer.h> 6 #include <linux/kthread.h> 7 #include <linux/preempt.h> 8 9 static inline long io_timer_cmp(io_timer_heap *h, 10 struct io_timer *l, 11 struct io_timer *r) 12 { 13 return l->expire - r->expire; 14 } 15 16 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer) 17 { 18 spin_lock(&clock->timer_lock); 19 20 if (time_after_eq64((u64) atomic64_read(&clock->now), timer->expire)) { 21 spin_unlock(&clock->timer_lock); 22 timer->fn(timer); 23 return; 24 } 25 26 for (size_t i = 0; i < clock->timers.used; i++) 27 if (clock->timers.data[i] == timer) 28 goto out; 29 30 BUG_ON(!heap_add(&clock->timers, timer, io_timer_cmp, NULL)); 31 out: 32 spin_unlock(&clock->timer_lock); 33 } 34 35 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer) 36 { 37 spin_lock(&clock->timer_lock); 38 39 for (size_t i = 0; i < clock->timers.used; i++) 40 if (clock->timers.data[i] == timer) { 41 heap_del(&clock->timers, i, io_timer_cmp, NULL); 42 break; 43 } 44 45 spin_unlock(&clock->timer_lock); 46 } 47 48 struct io_clock_wait { 49 struct io_timer io_timer; 50 struct timer_list cpu_timer; 51 struct task_struct *task; 52 int expired; 53 }; 54 55 static void io_clock_wait_fn(struct io_timer *timer) 56 { 57 struct io_clock_wait *wait = container_of(timer, 58 struct io_clock_wait, io_timer); 59 60 wait->expired = 1; 61 wake_up_process(wait->task); 62 } 63 64 static void io_clock_cpu_timeout(struct timer_list *timer) 65 { 66 struct io_clock_wait *wait = container_of(timer, 67 struct io_clock_wait, cpu_timer); 68 69 wait->expired = 1; 70 wake_up_process(wait->task); 71 } 72 73 void bch2_io_clock_schedule_timeout(struct io_clock *clock, u64 until) 74 { 75 struct io_clock_wait wait = { 76 .io_timer.expire = until, 77 .io_timer.fn = io_clock_wait_fn, 78 .io_timer.fn2 = (void *) _RET_IP_, 79 .task = current, 80 }; 81 82 bch2_io_timer_add(clock, &wait.io_timer); 83 schedule(); 84 bch2_io_timer_del(clock, &wait.io_timer); 85 } 86 87 void bch2_kthread_io_clock_wait(struct io_clock *clock, 88 u64 io_until, unsigned long cpu_timeout) 89 { 90 bool kthread = (current->flags & PF_KTHREAD) != 0; 91 struct io_clock_wait wait = { 92 .io_timer.expire = io_until, 93 .io_timer.fn = io_clock_wait_fn, 94 .io_timer.fn2 = (void *) _RET_IP_, 95 .task = current, 96 }; 97 98 bch2_io_timer_add(clock, &wait.io_timer); 99 100 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0); 101 102 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT) 103 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies); 104 105 do { 106 set_current_state(TASK_INTERRUPTIBLE); 107 if (kthread && kthread_should_stop()) 108 break; 109 110 if (wait.expired) 111 break; 112 113 schedule(); 114 try_to_freeze(); 115 } while (0); 116 117 __set_current_state(TASK_RUNNING); 118 del_timer_sync(&wait.cpu_timer); 119 destroy_timer_on_stack(&wait.cpu_timer); 120 bch2_io_timer_del(clock, &wait.io_timer); 121 } 122 123 static struct io_timer *get_expired_timer(struct io_clock *clock, u64 now) 124 { 125 struct io_timer *ret = NULL; 126 127 if (clock->timers.used && 128 time_after_eq64(now, clock->timers.data[0]->expire)) 129 heap_pop(&clock->timers, ret, io_timer_cmp, NULL); 130 return ret; 131 } 132 133 void __bch2_increment_clock(struct io_clock *clock, u64 sectors) 134 { 135 struct io_timer *timer; 136 u64 now = atomic64_add_return(sectors, &clock->now); 137 138 spin_lock(&clock->timer_lock); 139 while ((timer = get_expired_timer(clock, now))) 140 timer->fn(timer); 141 spin_unlock(&clock->timer_lock); 142 } 143 144 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock) 145 { 146 out->atomic++; 147 spin_lock(&clock->timer_lock); 148 u64 now = atomic64_read(&clock->now); 149 150 printbuf_tabstop_push(out, 40); 151 prt_printf(out, "current time:\t%llu\n", now); 152 153 for (unsigned i = 0; i < clock->timers.used; i++) 154 prt_printf(out, "%ps %ps:\t%llu\n", 155 clock->timers.data[i]->fn, 156 clock->timers.data[i]->fn2, 157 clock->timers.data[i]->expire); 158 spin_unlock(&clock->timer_lock); 159 --out->atomic; 160 } 161 162 void bch2_io_clock_exit(struct io_clock *clock) 163 { 164 free_heap(&clock->timers); 165 free_percpu(clock->pcpu_buf); 166 } 167 168 int bch2_io_clock_init(struct io_clock *clock) 169 { 170 atomic64_set(&clock->now, 0); 171 spin_lock_init(&clock->timer_lock); 172 173 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus(); 174 175 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf); 176 if (!clock->pcpu_buf) 177 return -BCH_ERR_ENOMEM_io_clock_init; 178 179 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL)) 180 return -BCH_ERR_ENOMEM_io_clock_init; 181 182 return 0; 183 } 184