1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2001 Jake Burkholder.
5 * Copyright (c) 2005, 2008 Marius Strobl <[email protected]>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/pcpu.h>
37 #include <sys/proc.h>
38 #include <sys/sched.h>
39 #include <sys/smp.h>
40 #include <sys/sysctl.h>
41 #include <sys/timeet.h>
42 #include <sys/timetc.h>
43
44 #include <dev/ofw/openfirm.h>
45
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48
49 #include <machine/frame.h>
50 #include <machine/intr_machdep.h>
51 #include <machine/smp.h>
52 #include <machine/tick.h>
53 #include <machine/ver.h>
54
55 #define TICK_QUALITY_MP 10
56 #define TICK_QUALITY_UP 1000
57
58 static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
59
60 static int adjust_edges = 0;
61 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges,
62 0, "total number of times tick interrupts got more than 12.5% behind");
63
64 static int adjust_excess = 0;
65 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess,
66 0, "total number of ignored tick interrupts");
67
68 static int adjust_missed = 0;
69 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed,
70 0, "total number of missed tick interrupts");
71
72 static int adjust_ticks = 0;
73 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks,
74 0, "total number of tick interrupts with adjustment");
75
76 u_int tick_et_use_stick = 0;
77 SYSCTL_INT(_machdep_tick, OID_AUTO, tick_et_use_stick, CTLFLAG_RD,
78 &tick_et_use_stick, 0, "tick event timer uses STICK instead of TICK");
79
80 typedef uint64_t rd_tick_t(void);
81 static rd_tick_t *rd_tick;
82 typedef void wr_tick_cmpr_t(uint64_t);
83 static wr_tick_cmpr_t *wr_tick_cmpr;
84
85 static struct timecounter stick_tc;
86 static struct eventtimer tick_et;
87 static struct timecounter tick_tc;
88
89 #ifdef SMP
90 static timecounter_get_t stick_get_timecount_mp;
91 #endif
92 static timecounter_get_t stick_get_timecount_up;
93 static rd_tick_t stick_rd;
94 static wr_tick_cmpr_t stick_wr_cmpr;
95 static int tick_et_start(struct eventtimer *et, sbintime_t first,
96 sbintime_t period);
97 static int tick_et_stop(struct eventtimer *et);
98 #ifdef SMP
99 static timecounter_get_t tick_get_timecount_mp;
100 #endif
101 static timecounter_get_t tick_get_timecount_up;
102 static void tick_intr(struct trapframe *tf);
103 static inline void tick_process(struct trapframe *tf);
104 static rd_tick_t tick_rd;
105 static wr_tick_cmpr_t tick_wr_cmpr;
106 static wr_tick_cmpr_t tick_wr_cmpr_bbwar;
107 static uint64_t tick_cputicks(void);
108
109 static uint64_t
stick_rd(void)110 stick_rd(void)
111 {
112
113 return (rdstick());
114 }
115
116 static void
stick_wr_cmpr(uint64_t tick)117 stick_wr_cmpr(uint64_t tick)
118 {
119
120 wrstickcmpr(tick, 0);
121 }
122
123 static uint64_t
tick_rd(void)124 tick_rd(void)
125 {
126
127 return (rd(tick));
128 }
129
130 static void
tick_wr_cmpr(uint64_t tick_cmpr)131 tick_wr_cmpr(uint64_t tick_cmpr)
132 {
133
134 wrtickcmpr(tick_cmpr, 0);
135 }
136
137 static void
tick_wr_cmpr_bbwar(uint64_t tick_cmpr)138 tick_wr_cmpr_bbwar(uint64_t tick_cmpr)
139 {
140
141 wrtickcmpr_bbwar(tick_cmpr, 0);
142 }
143
144 static uint64_t
tick_cputicks(void)145 tick_cputicks(void)
146 {
147
148 return (rd(tick));
149 }
150
151 void
cpu_initclocks(void)152 cpu_initclocks(void)
153 {
154 uint32_t clock, sclock;
155
156 clock = PCPU_GET(clock);
157 sclock = 0;
158 if (PCPU_GET(impl) == CPU_IMPL_SPARC64V ||
159 PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCIII) {
160 if (OF_getprop(OF_peer(0), "stick-frequency", &sclock,
161 sizeof(sclock)) == -1) {
162 panic("%s: could not determine STICK frequency",
163 __func__);
164 }
165 }
166 /*
167 * Given that the STICK timers typically are driven at rather low
168 * frequencies they shouldn't be used except when really necessary.
169 */
170 if (tick_et_use_stick != 0) {
171 rd_tick = stick_rd;
172 wr_tick_cmpr = stick_wr_cmpr;
173 /*
174 * We don't provide a CPU ticker as long as the frequency
175 * supplied isn't actually used per-CPU.
176 */
177 } else {
178 rd_tick = tick_rd;
179 if (PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCI &&
180 PCPU_GET(impl) < CPU_IMPL_ULTRASPARCIII)
181 wr_tick_cmpr = tick_wr_cmpr_bbwar;
182 else
183 wr_tick_cmpr = tick_wr_cmpr;
184 set_cputicker(tick_cputicks, clock, 0);
185 }
186 intr_setup(PIL_TICK, tick_intr, -1, NULL, NULL);
187
188 /*
189 * Initialize the (S)TICK-based timecounter(s).
190 * Note that we (try to) sync the (S)TICK timers of APs with the BSP
191 * during their startup but not afterwards. The resulting drift can
192 * cause problems when the time is calculated based on (S)TICK values
193 * read on different CPUs. Thus we always read the register on the
194 * BSP (if necessary via an IPI as sched_bind(9) isn't available in
195 * all circumstances) and use a low quality for the otherwise high
196 * quality (S)TICK timers in the MP case.
197 */
198 tick_tc.tc_get_timecount = tick_get_timecount_up;
199 tick_tc.tc_counter_mask = ~0u;
200 tick_tc.tc_frequency = clock;
201 tick_tc.tc_name = "tick";
202 tick_tc.tc_quality = TICK_QUALITY_UP;
203 #ifdef SMP
204 if (cpu_mp_probe()) {
205 tick_tc.tc_get_timecount = tick_get_timecount_mp;
206 tick_tc.tc_quality = TICK_QUALITY_MP;
207 }
208 #endif
209 tc_init(&tick_tc);
210 if (sclock != 0) {
211 stick_tc.tc_get_timecount = stick_get_timecount_up;
212 stick_tc.tc_counter_mask = ~0u;
213 stick_tc.tc_frequency = sclock;
214 stick_tc.tc_name = "stick";
215 stick_tc.tc_quality = TICK_QUALITY_UP;
216 #ifdef SMP
217 if (cpu_mp_probe()) {
218 stick_tc.tc_get_timecount = stick_get_timecount_mp;
219 stick_tc.tc_quality = TICK_QUALITY_MP;
220 }
221 #endif
222 tc_init(&stick_tc);
223 }
224 tick_et.et_name = tick_et_use_stick ? "stick" : "tick";
225 tick_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
226 ET_FLAGS_PERCPU;
227 tick_et.et_quality = 1000;
228 tick_et.et_frequency = tick_et_use_stick ? sclock : clock;
229 tick_et.et_min_period = 0x00010000LLU; /* To be safe. */
230 tick_et.et_max_period = (0xfffffffeLLU << 32) / tick_et.et_frequency;
231 tick_et.et_start = tick_et_start;
232 tick_et.et_stop = tick_et_stop;
233 tick_et.et_priv = NULL;
234 et_register(&tick_et);
235
236 cpu_initclocks_bsp();
237 }
238
239 static inline void
tick_process(struct trapframe * tf)240 tick_process(struct trapframe *tf)
241 {
242 struct trapframe *oldframe;
243 struct thread *td;
244
245 td = curthread;
246 td->td_intr_nesting_level++;
247 critical_enter();
248 if (tick_et.et_active) {
249 oldframe = td->td_intr_frame;
250 td->td_intr_frame = tf;
251 tick_et.et_event_cb(&tick_et, tick_et.et_arg);
252 td->td_intr_frame = oldframe;
253 }
254 td->td_intr_nesting_level--;
255 critical_exit();
256 }
257
258 static void
tick_intr(struct trapframe * tf)259 tick_intr(struct trapframe *tf)
260 {
261 u_long adj, ref, tick, tick_increment;
262 long delta;
263 register_t s;
264 int count;
265
266 tick_increment = PCPU_GET(tickincrement);
267 if (tick_increment != 0) {
268 /*
269 * NB: the sequence of reading the (S)TICK register,
270 * calculating the value of the next tick and writing it to
271 * the (S)TICK_COMPARE register must not be interrupted, not
272 * even by an IPI, otherwise a value that is in the past could
273 * be written in the worst case and thus causing the periodic
274 * timer to stop.
275 */
276 s = intr_disable();
277 adj = PCPU_GET(tickadj);
278 tick = rd_tick();
279 wr_tick_cmpr(tick + tick_increment - adj);
280 intr_restore(s);
281 ref = PCPU_GET(tickref);
282 delta = tick - ref;
283 count = 0;
284 while (delta >= tick_increment) {
285 tick_process(tf);
286 delta -= tick_increment;
287 ref += tick_increment;
288 if (adj != 0)
289 adjust_ticks++;
290 count++;
291 }
292 if (count > 0) {
293 adjust_missed += count - 1;
294 if (delta > (tick_increment >> 3)) {
295 if (adj == 0)
296 adjust_edges++;
297 adj = tick_increment >> 4;
298 } else
299 adj = 0;
300 } else {
301 adj = 0;
302 adjust_excess++;
303 }
304 PCPU_SET(tickref, ref);
305 PCPU_SET(tickadj, adj);
306 } else
307 tick_process(tf);
308 }
309
310 static u_int
stick_get_timecount_up(struct timecounter * tc)311 stick_get_timecount_up(struct timecounter *tc)
312 {
313
314 return ((u_int)rdstick());
315 }
316
317 static u_int
tick_get_timecount_up(struct timecounter * tc)318 tick_get_timecount_up(struct timecounter *tc)
319 {
320
321 return ((u_int)rd(tick));
322 }
323
324 #ifdef SMP
325 static u_int
stick_get_timecount_mp(struct timecounter * tc)326 stick_get_timecount_mp(struct timecounter *tc)
327 {
328 static u_long stick;
329
330 sched_pin();
331 if (curcpu == 0)
332 stick = rdstick();
333 else
334 ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
335 sched_unpin();
336 return (stick);
337 }
338
339 static u_int
tick_get_timecount_mp(struct timecounter * tc)340 tick_get_timecount_mp(struct timecounter *tc)
341 {
342 static u_long tick;
343
344 sched_pin();
345 if (curcpu == 0)
346 tick = rd(tick);
347 else
348 ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
349 sched_unpin();
350 return (tick);
351 }
352 #endif
353
354 static int
tick_et_start(struct eventtimer * et,sbintime_t first,sbintime_t period)355 tick_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
356 {
357 u_long base, div, fdiv;
358 register_t s;
359
360 if (period != 0)
361 div = (tick_et.et_frequency * period) >> 32;
362 else
363 div = 0;
364 if (first != 0)
365 fdiv = (tick_et.et_frequency * first) >> 32;
366 else
367 fdiv = div;
368 PCPU_SET(tickincrement, div);
369
370 /*
371 * Try to make the (S)TICK interrupts as synchronously as possible
372 * on all CPUs to avoid inaccuracies for migrating processes. Leave
373 * out one tick to make sure that it is not missed.
374 */
375 s = intr_disable();
376 base = rd_tick();
377 if (div != 0) {
378 PCPU_SET(tickadj, 0);
379 base = roundup(base, div);
380 }
381 PCPU_SET(tickref, base);
382 wr_tick_cmpr(base + fdiv);
383 intr_restore(s);
384 return (0);
385 }
386
387 static int
tick_et_stop(struct eventtimer * et)388 tick_et_stop(struct eventtimer *et)
389 {
390
391 PCPU_SET(tickincrement, 0);
392 tick_stop(PCPU_GET(impl));
393 return (0);
394 }
395
396 void
tick_clear(u_int cpu_impl)397 tick_clear(u_int cpu_impl)
398 {
399
400 if (cpu_impl == CPU_IMPL_SPARC64V ||
401 cpu_impl >= CPU_IMPL_ULTRASPARCIII)
402 wrstick(0, 0);
403 wrpr(tick, 0, 0);
404 }
405
406 void
tick_stop(u_int cpu_impl)407 tick_stop(u_int cpu_impl)
408 {
409
410 if (cpu_impl == CPU_IMPL_SPARC64V ||
411 cpu_impl >= CPU_IMPL_ULTRASPARCIII)
412 wrstickcmpr(1L << 63, 0);
413 wrtickcmpr(1L << 63, 0);
414 }
415