1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2011 The FreeBSD Foundation
5 * Copyright (c) 2013 Ruslan Bukin <[email protected]>
6 * All rights reserved.
7 *
8 * Based on mpcore_timer.c developed by Ben Gray <[email protected]>
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. The name of the company nor the name of the author may be used to
19 * endorse or promote products derived from this software without specific
20 * prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 /**
36 * Cortex-A7, Cortex-A15, ARMv8 and later Generic Timer
37 */
38
39 #include "opt_acpi.h"
40 #include "opt_platform.h"
41
42 #include <sys/cdefs.h>
43 __FBSDID("$FreeBSD$");
44
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/bus.h>
48 #include <sys/kernel.h>
49 #include <sys/module.h>
50 #include <sys/malloc.h>
51 #include <sys/rman.h>
52 #include <sys/timeet.h>
53 #include <sys/timetc.h>
54 #include <sys/smp.h>
55 #include <sys/vdso.h>
56 #include <sys/watchdog.h>
57 #include <machine/bus.h>
58 #include <machine/cpu.h>
59 #include <machine/intr.h>
60 #include <machine/md_var.h>
61
62 #if defined(__arm__)
63 #include <machine/machdep.h> /* For arm_set_delay */
64 #endif
65
66 #ifdef FDT
67 #include <dev/ofw/openfirm.h>
68 #include <dev/ofw/ofw_bus.h>
69 #include <dev/ofw/ofw_bus_subr.h>
70 #endif
71
72 #ifdef DEV_ACPI
73 #include <contrib/dev/acpica/include/acpi.h>
74 #include <dev/acpica/acpivar.h>
75 #endif
76
77 #define GT_CTRL_ENABLE (1 << 0)
78 #define GT_CTRL_INT_MASK (1 << 1)
79 #define GT_CTRL_INT_STAT (1 << 2)
80 #define GT_REG_CTRL 0
81 #define GT_REG_TVAL 1
82
83 #define GT_CNTKCTL_PL0PTEN (1 << 9) /* PL0 Physical timer reg access */
84 #define GT_CNTKCTL_PL0VTEN (1 << 8) /* PL0 Virtual timer reg access */
85 #define GT_CNTKCTL_EVNTI (0xf << 4) /* Virtual counter event bits */
86 #define GT_CNTKCTL_EVNTDIR (1 << 3) /* Virtual counter event transition */
87 #define GT_CNTKCTL_EVNTEN (1 << 2) /* Enables virtual counter events */
88 #define GT_CNTKCTL_PL0VCTEN (1 << 1) /* PL0 CNTVCT and CNTFRQ access */
89 #define GT_CNTKCTL_PL0PCTEN (1 << 0) /* PL0 CNTPCT and CNTFRQ access */
90
91 struct arm_tmr_softc {
92 struct resource *res[4];
93 void *ihl[4];
94 uint64_t (*get_cntxct)(bool);
95 uint32_t clkfreq;
96 struct eventtimer et;
97 bool physical;
98 };
99
100 static struct arm_tmr_softc *arm_tmr_sc = NULL;
101
102 static struct resource_spec timer_spec[] = {
103 { SYS_RES_IRQ, 0, RF_ACTIVE }, /* Secure */
104 { SYS_RES_IRQ, 1, RF_ACTIVE }, /* Non-secure */
105 { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, /* Virt */
106 { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, /* Hyp */
107 { -1, 0 }
108 };
109
110 static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th,
111 struct timecounter *tc);
112 static void arm_tmr_do_delay(int usec, void *);
113
114 static timecounter_get_t arm_tmr_get_timecount;
115
116 static struct timecounter arm_tmr_timecount = {
117 .tc_name = "ARM MPCore Timecounter",
118 .tc_get_timecount = arm_tmr_get_timecount,
119 .tc_poll_pps = NULL,
120 .tc_counter_mask = ~0u,
121 .tc_frequency = 0,
122 .tc_quality = 1000,
123 .tc_fill_vdso_timehands = arm_tmr_fill_vdso_timehands,
124 };
125
126 #ifdef __arm__
127 #define get_el0(x) cp15_## x ##_get()
128 #define get_el1(x) cp15_## x ##_get()
129 #define set_el0(x, val) cp15_## x ##_set(val)
130 #define set_el1(x, val) cp15_## x ##_set(val)
131 #else /* __aarch64__ */
132 #define get_el0(x) READ_SPECIALREG(x ##_el0)
133 #define get_el1(x) READ_SPECIALREG(x ##_el1)
134 #define set_el0(x, val) WRITE_SPECIALREG(x ##_el0, val)
135 #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val)
136 #endif
137
138 static int
get_freq(void)139 get_freq(void)
140 {
141 return (get_el0(cntfrq));
142 }
143
144 static uint64_t
get_cntxct_a64_unstable(bool physical)145 get_cntxct_a64_unstable(bool physical)
146 {
147 uint64_t val
148 ;
149 isb();
150 if (physical) {
151 do {
152 val = get_el0(cntpct);
153 }
154 while (((val + 1) & 0x7FF) <= 1);
155 }
156 else {
157 do {
158 val = get_el0(cntvct);
159 }
160 while (((val + 1) & 0x7FF) <= 1);
161 }
162
163 return (val);
164 }
165
166 static uint64_t
get_cntxct(bool physical)167 get_cntxct(bool physical)
168 {
169 uint64_t val;
170
171 isb();
172 if (physical)
173 val = get_el0(cntpct);
174 else
175 val = get_el0(cntvct);
176
177 return (val);
178 }
179
180 static int
set_ctrl(uint32_t val,bool physical)181 set_ctrl(uint32_t val, bool physical)
182 {
183
184 if (physical)
185 set_el0(cntp_ctl, val);
186 else
187 set_el0(cntv_ctl, val);
188 isb();
189
190 return (0);
191 }
192
193 static int
set_tval(uint32_t val,bool physical)194 set_tval(uint32_t val, bool physical)
195 {
196
197 if (physical)
198 set_el0(cntp_tval, val);
199 else
200 set_el0(cntv_tval, val);
201 isb();
202
203 return (0);
204 }
205
206 static int
get_ctrl(bool physical)207 get_ctrl(bool physical)
208 {
209 uint32_t val;
210
211 if (physical)
212 val = get_el0(cntp_ctl);
213 else
214 val = get_el0(cntv_ctl);
215
216 return (val);
217 }
218
219 static void
setup_user_access(void * arg __unused)220 setup_user_access(void *arg __unused)
221 {
222 uint32_t cntkctl;
223
224 cntkctl = get_el1(cntkctl);
225 cntkctl &= ~(GT_CNTKCTL_PL0PTEN | GT_CNTKCTL_PL0VTEN |
226 GT_CNTKCTL_EVNTEN);
227 if (arm_tmr_sc->physical) {
228 cntkctl |= GT_CNTKCTL_PL0PCTEN;
229 cntkctl &= ~GT_CNTKCTL_PL0VCTEN;
230 } else {
231 cntkctl |= GT_CNTKCTL_PL0VCTEN;
232 cntkctl &= ~GT_CNTKCTL_PL0PCTEN;
233 }
234 set_el1(cntkctl, cntkctl);
235 isb();
236 }
237
238 static void
tmr_setup_user_access(void * arg __unused)239 tmr_setup_user_access(void *arg __unused)
240 {
241
242 if (arm_tmr_sc != NULL)
243 smp_rendezvous(NULL, setup_user_access, NULL, NULL);
244 }
245 SYSINIT(tmr_ua, SI_SUB_SMP, SI_ORDER_ANY, tmr_setup_user_access, NULL);
246
247 static unsigned
arm_tmr_get_timecount(struct timecounter * tc)248 arm_tmr_get_timecount(struct timecounter *tc)
249 {
250
251 return (arm_tmr_sc->get_cntxct(arm_tmr_sc->physical));
252 }
253
254 static int
arm_tmr_start(struct eventtimer * et,sbintime_t first,sbintime_t period __unused)255 arm_tmr_start(struct eventtimer *et, sbintime_t first,
256 sbintime_t period __unused)
257 {
258 struct arm_tmr_softc *sc;
259 int counts, ctrl;
260
261 sc = (struct arm_tmr_softc *)et->et_priv;
262
263 if (first != 0) {
264 counts = ((uint32_t)et->et_frequency * first) >> 32;
265 ctrl = get_ctrl(sc->physical);
266 ctrl &= ~GT_CTRL_INT_MASK;
267 ctrl |= GT_CTRL_ENABLE;
268 set_tval(counts, sc->physical);
269 set_ctrl(ctrl, sc->physical);
270 return (0);
271 }
272
273 return (EINVAL);
274
275 }
276
277 static void
arm_tmr_disable(bool physical)278 arm_tmr_disable(bool physical)
279 {
280 int ctrl;
281
282 ctrl = get_ctrl(physical);
283 ctrl &= ~GT_CTRL_ENABLE;
284 set_ctrl(ctrl, physical);
285 }
286
287 static int
arm_tmr_stop(struct eventtimer * et)288 arm_tmr_stop(struct eventtimer *et)
289 {
290 struct arm_tmr_softc *sc;
291
292 sc = (struct arm_tmr_softc *)et->et_priv;
293 arm_tmr_disable(sc->physical);
294
295 return (0);
296 }
297
298 static int
arm_tmr_intr(void * arg)299 arm_tmr_intr(void *arg)
300 {
301 struct arm_tmr_softc *sc;
302 int ctrl;
303
304 sc = (struct arm_tmr_softc *)arg;
305 ctrl = get_ctrl(sc->physical);
306 if (ctrl & GT_CTRL_INT_STAT) {
307 ctrl |= GT_CTRL_INT_MASK;
308 set_ctrl(ctrl, sc->physical);
309 }
310
311 if (sc->et.et_active)
312 sc->et.et_event_cb(&sc->et, sc->et.et_arg);
313
314 return (FILTER_HANDLED);
315 }
316
317 #ifdef FDT
318 static int
arm_tmr_fdt_probe(device_t dev)319 arm_tmr_fdt_probe(device_t dev)
320 {
321
322 if (!ofw_bus_status_okay(dev))
323 return (ENXIO);
324
325 if (ofw_bus_is_compatible(dev, "arm,armv8-timer")) {
326 device_set_desc(dev, "ARMv8 Generic Timer");
327 return (BUS_PROBE_DEFAULT);
328 } else if (ofw_bus_is_compatible(dev, "arm,armv7-timer")) {
329 device_set_desc(dev, "ARMv7 Generic Timer");
330 return (BUS_PROBE_DEFAULT);
331 }
332
333 return (ENXIO);
334 }
335 #endif
336
337 #ifdef DEV_ACPI
338 static void
arm_tmr_acpi_add_irq(device_t parent,device_t dev,int rid,u_int irq)339 arm_tmr_acpi_add_irq(device_t parent, device_t dev, int rid, u_int irq)
340 {
341
342 BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, rid, irq, 1);
343 }
344
345 static void
arm_tmr_acpi_identify(driver_t * driver,device_t parent)346 arm_tmr_acpi_identify(driver_t *driver, device_t parent)
347 {
348 ACPI_TABLE_GTDT *gtdt;
349 vm_paddr_t physaddr;
350 device_t dev;
351
352 physaddr = acpi_find_table(ACPI_SIG_GTDT);
353 if (physaddr == 0)
354 return;
355
356 gtdt = acpi_map_table(physaddr, ACPI_SIG_GTDT);
357 if (gtdt == NULL) {
358 device_printf(parent, "gic: Unable to map the GTDT\n");
359 return;
360 }
361
362 dev = BUS_ADD_CHILD(parent, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE,
363 "generic_timer", -1);
364 if (dev == NULL) {
365 device_printf(parent, "add gic child failed\n");
366 goto out;
367 }
368
369 arm_tmr_acpi_add_irq(parent, dev, 0, gtdt->SecureEl1Interrupt);
370 arm_tmr_acpi_add_irq(parent, dev, 1, gtdt->NonSecureEl1Interrupt);
371 arm_tmr_acpi_add_irq(parent, dev, 2, gtdt->VirtualTimerInterrupt);
372
373 out:
374 acpi_unmap_table(gtdt);
375 }
376
377 static int
arm_tmr_acpi_probe(device_t dev)378 arm_tmr_acpi_probe(device_t dev)
379 {
380
381 device_set_desc(dev, "ARM Generic Timer");
382 return (BUS_PROBE_NOWILDCARD);
383 }
384 #endif
385
386 static int
arm_tmr_attach(device_t dev)387 arm_tmr_attach(device_t dev)
388 {
389 struct arm_tmr_softc *sc;
390 #ifdef FDT
391 phandle_t node;
392 pcell_t clock;
393 #endif
394 int error;
395 int i, first_timer, last_timer;
396
397 sc = device_get_softc(dev);
398 if (arm_tmr_sc)
399 return (ENXIO);
400
401 sc->get_cntxct = &get_cntxct;
402 #ifdef FDT
403 /* Get the base clock frequency */
404 node = ofw_bus_get_node(dev);
405 if (node > 0) {
406 error = OF_getencprop(node, "clock-frequency", &clock,
407 sizeof(clock));
408 if (error > 0)
409 sc->clkfreq = clock;
410
411 if (OF_hasprop(node, "allwinner,sun50i-a64-unstable-timer")) {
412 sc->get_cntxct = &get_cntxct_a64_unstable;
413 if (bootverbose)
414 device_printf(dev,
415 "Enabling allwinner unstable timer workaround\n");
416 }
417 }
418 #endif
419
420 if (sc->clkfreq == 0) {
421 /* Try to get clock frequency from timer */
422 sc->clkfreq = get_freq();
423 }
424
425 if (sc->clkfreq == 0) {
426 device_printf(dev, "No clock frequency specified\n");
427 return (ENXIO);
428 }
429
430 if (bus_alloc_resources(dev, timer_spec, sc->res)) {
431 device_printf(dev, "could not allocate resources\n");
432 return (ENXIO);
433 }
434
435 #ifdef __aarch64__
436 /* Use the virtual timer if we have one. */
437 if (sc->res[2] != NULL) {
438 sc->physical = false;
439 first_timer = 2;
440 last_timer = 2;
441 } else
442 #endif
443 /* Otherwise set up the secure and non-secure physical timers. */
444 {
445 sc->physical = true;
446 first_timer = 0;
447 last_timer = 1;
448 }
449
450 arm_tmr_sc = sc;
451
452 /* Setup secure, non-secure and virtual IRQs handler */
453 for (i = first_timer; i <= last_timer; i++) {
454 /* If we do not have the interrupt, skip it. */
455 if (sc->res[i] == NULL)
456 continue;
457 error = bus_setup_intr(dev, sc->res[i], INTR_TYPE_CLK,
458 arm_tmr_intr, NULL, sc, &sc->ihl[i]);
459 if (error) {
460 device_printf(dev, "Unable to alloc int resource.\n");
461 return (ENXIO);
462 }
463 }
464
465 /* Disable the virtual timer until we are ready */
466 if (sc->res[2] != NULL)
467 arm_tmr_disable(false);
468 /* And the physical */
469 if (sc->physical)
470 arm_tmr_disable(true);
471
472 arm_tmr_timecount.tc_frequency = sc->clkfreq;
473 tc_init(&arm_tmr_timecount);
474
475 sc->et.et_name = "ARM MPCore Eventtimer";
476 sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU;
477 sc->et.et_quality = 1000;
478
479 sc->et.et_frequency = sc->clkfreq;
480 sc->et.et_min_period = (0x00000010LLU << 32) / sc->et.et_frequency;
481 sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency;
482 sc->et.et_start = arm_tmr_start;
483 sc->et.et_stop = arm_tmr_stop;
484 sc->et.et_priv = sc;
485 et_register(&sc->et);
486
487 #if defined(__arm__)
488 arm_set_delay(arm_tmr_do_delay, sc);
489 #endif
490
491 return (0);
492 }
493
494 #ifdef FDT
495 static device_method_t arm_tmr_fdt_methods[] = {
496 DEVMETHOD(device_probe, arm_tmr_fdt_probe),
497 DEVMETHOD(device_attach, arm_tmr_attach),
498 { 0, 0 }
499 };
500
501 static driver_t arm_tmr_fdt_driver = {
502 "generic_timer",
503 arm_tmr_fdt_methods,
504 sizeof(struct arm_tmr_softc),
505 };
506
507 static devclass_t arm_tmr_fdt_devclass;
508
509 EARLY_DRIVER_MODULE(timer, simplebus, arm_tmr_fdt_driver, arm_tmr_fdt_devclass,
510 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE);
511 EARLY_DRIVER_MODULE(timer, ofwbus, arm_tmr_fdt_driver, arm_tmr_fdt_devclass,
512 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE);
513 #endif
514
515 #ifdef DEV_ACPI
516 static device_method_t arm_tmr_acpi_methods[] = {
517 DEVMETHOD(device_identify, arm_tmr_acpi_identify),
518 DEVMETHOD(device_probe, arm_tmr_acpi_probe),
519 DEVMETHOD(device_attach, arm_tmr_attach),
520 { 0, 0 }
521 };
522
523 static driver_t arm_tmr_acpi_driver = {
524 "generic_timer",
525 arm_tmr_acpi_methods,
526 sizeof(struct arm_tmr_softc),
527 };
528
529 static devclass_t arm_tmr_acpi_devclass;
530
531 EARLY_DRIVER_MODULE(timer, acpi, arm_tmr_acpi_driver, arm_tmr_acpi_devclass,
532 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE);
533 #endif
534
535 static void
arm_tmr_do_delay(int usec,void * arg)536 arm_tmr_do_delay(int usec, void *arg)
537 {
538 struct arm_tmr_softc *sc = arg;
539 int32_t counts, counts_per_usec;
540 uint32_t first, last;
541
542 /* Get the number of times to count */
543 counts_per_usec = ((arm_tmr_timecount.tc_frequency / 1000000) + 1);
544
545 /*
546 * Clamp the timeout at a maximum value (about 32 seconds with
547 * a 66MHz clock). *Nobody* should be delay()ing for anywhere
548 * near that length of time and if they are, they should be hung
549 * out to dry.
550 */
551 if (usec >= (0x80000000U / counts_per_usec))
552 counts = (0x80000000U / counts_per_usec) - 1;
553 else
554 counts = usec * counts_per_usec;
555
556 first = sc->get_cntxct(sc->physical);
557
558 while (counts > 0) {
559 last = sc->get_cntxct(sc->physical);
560 counts -= (int32_t)(last - first);
561 first = last;
562 }
563 }
564
565 #if defined(__aarch64__)
566 void
DELAY(int usec)567 DELAY(int usec)
568 {
569 int32_t counts;
570
571 TSENTER();
572 /*
573 * Check the timers are setup, if not just
574 * use a for loop for the meantime
575 */
576 if (arm_tmr_sc == NULL) {
577 for (; usec > 0; usec--)
578 for (counts = 200; counts > 0; counts--)
579 /*
580 * Prevent the compiler from optimizing
581 * out the loop
582 */
583 cpufunc_nullop();
584 } else
585 arm_tmr_do_delay(usec, arm_tmr_sc);
586 TSEXIT();
587 }
588 #endif
589
590 static uint32_t
arm_tmr_fill_vdso_timehands(struct vdso_timehands * vdso_th,struct timecounter * tc)591 arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th,
592 struct timecounter *tc)
593 {
594
595 vdso_th->th_algo = VDSO_TH_ALGO_ARM_GENTIM;
596 vdso_th->th_physical = arm_tmr_sc->physical;
597 bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
598 return (1);
599 }
600