1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008 Marcel Moolenaar
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/ktr.h>
34 #include <sys/bus.h>
35 #include <sys/cpuset.h>
36 #include <sys/domainset.h>
37 #include <sys/lock.h>
38 #include <sys/malloc.h>
39 #include <sys/mutex.h>
40 #include <sys/pcpu.h>
41 #include <sys/proc.h>
42 #include <sys/sched.h>
43 #include <sys/smp.h>
44
45 #include <vm/vm.h>
46 #include <vm/vm_param.h>
47 #include <vm/pmap.h>
48 #include <vm/vm_map.h>
49 #include <vm/vm_extern.h>
50 #include <vm/vm_kern.h>
51
52 #include <machine/bus.h>
53 #include <machine/cpu.h>
54 #include <machine/intr_machdep.h>
55 #include <machine/pcb.h>
56 #include <machine/platform.h>
57 #include <machine/md_var.h>
58 #include <machine/setjmp.h>
59 #include <machine/smp.h>
60
61 #include "pic_if.h"
62
63 volatile static int ap_awake;
64 volatile static u_int ap_letgo;
65 volatile static u_quad_t ap_timebase;
66 static struct mtx ap_boot_mtx;
67
68 void
machdep_ap_bootstrap(void)69 machdep_ap_bootstrap(void)
70 {
71
72 PCPU_SET(awake, 1);
73 __asm __volatile("msync; isync");
74
75 while (ap_letgo == 0)
76 nop_prio_vlow();
77 nop_prio_medium();
78
79 /*
80 * Set timebase as soon as possible to meet an implicit rendezvous
81 * from cpu_mp_unleash(), which sets ap_letgo and then immediately
82 * sets timebase.
83 *
84 * Note that this is instrinsically racy and is only relevant on
85 * platforms that do not support better mechanisms.
86 */
87 platform_smp_timebase_sync(ap_timebase, 1);
88
89 /* Give platform code a chance to do anything else necessary */
90 platform_smp_ap_init();
91
92 /* Initialize decrementer */
93 decr_ap_init();
94
95 /* Serialize console output and AP count increment */
96 mtx_lock_spin(&ap_boot_mtx);
97 ap_awake++;
98 if (bootverbose)
99 printf("SMP: AP CPU #%d launched\n", PCPU_GET(cpuid));
100 else
101 printf("%s%d%s", ap_awake == 2 ? "Launching APs: " : "",
102 PCPU_GET(cpuid), ap_awake == mp_ncpus ? "\n" : " ");
103 mtx_unlock_spin(&ap_boot_mtx);
104
105 while(smp_started == 0)
106 ;
107
108 /* Start per-CPU event timers. */
109 cpu_initclocks_ap();
110
111 /* Announce ourselves awake, and enter the scheduler */
112 sched_ap_entry();
113 }
114
115 void
cpu_mp_setmaxid(void)116 cpu_mp_setmaxid(void)
117 {
118 struct cpuref cpuref;
119 int error;
120
121 mp_ncpus = 0;
122 mp_maxid = 0;
123 error = platform_smp_first_cpu(&cpuref);
124 while (!error) {
125 mp_ncpus++;
126 mp_maxid = max(cpuref.cr_cpuid, mp_maxid);
127 error = platform_smp_next_cpu(&cpuref);
128 }
129 /* Sanity. */
130 if (mp_ncpus == 0)
131 mp_ncpus = 1;
132 }
133
134 int
cpu_mp_probe(void)135 cpu_mp_probe(void)
136 {
137
138 /*
139 * We're not going to enable SMP if there's only 1 processor.
140 */
141 return (mp_ncpus > 1);
142 }
143
144 void
cpu_mp_start(void)145 cpu_mp_start(void)
146 {
147 struct cpuref bsp, cpu;
148 struct pcpu *pc;
149 int domain, error;
150
151 error = platform_smp_get_bsp(&bsp);
152 KASSERT(error == 0, ("Don't know BSP"));
153
154 error = platform_smp_first_cpu(&cpu);
155 while (!error) {
156 if (cpu.cr_cpuid >= MAXCPU) {
157 printf("SMP: cpu%d: skipped -- ID out of range\n",
158 cpu.cr_cpuid);
159 goto next;
160 }
161 if (CPU_ISSET(cpu.cr_cpuid, &all_cpus)) {
162 printf("SMP: cpu%d: skipped - duplicate ID\n",
163 cpu.cr_cpuid);
164 goto next;
165 }
166
167 if (vm_ndomains > 1)
168 domain = cpu.cr_domain;
169 else
170 domain = 0;
171
172 if (cpu.cr_cpuid != bsp.cr_cpuid) {
173 void *dpcpu;
174
175 pc = &__pcpu[cpu.cr_cpuid];
176 dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain),
177 DPCPU_SIZE, M_WAITOK | M_ZERO);
178 pcpu_init(pc, cpu.cr_cpuid, sizeof(*pc));
179 dpcpu_init(dpcpu, cpu.cr_cpuid);
180 } else {
181 pc = pcpup;
182 pc->pc_cpuid = bsp.cr_cpuid;
183 pc->pc_bsp = 1;
184 }
185 pc->pc_domain = domain;
186 pc->pc_hwref = cpu.cr_hwref;
187
188 CPU_SET(pc->pc_cpuid, &cpuset_domain[pc->pc_domain]);
189 KASSERT(pc->pc_domain < MAXMEMDOM, ("bad domain value %d\n",
190 pc->pc_domain));
191 CPU_SET(pc->pc_cpuid, &all_cpus);
192 next:
193 error = platform_smp_next_cpu(&cpu);
194 }
195
196 #ifdef SMP
197 platform_smp_probe_threads();
198 #endif
199 }
200
201 void
cpu_mp_announce(void)202 cpu_mp_announce(void)
203 {
204 struct pcpu *pc;
205 int i;
206
207 if (!bootverbose)
208 return;
209
210 CPU_FOREACH(i) {
211 pc = pcpu_find(i);
212 if (pc == NULL)
213 continue;
214 printf("cpu%d: dev=%x domain=%d ", i, (int)pc->pc_hwref, pc->pc_domain);
215 if (pc->pc_bsp)
216 printf(" (BSP)");
217 printf("\n");
218 }
219 }
220
221 static void
cpu_mp_unleash(void * dummy)222 cpu_mp_unleash(void *dummy)
223 {
224 struct pcpu *pc;
225 int cpus, timeout;
226 int ret;
227
228 if (mp_ncpus <= 1)
229 return;
230
231 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
232
233 cpus = 0;
234 smp_cpus = 0;
235 #ifdef BOOKE
236 tlb1_ap_prep();
237 #endif
238 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
239 cpus++;
240 if (!pc->pc_bsp) {
241 if (bootverbose)
242 printf("Waking up CPU %d (dev=%x)\n",
243 pc->pc_cpuid, (int)pc->pc_hwref);
244
245 pc->pc_flags = PCPU_GET(flags); /* Copy cached CPU flags */
246 ret = platform_smp_start_cpu(pc);
247 if (ret == 0) {
248 timeout = 2000; /* wait 2sec for the AP */
249 while (!pc->pc_awake && --timeout > 0)
250 DELAY(1000);
251 }
252 } else {
253 pc->pc_awake = 1;
254 }
255 if (pc->pc_awake) {
256 if (bootverbose)
257 printf("Adding CPU %d, hwref=%jx, awake=%x\n",
258 pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
259 pc->pc_awake);
260 smp_cpus++;
261 } else
262 CPU_SET(pc->pc_cpuid, &stopped_cpus);
263 }
264
265 ap_awake = 1;
266
267 /* Provide our current DEC and TB values for APs */
268 ap_timebase = mftb() + 10;
269 __asm __volatile("msync; isync");
270
271 /* Let APs continue */
272 atomic_store_rel_int(&ap_letgo, 1);
273
274 platform_smp_timebase_sync(ap_timebase, 0);
275
276 while (ap_awake < smp_cpus)
277 ;
278
279 if (smp_cpus != cpus || cpus != mp_ncpus) {
280 printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n",
281 mp_ncpus, cpus, smp_cpus);
282 }
283
284 if (smp_cpus > 1)
285 atomic_store_rel_int(&smp_started, 1);
286
287 /* Let the APs get into the scheduler */
288 DELAY(10000);
289
290 }
291
292 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
293
294 int
powerpc_ipi_handler(void * arg)295 powerpc_ipi_handler(void *arg)
296 {
297 u_int cpuid;
298 uint32_t ipimask;
299
300 CTR2(KTR_SMP, "%s: MSR 0x%08x", __func__, mfmsr());
301
302 ipimask = atomic_readandclear_32(&(pcpup->pc_ipimask));
303 if (ipimask == 0)
304 return (FILTER_STRAY);
305 if (ipimask & (1 << IPI_AST)) {
306 CTR1(KTR_SMP, "%s: IPI_AST", __func__);
307 }
308 if (ipimask & (1 << IPI_PREEMPT)) {
309 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
310 sched_preempt(curthread);
311 }
312 if (ipimask & (1 << IPI_RENDEZVOUS)) {
313 CTR1(KTR_SMP, "%s: IPI_RENDEZVOUS", __func__);
314 smp_rendezvous_action();
315 }
316 if (ipimask & (1 << IPI_STOP)) {
317
318 /*
319 * IPI_STOP_HARD is mapped to IPI_STOP so it is not
320 * necessary to add such case.
321 */
322 CTR1(KTR_SMP, "%s: IPI_STOP or IPI_STOP_HARD (stop)",
323 __func__);
324 cpuid = PCPU_GET(cpuid);
325 savectx(&stoppcbs[cpuid]);
326 CPU_SET_ATOMIC(cpuid, &stopped_cpus);
327 while (!CPU_ISSET(cpuid, &started_cpus))
328 cpu_spinwait();
329 CPU_CLR_ATOMIC(cpuid, &stopped_cpus);
330 CPU_CLR_ATOMIC(cpuid, &started_cpus);
331 CTR1(KTR_SMP, "%s: IPI_STOP (restart)", __func__);
332 }
333 if (ipimask & (1 << IPI_HARDCLOCK)) {
334 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
335 hardclockintr();
336 }
337
338 return (FILTER_HANDLED);
339 }
340
341 static void
ipi_send(struct pcpu * pc,int ipi)342 ipi_send(struct pcpu *pc, int ipi)
343 {
344
345 CTR4(KTR_SMP, "%s: pc=%p, targetcpu=%d, IPI=%d", __func__,
346 pc, pc->pc_cpuid, ipi);
347
348 atomic_set_32(&pc->pc_ipimask, (1 << ipi));
349 powerpc_sync();
350 PIC_IPI(root_pic, pc->pc_cpuid);
351
352 CTR1(KTR_SMP, "%s: sent", __func__);
353 }
354
355 /* Send an IPI to a set of cpus. */
356 void
ipi_selected(cpuset_t cpus,int ipi)357 ipi_selected(cpuset_t cpus, int ipi)
358 {
359 struct pcpu *pc;
360
361 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
362 if (CPU_ISSET(pc->pc_cpuid, &cpus))
363 ipi_send(pc, ipi);
364 }
365 }
366
367 /* Send an IPI to a specific CPU. */
368 void
ipi_cpu(int cpu,u_int ipi)369 ipi_cpu(int cpu, u_int ipi)
370 {
371
372 ipi_send(cpuid_to_pcpu[cpu], ipi);
373 }
374
375 /* Send an IPI to all CPUs EXCEPT myself. */
376 void
ipi_all_but_self(int ipi)377 ipi_all_but_self(int ipi)
378 {
379 struct pcpu *pc;
380
381 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
382 if (pc != pcpup)
383 ipi_send(pc, ipi);
384 }
385 }
386