1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2001 Jake Burkholder.
5 * Copyright (c) 2007 - 2011 Marius Strobl <[email protected]>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31
32 #ifndef _MACHINE_SMP_H_
33 #define _MACHINE_SMP_H_
34
35 #ifdef SMP
36
37 #define CPU_TICKSYNC 1
38 #define CPU_STICKSYNC 2
39 #define CPU_INIT 3
40 #define CPU_BOOTSTRAP 4
41
42 #ifndef LOCORE
43
44 #include <sys/param.h>
45 #include <sys/cpuset.h>
46 #include <sys/lock.h>
47 #include <sys/mutex.h>
48 #include <sys/proc.h>
49 #include <sys/sched.h>
50 #include <sys/smp.h>
51
52 #include <machine/atomic.h>
53 #include <machine/intr_machdep.h>
54 #include <machine/tte.h>
55
56 #define IDR_BUSY 0x0000000000000001ULL
57 #define IDR_NACK 0x0000000000000002ULL
58 #define IDR_CHEETAH_ALL_BUSY 0x5555555555555555ULL
59 #define IDR_CHEETAH_ALL_NACK (~IDR_CHEETAH_ALL_BUSY)
60 #define IDR_CHEETAH_MAX_BN_PAIRS 32
61 #define IDR_JALAPENO_MAX_BN_PAIRS 4
62
63 #define IDC_ITID_SHIFT 14
64 #define IDC_BN_SHIFT 24
65
66 #define IPI_AST PIL_AST
67 #define IPI_RENDEZVOUS PIL_RENDEZVOUS
68 #define IPI_PREEMPT PIL_PREEMPT
69 #define IPI_HARDCLOCK PIL_HARDCLOCK
70 #define IPI_STOP PIL_STOP
71 #define IPI_STOP_HARD PIL_STOP
72
73 #define IPI_RETRIES 5000
74
75 struct cpu_start_args {
76 u_int csa_count;
77 u_int csa_mid;
78 u_int csa_state;
79 vm_offset_t csa_pcpu;
80 u_long csa_tick;
81 u_long csa_stick;
82 u_long csa_ver;
83 struct tte csa_ttes[PCPU_PAGES];
84 };
85
86 struct ipi_cache_args {
87 cpuset_t ica_mask;
88 vm_paddr_t ica_pa;
89 };
90
91 struct ipi_rd_args {
92 cpuset_t ira_mask;
93 register_t *ira_val;
94 };
95
96 struct ipi_tlb_args {
97 cpuset_t ita_mask;
98 struct pmap *ita_pmap;
99 u_long ita_start;
100 u_long ita_end;
101 };
102 #define ita_va ita_start
103
104 struct pcb;
105 struct pcpu;
106
107 extern struct pcb stoppcbs[];
108
109 void cpu_mp_bootstrap(struct pcpu *pc);
110 void cpu_mp_shutdown(void);
111
112 typedef void cpu_ipi_selected_t(cpuset_t, u_long, u_long, u_long);
113 extern cpu_ipi_selected_t *cpu_ipi_selected;
114 typedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long);
115 extern cpu_ipi_single_t *cpu_ipi_single;
116
117 void mp_init(void);
118
119 extern struct mtx ipi_mtx;
120 extern struct ipi_cache_args ipi_cache_args;
121 extern struct ipi_rd_args ipi_rd_args;
122 extern struct ipi_tlb_args ipi_tlb_args;
123
124 extern char *mp_tramp_code;
125 extern u_long mp_tramp_code_len;
126 extern u_long mp_tramp_tlb_slots;
127 extern u_long mp_tramp_func;
128
129 extern void mp_startup(void);
130
131 extern char tl_ipi_cheetah_dcache_page_inval[];
132 extern char tl_ipi_spitfire_dcache_page_inval[];
133 extern char tl_ipi_spitfire_icache_page_inval[];
134
135 extern char tl_ipi_level[];
136
137 extern char tl_ipi_stick_rd[];
138 extern char tl_ipi_tick_rd[];
139
140 extern char tl_ipi_tlb_context_demap[];
141 extern char tl_ipi_tlb_page_demap[];
142 extern char tl_ipi_tlb_range_demap[];
143
144 static __inline void
ipi_all_but_self(u_int ipi)145 ipi_all_but_self(u_int ipi)
146 {
147 cpuset_t cpus;
148
149 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
150 return;
151 cpus = all_cpus;
152 sched_pin();
153 CPU_CLR(PCPU_GET(cpuid), &cpus);
154 mtx_lock_spin(&ipi_mtx);
155 cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
156 mtx_unlock_spin(&ipi_mtx);
157 sched_unpin();
158 }
159
160 static __inline void
ipi_selected(cpuset_t cpus,u_int ipi)161 ipi_selected(cpuset_t cpus, u_int ipi)
162 {
163
164 if (__predict_false(atomic_load_acq_int(&smp_started) == 0 ||
165 CPU_EMPTY(&cpus)))
166 return;
167 mtx_lock_spin(&ipi_mtx);
168 cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
169 mtx_unlock_spin(&ipi_mtx);
170 }
171
172 static __inline void
ipi_cpu(int cpu,u_int ipi)173 ipi_cpu(int cpu, u_int ipi)
174 {
175
176 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
177 return;
178 mtx_lock_spin(&ipi_mtx);
179 cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi);
180 mtx_unlock_spin(&ipi_mtx);
181 }
182
183 #if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_)
184
185 static __inline void *
ipi_dcache_page_inval(void * func,vm_paddr_t pa)186 ipi_dcache_page_inval(void *func, vm_paddr_t pa)
187 {
188 struct ipi_cache_args *ica;
189
190 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
191 return (NULL);
192 sched_pin();
193 ica = &ipi_cache_args;
194 mtx_lock_spin(&ipi_mtx);
195 ica->ica_mask = all_cpus;
196 CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask);
197 ica->ica_pa = pa;
198 cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica);
199 return (&ica->ica_mask);
200 }
201
202 static __inline void *
ipi_icache_page_inval(void * func,vm_paddr_t pa)203 ipi_icache_page_inval(void *func, vm_paddr_t pa)
204 {
205 struct ipi_cache_args *ica;
206
207 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
208 return (NULL);
209 sched_pin();
210 ica = &ipi_cache_args;
211 mtx_lock_spin(&ipi_mtx);
212 ica->ica_mask = all_cpus;
213 CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask);
214 ica->ica_pa = pa;
215 cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica);
216 return (&ica->ica_mask);
217 }
218
219 static __inline void *
ipi_rd(u_int cpu,void * func,u_long * val)220 ipi_rd(u_int cpu, void *func, u_long *val)
221 {
222 struct ipi_rd_args *ira;
223
224 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
225 return (NULL);
226 sched_pin();
227 ira = &ipi_rd_args;
228 mtx_lock_spin(&ipi_mtx);
229 CPU_SETOF(cpu, &ira->ira_mask);
230 ira->ira_val = val;
231 cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira);
232 return (&ira->ira_mask);
233 }
234
235 static __inline void *
ipi_tlb_context_demap(struct pmap * pm)236 ipi_tlb_context_demap(struct pmap *pm)
237 {
238 struct ipi_tlb_args *ita;
239 cpuset_t cpus;
240
241 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
242 return (NULL);
243 sched_pin();
244 cpus = pm->pm_active;
245 CPU_AND(&cpus, &all_cpus);
246 CPU_CLR(PCPU_GET(cpuid), &cpus);
247 if (CPU_EMPTY(&cpus)) {
248 sched_unpin();
249 return (NULL);
250 }
251 ita = &ipi_tlb_args;
252 mtx_lock_spin(&ipi_mtx);
253 ita->ita_mask = cpus;
254 ita->ita_pmap = pm;
255 cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap,
256 (u_long)ita);
257 return (&ita->ita_mask);
258 }
259
260 static __inline void *
ipi_tlb_page_demap(struct pmap * pm,vm_offset_t va)261 ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va)
262 {
263 struct ipi_tlb_args *ita;
264 cpuset_t cpus;
265
266 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
267 return (NULL);
268 sched_pin();
269 cpus = pm->pm_active;
270 CPU_AND(&cpus, &all_cpus);
271 CPU_CLR(PCPU_GET(cpuid), &cpus);
272 if (CPU_EMPTY(&cpus)) {
273 sched_unpin();
274 return (NULL);
275 }
276 ita = &ipi_tlb_args;
277 mtx_lock_spin(&ipi_mtx);
278 ita->ita_mask = cpus;
279 ita->ita_pmap = pm;
280 ita->ita_va = va;
281 cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_page_demap, (u_long)ita);
282 return (&ita->ita_mask);
283 }
284
285 static __inline void *
ipi_tlb_range_demap(struct pmap * pm,vm_offset_t start,vm_offset_t end)286 ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
287 {
288 struct ipi_tlb_args *ita;
289 cpuset_t cpus;
290
291 if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
292 return (NULL);
293 sched_pin();
294 cpus = pm->pm_active;
295 CPU_AND(&cpus, &all_cpus);
296 CPU_CLR(PCPU_GET(cpuid), &cpus);
297 if (CPU_EMPTY(&cpus)) {
298 sched_unpin();
299 return (NULL);
300 }
301 ita = &ipi_tlb_args;
302 mtx_lock_spin(&ipi_mtx);
303 ita->ita_mask = cpus;
304 ita->ita_pmap = pm;
305 ita->ita_start = start;
306 ita->ita_end = end;
307 cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap,
308 (u_long)ita);
309 return (&ita->ita_mask);
310 }
311
312 static __inline void
ipi_wait(void * cookie)313 ipi_wait(void *cookie)
314 {
315 volatile cpuset_t *mask;
316
317 if (__predict_false((mask = cookie) != NULL)) {
318 while (!CPU_EMPTY(mask))
319 ;
320 mtx_unlock_spin(&ipi_mtx);
321 sched_unpin();
322 }
323 }
324
325 #endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */
326
327 #endif /* !LOCORE */
328
329 #else
330
331 #ifndef LOCORE
332
333 static __inline void *
ipi_dcache_page_inval(void * func __unused,vm_paddr_t pa __unused)334 ipi_dcache_page_inval(void *func __unused, vm_paddr_t pa __unused)
335 {
336
337 return (NULL);
338 }
339
340 static __inline void *
ipi_icache_page_inval(void * func __unused,vm_paddr_t pa __unused)341 ipi_icache_page_inval(void *func __unused, vm_paddr_t pa __unused)
342 {
343
344 return (NULL);
345 }
346
347 static __inline void *
ipi_rd(u_int cpu __unused,void * func __unused,u_long * val __unused)348 ipi_rd(u_int cpu __unused, void *func __unused, u_long *val __unused)
349 {
350
351 return (NULL);
352 }
353
354 static __inline void *
ipi_tlb_context_demap(struct pmap * pm __unused)355 ipi_tlb_context_demap(struct pmap *pm __unused)
356 {
357
358 return (NULL);
359 }
360
361 static __inline void *
ipi_tlb_page_demap(struct pmap * pm __unused,vm_offset_t va __unused)362 ipi_tlb_page_demap(struct pmap *pm __unused, vm_offset_t va __unused)
363 {
364
365 return (NULL);
366 }
367
368 static __inline void *
ipi_tlb_range_demap(struct pmap * pm __unused,vm_offset_t start __unused,__unused vm_offset_t end)369 ipi_tlb_range_demap(struct pmap *pm __unused, vm_offset_t start __unused,
370 __unused vm_offset_t end)
371 {
372
373 return (NULL);
374 }
375
376 static __inline void
ipi_wait(void * cookie __unused)377 ipi_wait(void *cookie __unused)
378 {
379
380 }
381
382 static __inline void
tl_ipi_cheetah_dcache_page_inval(void)383 tl_ipi_cheetah_dcache_page_inval(void)
384 {
385
386 }
387
388 static __inline void
tl_ipi_spitfire_dcache_page_inval(void)389 tl_ipi_spitfire_dcache_page_inval(void)
390 {
391
392 }
393
394 static __inline void
tl_ipi_spitfire_icache_page_inval(void)395 tl_ipi_spitfire_icache_page_inval(void)
396 {
397
398 }
399
400 #endif /* !LOCORE */
401
402 #endif /* SMP */
403
404 #endif /* !_MACHINE_SMP_H_ */
405