1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 Andrew Turner
5 * All rights reserved.
6 *
7 * This software was developed by SRI International and the University of
8 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
9 * ("CTSRD"), as part of the DARPA CRASH research programme.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include "opt_platform.h"
34
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <sys/kernel.h>
38 #include <sys/pcpu.h>
39 #include <sys/systm.h>
40
41 #include <machine/cpu.h>
42
43 #include <dev/psci/psci.h>
44 #include <dev/psci/smccc.h>
45
46 typedef void (cpu_quirk_install)(void);
47 struct cpu_quirks {
48 cpu_quirk_install *quirk_install;
49 u_int midr_mask;
50 u_int midr_value;
51 #define CPU_QUIRK_POST_DEVICE (1 << 0) /* After device attach */
52 /* e.g. needs SMCCC */
53 u_int flags;
54 };
55
56 static enum {
57 SSBD_FORCE_ON,
58 SSBD_FORCE_OFF,
59 SSBD_KERNEL,
60 } ssbd_method = SSBD_KERNEL;
61
62 static cpu_quirk_install install_psci_bp_hardening;
63 static cpu_quirk_install install_ssbd_workaround;
64 static cpu_quirk_install install_thunderx_bcast_tlbi_workaround;
65
66 static struct cpu_quirks cpu_quirks[] = {
67 {
68 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
69 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A57,0,0),
70 .quirk_install = install_psci_bp_hardening,
71 .flags = CPU_QUIRK_POST_DEVICE,
72 },
73 {
74 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
75 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A72,0,0),
76 .quirk_install = install_psci_bp_hardening,
77 .flags = CPU_QUIRK_POST_DEVICE,
78 },
79 {
80 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
81 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A73,0,0),
82 .quirk_install = install_psci_bp_hardening,
83 .flags = CPU_QUIRK_POST_DEVICE,
84 },
85 {
86 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
87 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A75,0,0),
88 .quirk_install = install_psci_bp_hardening,
89 .flags = CPU_QUIRK_POST_DEVICE,
90 },
91 {
92 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
93 .midr_value =
94 CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX2, 0,0),
95 .quirk_install = install_psci_bp_hardening,
96 .flags = CPU_QUIRK_POST_DEVICE,
97 },
98 {
99 .midr_mask = 0,
100 .midr_value = 0,
101 .quirk_install = install_ssbd_workaround,
102 .flags = CPU_QUIRK_POST_DEVICE,
103 },
104 {
105 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
106 .midr_value =
107 CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX, 0, 0),
108 .quirk_install = install_thunderx_bcast_tlbi_workaround,
109 },
110 {
111 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
112 .midr_value =
113 CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX_81XX, 0, 0),
114 .quirk_install = install_thunderx_bcast_tlbi_workaround,
115 },
116 };
117
118 static void
install_psci_bp_hardening(void)119 install_psci_bp_hardening(void)
120 {
121 /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */
122 if (!psci_present)
123 return;
124
125 if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_1) != SMCCC_RET_SUCCESS)
126 return;
127
128 PCPU_SET(bp_harden, smccc_arch_workaround_1);
129 }
130
131 static void
install_ssbd_workaround(void)132 install_ssbd_workaround(void)
133 {
134 char *env;
135
136 if (PCPU_GET(cpuid) == 0) {
137 env = kern_getenv("kern.cfg.ssbd");
138 if (env != NULL) {
139 if (strcmp(env, "force-on") == 0) {
140 ssbd_method = SSBD_FORCE_ON;
141 } else if (strcmp(env, "force-off") == 0) {
142 ssbd_method = SSBD_FORCE_OFF;
143 }
144 }
145 }
146
147 /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */
148 if (!psci_present)
149 return;
150
151 /* Enable the workaround on this CPU if it's enabled in the firmware */
152 if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_2) != SMCCC_RET_SUCCESS)
153 return;
154
155 switch(ssbd_method) {
156 case SSBD_FORCE_ON:
157 smccc_arch_workaround_2(1);
158 break;
159 case SSBD_FORCE_OFF:
160 smccc_arch_workaround_2(0);
161 break;
162 case SSBD_KERNEL:
163 default:
164 PCPU_SET(ssbd, smccc_arch_workaround_2);
165 break;
166 }
167 }
168
169 /*
170 * Workaround Cavium erratum 27456.
171 *
172 * Invalidate the local icache when changing address spaces.
173 */
174 static void
install_thunderx_bcast_tlbi_workaround(void)175 install_thunderx_bcast_tlbi_workaround(void)
176 {
177 u_int midr;
178
179 midr = get_midr();
180 if (CPU_PART(midr) == CPU_PART_THUNDERX_81XX)
181 PCPU_SET(bcast_tlbi_workaround, 1);
182 else if (CPU_PART(midr) == CPU_PART_THUNDERX) {
183 if (CPU_VAR(midr) == 0) {
184 /* ThunderX 1.x */
185 PCPU_SET(bcast_tlbi_workaround, 1);
186 } else if (CPU_VAR(midr) == 1 && CPU_REV(midr) <= 1) {
187 /* ThunderX 2.0 - 2.1 */
188 PCPU_SET(bcast_tlbi_workaround, 1);
189 }
190 }
191 }
192
193 static void
install_cpu_errata_flags(u_int mask,u_int flags)194 install_cpu_errata_flags(u_int mask, u_int flags)
195 {
196 u_int midr;
197 size_t i;
198
199 midr = get_midr();
200
201 for (i = 0; i < nitems(cpu_quirks); i++) {
202 if ((midr & cpu_quirks[i].midr_mask) ==
203 cpu_quirks[i].midr_value &&
204 (cpu_quirks[i].flags & mask) == flags) {
205 cpu_quirks[i].quirk_install();
206 }
207 }
208 }
209
210 /*
211 * Install any CPU errata we need. On CPU 0 we only install the errata that
212 * don't depend on device drivers as this is called early in the boot process.
213 * On other CPUs the device drivers have already attached so install all
214 * applicable errata.
215 */
216 void
install_cpu_errata(void)217 install_cpu_errata(void)
218 {
219 /*
220 * Only install early CPU errata on CPU 0, device drivers may not
221 * have attached and some workarounds depend on them, e.g. to query
222 * SMCCC.
223 */
224 if (PCPU_GET(cpuid) == 0) {
225 install_cpu_errata_flags(CPU_QUIRK_POST_DEVICE, 0);
226 } else {
227 install_cpu_errata_flags(0, 0);
228 }
229 }
230
231 /*
232 * Install any errata workarounds that depend on device drivers, e.g. use
233 * SMCCC to install a workaround.
234 */
235 static void
install_cpu_errata_late(void * dummy __unused)236 install_cpu_errata_late(void *dummy __unused)
237 {
238 MPASS(PCPU_GET(cpuid) == 0);
239 install_cpu_errata_flags(CPU_QUIRK_POST_DEVICE, CPU_QUIRK_POST_DEVICE);
240 }
241 SYSINIT(install_cpu_errata_late, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE,
242 install_cpu_errata_late, NULL);
243