1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Semihalf
7 * under sponsorship of the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/kernel.h>
35 #include <sys/malloc.h>
36 #include <sys/proc.h>
37 #include <sys/pcpu.h>
38 #include <sys/sbuf.h>
39 #include <sys/smp.h>
40 #include <sys/sysctl.h>
41 #include <sys/sysent.h>
42 #include <sys/systm.h>
43
44 #include <machine/atomic.h>
45 #include <machine/cpu.h>
46 #include <machine/cpufunc.h>
47 #include <machine/elf.h>
48 #include <machine/md_var.h>
49 #include <machine/undefined.h>
50
51 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
52
53 struct cpu_desc;
54
55 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
56 static void print_cpu_features(u_int cpu, struct cpu_desc *desc,
57 struct cpu_desc *prev_desc);
58 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc);
59 #ifdef COMPAT_FREEBSD32
60 static u_long parse_cpu_features_hwcap32(void);
61 #endif
62
63 const char machine[] = "arm64";
64
65 #ifdef SCTL_MASK32
66 extern int adaptive_machine_arch;
67 #endif
68
69 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
70 "Cache management tuning");
71
72 static int allow_dic = 1;
73 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
74 "Allow optimizations based on the DIC cache bit");
75
76 static int allow_idc = 1;
77 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
78 "Allow optimizations based on the IDC cache bit");
79
80 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
81 struct cpu_desc *prev_desc);
82
83 /*
84 * The default implementation of I-cache sync assumes we have an
85 * aliasing cache until we know otherwise.
86 */
87 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
88 &arm64_aliasing_icache_sync_range;
89
90 static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)91 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
92 {
93 #ifdef SCTL_MASK32
94 static const char machine32[] = "arm";
95 #endif
96 int error;
97
98 #ifdef SCTL_MASK32
99 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
100 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
101 else
102 #endif
103 error = SYSCTL_OUT(req, machine, sizeof(machine));
104 return (error);
105 }
106
107 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
108 CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
109 "Machine class");
110
111 static char cpu_model[64];
112 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD,
113 cpu_model, sizeof(cpu_model), "Machine model");
114
115 #define MAX_CACHES 8 /* Maximum number of caches supported
116 architecturally. */
117 /*
118 * Per-CPU affinity as provided in MPIDR_EL1
119 * Indexed by CPU number in logical order selected by the system.
120 * Relevant fields can be extracted using CPU_AFFn macros,
121 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
122 *
123 * Fields used by us:
124 * Aff1 - Cluster number
125 * Aff0 - CPU number in Aff1 cluster
126 */
127 uint64_t __cpu_affinity[MAXCPU];
128 static u_int cpu_aff_levels;
129
130 struct cpu_desc {
131 uint64_t mpidr;
132 uint64_t id_aa64afr0;
133 uint64_t id_aa64afr1;
134 uint64_t id_aa64dfr0;
135 uint64_t id_aa64dfr1;
136 uint64_t id_aa64isar0;
137 uint64_t id_aa64isar1;
138 uint64_t id_aa64isar2;
139 uint64_t id_aa64mmfr0;
140 uint64_t id_aa64mmfr1;
141 uint64_t id_aa64mmfr2;
142 uint64_t id_aa64mmfr3;
143 uint64_t id_aa64mmfr4;
144 uint64_t id_aa64pfr0;
145 uint64_t id_aa64pfr1;
146 uint64_t id_aa64pfr2;
147 uint64_t id_aa64zfr0;
148 uint64_t ctr;
149 #ifdef COMPAT_FREEBSD32
150 uint64_t id_isar5;
151 uint64_t mvfr0;
152 uint64_t mvfr1;
153 #endif
154 uint64_t clidr;
155 uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */
156 bool have_sve;
157 };
158
159 static struct cpu_desc cpu_desc0;
160 static struct cpu_desc *cpu_desc;
161 static struct cpu_desc kern_cpu_desc;
162 static struct cpu_desc user_cpu_desc;
163 static struct cpu_desc l_user_cpu_desc;
164
165 static struct cpu_desc *
get_cpu_desc(u_int cpu)166 get_cpu_desc(u_int cpu)
167 {
168 /* The cpu_desc for CPU 0 is used before the allocator is ready. */
169 if (cpu == 0)
170 return (&cpu_desc0);
171
172 MPASS(cpu_desc != NULL);
173 return (&cpu_desc[cpu - 1]);
174 }
175
176 struct cpu_parts {
177 u_int part_id;
178 const char *part_name;
179 };
180 #define CPU_PART_NONE { 0, NULL }
181
182 struct cpu_implementers {
183 u_int impl_id;
184 const char *impl_name;
185 /*
186 * Part number is implementation defined
187 * so each vendor will have its own set of values and names.
188 */
189 const struct cpu_parts *cpu_parts;
190 };
191 #define CPU_IMPLEMENTER_NONE { 0, NULL, NULL }
192
193 /*
194 * Per-implementer table of (PartNum, CPU Name) pairs.
195 */
196 /* ARM Ltd. */
197 static const struct cpu_parts cpu_parts_arm[] = {
198 { CPU_PART_AEM_V8, "AEMv8" },
199 { CPU_PART_FOUNDATION, "Foundation-Model" },
200 { CPU_PART_CORTEX_A34, "Cortex-A34" },
201 { CPU_PART_CORTEX_A35, "Cortex-A35" },
202 { CPU_PART_CORTEX_A53, "Cortex-A53" },
203 { CPU_PART_CORTEX_A55, "Cortex-A55" },
204 { CPU_PART_CORTEX_A57, "Cortex-A57" },
205 { CPU_PART_CORTEX_A65, "Cortex-A65" },
206 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
207 { CPU_PART_CORTEX_A72, "Cortex-A72" },
208 { CPU_PART_CORTEX_A73, "Cortex-A73" },
209 { CPU_PART_CORTEX_A75, "Cortex-A75" },
210 { CPU_PART_CORTEX_A76, "Cortex-A76" },
211 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
212 { CPU_PART_CORTEX_A77, "Cortex-A77" },
213 { CPU_PART_CORTEX_A78, "Cortex-A78" },
214 { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
215 { CPU_PART_CORTEX_A510, "Cortex-A510" },
216 { CPU_PART_CORTEX_A710, "Cortex-A710" },
217 { CPU_PART_CORTEX_A715, "Cortex-A715" },
218 { CPU_PART_CORTEX_X1, "Cortex-X1" },
219 { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
220 { CPU_PART_CORTEX_X2, "Cortex-X2" },
221 { CPU_PART_CORTEX_X3, "Cortex-X3" },
222 { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
223 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
224 { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
225 { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
226 { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
227 CPU_PART_NONE,
228 };
229
230 /* Cavium */
231 static const struct cpu_parts cpu_parts_cavium[] = {
232 { CPU_PART_THUNDERX, "ThunderX" },
233 { CPU_PART_THUNDERX2, "ThunderX2" },
234 CPU_PART_NONE,
235 };
236
237 /* APM / Ampere */
238 static const struct cpu_parts cpu_parts_apm[] = {
239 { CPU_PART_EMAG8180, "eMAG 8180" },
240 CPU_PART_NONE,
241 };
242
243 /* Qualcomm */
244 static const struct cpu_parts cpu_parts_qcom[] = {
245 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
246 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
247 CPU_PART_NONE,
248 };
249
250 /* Apple */
251 static const struct cpu_parts cpu_parts_apple[] = {
252 { CPU_PART_M1_ICESTORM, "M1 Icestorm" },
253 { CPU_PART_M1_FIRESTORM, "M1 Firestorm" },
254 { CPU_PART_M1_ICESTORM_PRO, "M1 Pro Icestorm" },
255 { CPU_PART_M1_FIRESTORM_PRO, "M1 Pro Firestorm" },
256 { CPU_PART_M1_ICESTORM_MAX, "M1 Max Icestorm" },
257 { CPU_PART_M1_FIRESTORM_MAX, "M1 Max Firestorm" },
258 { CPU_PART_M2_BLIZZARD, "M2 Blizzard" },
259 { CPU_PART_M2_AVALANCHE, "M2 Avalanche" },
260 { CPU_PART_M2_BLIZZARD_PRO, "M2 Pro Blizzard" },
261 { CPU_PART_M2_AVALANCHE_PRO, "M2 Pro Avalanche" },
262 { CPU_PART_M2_BLIZZARD_MAX, "M2 Max Blizzard" },
263 { CPU_PART_M2_AVALANCHE_MAX, "M2 Max Avalanche" },
264 CPU_PART_NONE,
265 };
266
267 /* Unknown */
268 static const struct cpu_parts cpu_parts_none[] = {
269 CPU_PART_NONE,
270 };
271
272 /*
273 * Implementers table.
274 */
275 const struct cpu_implementers cpu_implementers[] = {
276 { CPU_IMPL_AMPERE, "Ampere", cpu_parts_none },
277 { CPU_IMPL_APPLE, "Apple", cpu_parts_apple },
278 { CPU_IMPL_APM, "APM", cpu_parts_apm },
279 { CPU_IMPL_ARM, "ARM", cpu_parts_arm },
280 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none },
281 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium },
282 { CPU_IMPL_DEC, "DEC", cpu_parts_none },
283 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none },
284 { CPU_IMPL_FUJITSU, "Fujitsu", cpu_parts_none },
285 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none },
286 { CPU_IMPL_INTEL, "Intel", cpu_parts_none },
287 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none },
288 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none },
289 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_qcom },
290 CPU_IMPLEMENTER_NONE,
291 };
292
293 #define MRS_TYPE_MASK 0xf
294 #define MRS_TYPE_FBSD_SHIFT 0
295 #define MRS_TYPE_LNX_SHIFT 8
296 #define MRS_INVALID 0
297 #define MRS_EXACT 1
298 #define MRS_EXACT_VAL(x) (MRS_EXACT | ((x) << 4))
299 #define MRS_EXACT_FIELD(x) (((x) >> 4) & 0xf)
300 #define MRS_LOWER 2
301
302 struct mrs_field_value {
303 uint64_t value;
304 const char *desc;
305 };
306
307 #define MRS_FIELD_VALUE(_value, _desc) \
308 { \
309 .value = (_value), \
310 .desc = (_desc), \
311 }
312
313 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \
314 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \
315 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
316
317 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \
318 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
319 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
320 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
321 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
322 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
323 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
324 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
325 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
326 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
327 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
328 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
329 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
330 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
331 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
332 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
333 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
334
335 #define MRS_FIELD_VALUE_END { .desc = NULL }
336
337 struct mrs_field_hwcap {
338 uint64_t min;
339 u_long hwcap_val;
340 u_int hwcap_id;
341 };
342
343 #define MRS_HWCAP(_hwcap_id, _val, _min) \
344 { \
345 .hwcap_id = (_hwcap_id), \
346 .hwcap_val = (_val), \
347 .min = (_min), \
348 }
349
350 #define MRS_HWCAP_END { .hwcap_id = 0 }
351
352 struct mrs_field {
353 const char *name;
354 const struct mrs_field_value *values;
355 const struct mrs_field_hwcap *hwcaps;
356 uint64_t mask;
357 bool sign;
358 u_int type;
359 u_int shift;
360 };
361
362 #define MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _fbsd_type, \
363 _lnx_type, _values, _hwcap) \
364 { \
365 .name = #_name, \
366 .sign = (_sign), \
367 .type = ((_fbsd_type) << MRS_TYPE_FBSD_SHIFT) | \
368 ((_lnx_type) << MRS_TYPE_LNX_SHIFT), \
369 .shift = _register ## _ ## _name ## _SHIFT, \
370 .mask = _register ## _ ## _name ## _MASK, \
371 .values = (_values), \
372 .hwcaps = (_hwcap), \
373 }
374
375 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
376 MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _type, _type, \
377 _values, _hwcap)
378
379 #define MRS_FIELD(_register, _name, _sign, _type, _values) \
380 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
381
382 #define MRS_FIELD_END { .type = MRS_INVALID, }
383
384 /* ID_AA64AFR0_EL1 */
385 static const struct mrs_field id_aa64afr0_fields[] = {
386 MRS_FIELD_END,
387 };
388
389
390 /* ID_AA64AFR1_EL1 */
391 static const struct mrs_field id_aa64afr1_fields[] = {
392 MRS_FIELD_END,
393 };
394
395
396 /* ID_AA64DFR0_EL1 */
397 static const struct mrs_field_value id_aa64dfr0_hpmn0[] = {
398 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED),
399 MRS_FIELD_VALUE_END,
400 };
401
402 static const struct mrs_field_value id_aa64dfr0_brbe[] = {
403 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL),
404 MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"),
405 MRS_FIELD_VALUE_END,
406 };
407
408 static const struct mrs_field_value id_aa64dfr0_mtpmu[] = {
409 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL),
410 MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"),
411 MRS_FIELD_VALUE_END,
412 };
413
414 static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = {
415 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL),
416 MRS_FIELD_VALUE_END,
417 };
418
419 static const struct mrs_field_value id_aa64dfr0_tracefilt[] = {
420 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
421 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
422 MRS_FIELD_VALUE_END,
423 };
424
425 static const struct mrs_field_value id_aa64dfr0_doublelock[] = {
426 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
427 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
428 MRS_FIELD_VALUE_END,
429 };
430
431 static const struct mrs_field_value id_aa64dfr0_pmsver[] = {
432 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
433 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
434 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"),
435 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"),
436 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"),
437 MRS_FIELD_VALUE_END,
438 };
439
440 static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
441 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
442 MRS_FIELD_VALUE_END,
443 };
444
445 static const struct mrs_field_value id_aa64dfr0_wrps[] = {
446 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
447 MRS_FIELD_VALUE_END,
448 };
449
450 static const struct mrs_field_value id_aa64dfr0_brps[] = {
451 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
452 MRS_FIELD_VALUE_END,
453 };
454
455 static const struct mrs_field_value id_aa64dfr0_pmuver[] = {
456 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
457 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
458 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"),
459 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"),
460 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"),
461 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"),
462 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"),
463 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
464 MRS_FIELD_VALUE_END,
465 };
466
467 static const struct mrs_field_value id_aa64dfr0_tracever[] = {
468 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
469 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
470 MRS_FIELD_VALUE_END,
471 };
472
473 static const struct mrs_field_value id_aa64dfr0_debugver[] = {
474 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
475 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
476 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"),
477 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"),
478 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"),
479 MRS_FIELD_VALUE_END,
480 };
481
482 static const struct mrs_field id_aa64dfr0_fields[] = {
483 MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_EXACT, id_aa64dfr0_hpmn0),
484 MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_EXACT, id_aa64dfr0_brbe),
485 MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_EXACT, id_aa64dfr0_mtpmu),
486 MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_EXACT,
487 id_aa64dfr0_tracebuffer),
488 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
489 id_aa64dfr0_tracefilt),
490 MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
491 id_aa64dfr0_doublelock),
492 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
493 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
494 id_aa64dfr0_ctx_cmps),
495 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
496 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
497 MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
498 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
499 id_aa64dfr0_tracever),
500 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
501 id_aa64dfr0_debugver),
502 MRS_FIELD_END,
503 };
504
505
506 /* ID_AA64DFR1_EL1 */
507 static const struct mrs_field id_aa64dfr1_fields[] = {
508 MRS_FIELD_END,
509 };
510
511
512 /* ID_AA64ISAR0_EL1 */
513 static const struct mrs_field_value id_aa64isar0_rndr[] = {
514 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
515 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
516 MRS_FIELD_VALUE_END,
517 };
518
519 static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
520 MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
521 MRS_HWCAP_END
522 };
523
524 static const struct mrs_field_value id_aa64isar0_tlb[] = {
525 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
526 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
527 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
528 MRS_FIELD_VALUE_END,
529 };
530
531 static const struct mrs_field_value id_aa64isar0_ts[] = {
532 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
533 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
534 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
535 MRS_FIELD_VALUE_END,
536 };
537
538 static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
539 MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
540 MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
541 MRS_HWCAP_END
542 };
543
544 static const struct mrs_field_value id_aa64isar0_fhm[] = {
545 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
546 MRS_FIELD_VALUE_END,
547 };
548
549 static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
550 MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
551 MRS_HWCAP_END
552 };
553
554 static const struct mrs_field_value id_aa64isar0_dp[] = {
555 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
556 MRS_FIELD_VALUE_END,
557 };
558
559 static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
560 MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
561 MRS_HWCAP_END
562 };
563
564 static const struct mrs_field_value id_aa64isar0_sm4[] = {
565 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
566 MRS_FIELD_VALUE_END,
567 };
568
569 static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
570 MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
571 MRS_HWCAP_END
572 };
573
574 static const struct mrs_field_value id_aa64isar0_sm3[] = {
575 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
576 MRS_FIELD_VALUE_END,
577 };
578
579 static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
580 MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
581 MRS_HWCAP_END
582 };
583
584 static const struct mrs_field_value id_aa64isar0_sha3[] = {
585 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
586 MRS_FIELD_VALUE_END,
587 };
588
589 static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
590 MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
591 MRS_HWCAP_END
592 };
593
594 static const struct mrs_field_value id_aa64isar0_rdm[] = {
595 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
596 MRS_FIELD_VALUE_END,
597 };
598
599 static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
600 MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
601 MRS_HWCAP_END
602 };
603
604 static const struct mrs_field_value id_aa64isar0_tme[] = {
605 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL),
606 MRS_FIELD_VALUE_END,
607 };
608
609 static const struct mrs_field_value id_aa64isar0_atomic[] = {
610 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
611 MRS_FIELD_VALUE_END,
612 };
613
614 static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
615 MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
616 MRS_HWCAP_END
617 };
618
619 static const struct mrs_field_value id_aa64isar0_crc32[] = {
620 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
621 MRS_FIELD_VALUE_END,
622 };
623
624 static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
625 MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
626 MRS_HWCAP_END
627 };
628
629 static const struct mrs_field_value id_aa64isar0_sha2[] = {
630 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
631 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
632 MRS_FIELD_VALUE_END,
633 };
634
635 static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
636 MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
637 MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
638 MRS_HWCAP_END
639 };
640
641 static const struct mrs_field_value id_aa64isar0_sha1[] = {
642 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
643 MRS_FIELD_VALUE_END,
644 };
645
646 static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
647 MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
648 MRS_HWCAP_END
649 };
650
651 static const struct mrs_field_value id_aa64isar0_aes[] = {
652 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
653 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
654 MRS_FIELD_VALUE_END,
655 };
656
657 static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
658 MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
659 MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
660 MRS_HWCAP_END
661 };
662
663 static const struct mrs_field id_aa64isar0_fields[] = {
664 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
665 id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
666 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
667 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
668 id_aa64isar0_ts_caps),
669 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
670 id_aa64isar0_fhm_caps),
671 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
672 id_aa64isar0_dp_caps),
673 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
674 id_aa64isar0_sm4_caps),
675 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
676 id_aa64isar0_sm3_caps),
677 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
678 id_aa64isar0_sha3_caps),
679 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
680 id_aa64isar0_rdm_caps),
681 MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_EXACT, id_aa64isar0_tme),
682 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
683 id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
684 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
685 id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
686 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
687 id_aa64isar0_sha2_caps),
688 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
689 id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
690 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
691 id_aa64isar0_aes_caps),
692 MRS_FIELD_END,
693 };
694
695
696 /* ID_AA64ISAR1_EL1 */
697 static const struct mrs_field_value id_aa64isar1_ls64[] = {
698 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL),
699 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"),
700 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"),
701 MRS_FIELD_VALUE_END,
702 };
703
704 static const struct mrs_field_value id_aa64isar1_xs[] = {
705 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL),
706 MRS_FIELD_VALUE_END,
707 };
708
709 static const struct mrs_field_value id_aa64isar1_i8mm[] = {
710 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
711 MRS_FIELD_VALUE_END,
712 };
713
714 static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
715 MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
716 MRS_HWCAP_END
717 };
718
719 static const struct mrs_field_value id_aa64isar1_dgh[] = {
720 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
721 MRS_FIELD_VALUE_END,
722 };
723
724 static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
725 MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
726 MRS_HWCAP_END
727 };
728
729 static const struct mrs_field_value id_aa64isar1_bf16[] = {
730 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
731 MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"),
732 MRS_FIELD_VALUE_END,
733 };
734
735 static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
736 MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
737 MRS_HWCAP_END
738 };
739
740 static const struct mrs_field_value id_aa64isar1_specres[] = {
741 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
742 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
743 MRS_FIELD_VALUE_END,
744 };
745
746 static const struct mrs_field_value id_aa64isar1_sb[] = {
747 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
748 MRS_FIELD_VALUE_END,
749 };
750
751 static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
752 MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
753 MRS_HWCAP_END
754 };
755
756 static const struct mrs_field_value id_aa64isar1_frintts[] = {
757 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
758 MRS_FIELD_VALUE_END,
759 };
760
761 static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
762 MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
763 MRS_HWCAP_END
764 };
765
766 static const struct mrs_field_value id_aa64isar1_gpi[] = {
767 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
768 MRS_FIELD_VALUE_END,
769 };
770
771 static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
772 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
773 MRS_HWCAP_END
774 };
775
776 static const struct mrs_field_value id_aa64isar1_gpa[] = {
777 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
778 MRS_FIELD_VALUE_END,
779 };
780
781 static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
782 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
783 MRS_HWCAP_END
784 };
785
786 static const struct mrs_field_value id_aa64isar1_lrcpc[] = {
787 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
788 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
789 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
790 MRS_FIELD_VALUE_END,
791 };
792
793 static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
794 MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
795 MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
796 MRS_HWCAP_END
797 };
798
799 static const struct mrs_field_value id_aa64isar1_fcma[] = {
800 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
801 MRS_FIELD_VALUE_END,
802 };
803
804 static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
805 MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
806 MRS_HWCAP_END
807 };
808
809 static const struct mrs_field_value id_aa64isar1_jscvt[] = {
810 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
811 MRS_FIELD_VALUE_END,
812 };
813
814 static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
815 MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
816 MRS_HWCAP_END
817 };
818
819 static const struct mrs_field_value id_aa64isar1_api[] = {
820 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
821 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
822 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
823 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
824 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
825 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
826 "Impl PAuth+FPAC+Combined"),
827 MRS_FIELD_VALUE_END,
828 };
829
830 static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
831 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
832 MRS_HWCAP_END
833 };
834
835 static const struct mrs_field_value id_aa64isar1_apa[] = {
836 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
837 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
838 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
839 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
840 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
841 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
842 "APA FPAC+Combined"),
843 MRS_FIELD_VALUE_END,
844 };
845
846 static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
847 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
848 MRS_HWCAP_END
849 };
850
851 static const struct mrs_field_value id_aa64isar1_dpb[] = {
852 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
853 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
854 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
855 MRS_FIELD_VALUE_END,
856 };
857
858 static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
859 MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
860 MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
861 MRS_HWCAP_END
862 };
863
864 static const struct mrs_field id_aa64isar1_fields[] = {
865 MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_EXACT, id_aa64isar1_ls64),
866 MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_EXACT, id_aa64isar1_xs),
867 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
868 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
869 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
870 id_aa64isar1_dgh_caps),
871 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
872 id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
873 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
874 id_aa64isar1_specres),
875 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
876 id_aa64isar1_sb_caps),
877 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
878 id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
879 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
880 id_aa64isar1_gpi_caps),
881 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
882 id_aa64isar1_gpa_caps),
883 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
884 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
885 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
886 id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
887 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
888 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
889 MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
890 id_aa64isar1_api_caps),
891 MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
892 id_aa64isar1_apa_caps),
893 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
894 id_aa64isar1_dpb_caps),
895 MRS_FIELD_END,
896 };
897
898
899 /* ID_AA64ISAR2_EL1 */
900 static const struct mrs_field_value id_aa64isar2_pac_frac[] = {
901 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
902 MRS_FIELD_VALUE_END,
903 };
904
905 static const struct mrs_field_value id_aa64isar2_bc[] = {
906 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
907 MRS_FIELD_VALUE_END,
908 };
909
910 static const struct mrs_field_value id_aa64isar2_mops[] = {
911 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
912 MRS_FIELD_VALUE_END,
913 };
914
915 static const struct mrs_field_value id_aa64isar2_apa3[] = {
916 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
917 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
918 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
919 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
920 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
921 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
922 "APA3 FPAC+Combined"),
923 MRS_FIELD_VALUE_END,
924 };
925
926 static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
927 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
928 MRS_HWCAP_END
929 };
930
931 static const struct mrs_field_value id_aa64isar2_gpa3[] = {
932 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
933 MRS_FIELD_VALUE_END,
934 };
935
936 static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
937 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
938 MRS_HWCAP_END
939 };
940
941 static const struct mrs_field_value id_aa64isar2_rpres[] = {
942 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
943 MRS_FIELD_VALUE_END,
944 };
945
946 static const struct mrs_field_hwcap id_aa64isar2_rpres_caps[] = {
947 MRS_HWCAP(2, HWCAP2_RPRES, ID_AA64ISAR2_RPRES_IMPL),
948 MRS_HWCAP_END
949 };
950
951 static const struct mrs_field_value id_aa64isar2_wfxt[] = {
952 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
953 MRS_FIELD_VALUE_END,
954 };
955
956 static const struct mrs_field id_aa64isar2_fields[] = {
957 MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
958 id_aa64isar2_pac_frac),
959 MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
960 MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
961 MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
962 id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
963 MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
964 id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
965 MRS_FIELD_HWCAP(ID_AA64ISAR2, RPRES, false, MRS_LOWER,
966 id_aa64isar2_rpres, id_aa64isar2_rpres_caps),
967 MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
968 MRS_FIELD_END,
969 };
970
971
972 /* ID_AA64MMFR0_EL1 */
973 static const struct mrs_field_value id_aa64mmfr0_ecv[] = {
974 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL),
975 MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_CNTHCTL, "ECV+CNTHCTL"),
976 MRS_FIELD_VALUE_END,
977 };
978
979 static const struct mrs_field_value id_aa64mmfr0_fgt[] = {
980 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, FGT, NONE, IMPL),
981 MRS_FIELD_VALUE_END,
982 };
983
984 static const struct mrs_field_value id_aa64mmfr0_exs[] = {
985 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
986 MRS_FIELD_VALUE_END,
987 };
988
989 static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
990 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
991 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
992 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
993 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"),
994 MRS_FIELD_VALUE_END,
995 };
996
997 static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
998 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
999 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
1000 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
1001 MRS_FIELD_VALUE_END,
1002 };
1003
1004 static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
1005 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
1006 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
1007 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
1008 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"),
1009 MRS_FIELD_VALUE_END,
1010 };
1011
1012 static const struct mrs_field_value id_aa64mmfr0_tgran4[] = {
1013 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL),
1014 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"),
1015 MRS_FIELD_VALUE_END,
1016 };
1017
1018 static const struct mrs_field_value id_aa64mmfr0_tgran64[] = {
1019 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
1020 MRS_FIELD_VALUE_END,
1021 };
1022
1023 static const struct mrs_field_value id_aa64mmfr0_tgran16[] = {
1024 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
1025 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"),
1026 MRS_FIELD_VALUE_END,
1027 };
1028
1029 static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
1030 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
1031 MRS_FIELD_VALUE_END,
1032 };
1033
1034 static const struct mrs_field_value id_aa64mmfr0_snsmem[] = {
1035 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
1036 MRS_FIELD_VALUE_END,
1037 };
1038
1039 static const struct mrs_field_value id_aa64mmfr0_bigend[] = {
1040 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
1041 MRS_FIELD_VALUE_END,
1042 };
1043
1044 static const struct mrs_field_value id_aa64mmfr0_asidbits[] = {
1045 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
1046 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
1047 MRS_FIELD_VALUE_END,
1048 };
1049
1050 static const struct mrs_field_value id_aa64mmfr0_parange[] = {
1051 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
1052 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
1053 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
1054 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
1055 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
1056 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
1057 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
1058 MRS_FIELD_VALUE_END,
1059 };
1060
1061 static const struct mrs_field id_aa64mmfr0_fields[] = {
1062 MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_EXACT, id_aa64mmfr0_ecv),
1063 MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_EXACT, id_aa64mmfr0_fgt),
1064 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
1065 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
1066 id_aa64mmfr0_tgran4_2),
1067 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
1068 id_aa64mmfr0_tgran64_2),
1069 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
1070 id_aa64mmfr0_tgran16_2),
1071 MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
1072 MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
1073 id_aa64mmfr0_tgran64),
1074 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
1075 id_aa64mmfr0_tgran16),
1076 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
1077 id_aa64mmfr0_bigendel0),
1078 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
1079 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
1080 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
1081 id_aa64mmfr0_asidbits),
1082 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
1083 id_aa64mmfr0_parange),
1084 MRS_FIELD_END,
1085 };
1086
1087
1088 /* ID_AA64MMFR1_EL1 */
1089 static const struct mrs_field_value id_aa64mmfr1_cmovw[] = {
1090 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL),
1091 MRS_FIELD_VALUE_END,
1092 };
1093
1094 static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = {
1095 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL),
1096 MRS_FIELD_VALUE_END,
1097 };
1098
1099 static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = {
1100 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL),
1101 MRS_FIELD_VALUE_END,
1102 };
1103
1104 static const struct mrs_field_value id_aa64mmfr1_afp[] = {
1105 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL),
1106 MRS_FIELD_VALUE_END,
1107 };
1108
1109 static const struct mrs_field_hwcap id_aa64mmfr1_afp_caps[] = {
1110 MRS_HWCAP(2, HWCAP2_AFP, ID_AA64MMFR1_AFP_IMPL),
1111 MRS_HWCAP_END
1112 };
1113
1114 static const struct mrs_field_value id_aa64mmfr1_hcx[] = {
1115 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL),
1116 MRS_FIELD_VALUE_END,
1117 };
1118
1119 static const struct mrs_field_value id_aa64mmfr1_ets[] = {
1120 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ETS, NONE, IMPL),
1121 MRS_FIELD_VALUE_END,
1122 };
1123
1124 static const struct mrs_field_value id_aa64mmfr1_twed[] = {
1125 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL),
1126 MRS_FIELD_VALUE_END,
1127 };
1128
1129 static const struct mrs_field_value id_aa64mmfr1_xnx[] = {
1130 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
1131 MRS_FIELD_VALUE_END,
1132 };
1133
1134 static const struct mrs_field_value id_aa64mmfr1_specsei[] = {
1135 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
1136 MRS_FIELD_VALUE_END,
1137 };
1138
1139 static const struct mrs_field_value id_aa64mmfr1_pan[] = {
1140 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
1141 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
1142 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"),
1143 MRS_FIELD_VALUE_END,
1144 };
1145
1146 static const struct mrs_field_value id_aa64mmfr1_lo[] = {
1147 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
1148 MRS_FIELD_VALUE_END,
1149 };
1150
1151 static const struct mrs_field_value id_aa64mmfr1_hpds[] = {
1152 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
1153 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
1154 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
1155 MRS_FIELD_VALUE_END,
1156 };
1157
1158 static const struct mrs_field_value id_aa64mmfr1_vh[] = {
1159 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
1160 MRS_FIELD_VALUE_END,
1161 };
1162
1163 static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
1164 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1165 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1166 MRS_FIELD_VALUE_END,
1167 };
1168
1169 static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1170 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1171 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1172 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1173 MRS_FIELD_VALUE_END,
1174 };
1175
1176 static const struct mrs_field id_aa64mmfr1_fields[] = {
1177 MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_EXACT, id_aa64mmfr1_cmovw),
1178 MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_EXACT, id_aa64mmfr1_tidcp1),
1179 MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_EXACT, id_aa64mmfr1_ntlbpa),
1180 MRS_FIELD_HWCAP(ID_AA64MMFR1, AFP, false, MRS_LOWER, id_aa64mmfr1_afp,
1181 id_aa64mmfr1_afp_caps),
1182 MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_EXACT, id_aa64mmfr1_hcx),
1183 MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_EXACT, id_aa64mmfr1_ets),
1184 MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_EXACT, id_aa64mmfr1_twed),
1185 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
1186 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
1187 id_aa64mmfr1_specsei),
1188 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
1189 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
1190 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
1191 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
1192 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
1193 id_aa64mmfr1_vmidbits),
1194 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
1195 MRS_FIELD_END,
1196 };
1197
1198
1199 /* ID_AA64MMFR2_EL1 */
1200 static const struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1201 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1202 MRS_FIELD_VALUE_END,
1203 };
1204
1205 static const struct mrs_field_value id_aa64mmfr2_evt[] = {
1206 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1207 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1208 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1209 MRS_FIELD_VALUE_END,
1210 };
1211
1212 static const struct mrs_field_value id_aa64mmfr2_bbm[] = {
1213 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1214 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1215 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1216 MRS_FIELD_VALUE_END,
1217 };
1218
1219 static const struct mrs_field_value id_aa64mmfr2_ttl[] = {
1220 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1221 MRS_FIELD_VALUE_END,
1222 };
1223
1224 static const struct mrs_field_value id_aa64mmfr2_fwb[] = {
1225 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1226 MRS_FIELD_VALUE_END,
1227 };
1228
1229 static const struct mrs_field_value id_aa64mmfr2_ids[] = {
1230 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1231 MRS_FIELD_VALUE_END,
1232 };
1233
1234 static const struct mrs_field_value id_aa64mmfr2_at[] = {
1235 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1236 MRS_FIELD_VALUE_END,
1237 };
1238
1239 static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1240 MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1241 MRS_HWCAP_END
1242 };
1243
1244 static const struct mrs_field_value id_aa64mmfr2_st[] = {
1245 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1246 MRS_FIELD_VALUE_END,
1247 };
1248
1249 static const struct mrs_field_value id_aa64mmfr2_nv[] = {
1250 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1251 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1252 MRS_FIELD_VALUE_END,
1253 };
1254
1255 static const struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1256 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1257 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1258 MRS_FIELD_VALUE_END,
1259 };
1260
1261 static const struct mrs_field_value id_aa64mmfr2_varange[] = {
1262 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1263 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1264 MRS_FIELD_VALUE_END,
1265 };
1266
1267 static const struct mrs_field_value id_aa64mmfr2_iesb[] = {
1268 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1269 MRS_FIELD_VALUE_END,
1270 };
1271
1272 static const struct mrs_field_value id_aa64mmfr2_lsm[] = {
1273 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1274 MRS_FIELD_VALUE_END,
1275 };
1276
1277 static const struct mrs_field_value id_aa64mmfr2_uao[] = {
1278 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1279 MRS_FIELD_VALUE_END,
1280 };
1281
1282 static const struct mrs_field_value id_aa64mmfr2_cnp[] = {
1283 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1284 MRS_FIELD_VALUE_END,
1285 };
1286
1287 static const struct mrs_field id_aa64mmfr2_fields[] = {
1288 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1289 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1290 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1291 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1292 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1293 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1294 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1295 id_aa64mmfr2_at_caps),
1296 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1297 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1298 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1299 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1300 id_aa64mmfr2_varange),
1301 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1302 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1303 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1304 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1305 MRS_FIELD_END,
1306 };
1307
1308
1309 /* ID_AA64MMFR2_EL1 */
1310 static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = {
1311 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL),
1312 MRS_FIELD_VALUE_END,
1313 };
1314
1315 static const struct mrs_field_value id_aa64mmfr3_mec[] = {
1316 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL),
1317 MRS_FIELD_VALUE_END,
1318 };
1319
1320 static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = {
1321 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL),
1322 MRS_FIELD_VALUE_END,
1323 };
1324
1325 static const struct mrs_field_value id_aa64mmfr3_tcrx[] = {
1326 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL),
1327 MRS_FIELD_VALUE_END,
1328 };
1329
1330 static const struct mrs_field id_aa64mmfr3_fields[] = {
1331 MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_EXACT,
1332 id_aa64mmfr3_spec_fpacc),
1333 MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_EXACT, id_aa64mmfr3_mec),
1334 MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_EXACT, id_aa64mmfr3_sctlrx),
1335 MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_EXACT, id_aa64mmfr3_tcrx),
1336 MRS_FIELD_END,
1337 };
1338
1339
1340 /* ID_AA64MMFR4_EL1 */
1341 static const struct mrs_field id_aa64mmfr4_fields[] = {
1342 MRS_FIELD_END,
1343 };
1344
1345
1346 /* ID_AA64PFR0_EL1 */
1347 static const struct mrs_field_value id_aa64pfr0_csv3[] = {
1348 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1349 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1350 MRS_FIELD_VALUE_END,
1351 };
1352
1353 static const struct mrs_field_value id_aa64pfr0_csv2[] = {
1354 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1355 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1356 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"),
1357 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"),
1358 MRS_FIELD_VALUE_END,
1359 };
1360
1361 static const struct mrs_field_value id_aa64pfr0_rme[] = {
1362 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL),
1363 MRS_FIELD_VALUE_END,
1364 };
1365
1366 static const struct mrs_field_value id_aa64pfr0_dit[] = {
1367 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1368 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1369 MRS_FIELD_VALUE_END,
1370 };
1371
1372 static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1373 MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1374 MRS_HWCAP_END
1375 };
1376
1377 static const struct mrs_field_value id_aa64pfr0_amu[] = {
1378 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1379 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1380 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"),
1381 MRS_FIELD_VALUE_END,
1382 };
1383
1384 static const struct mrs_field_value id_aa64pfr0_mpam[] = {
1385 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1386 MRS_FIELD_VALUE_END,
1387 };
1388
1389 static const struct mrs_field_value id_aa64pfr0_sel2[] = {
1390 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1391 MRS_FIELD_VALUE_END,
1392 };
1393
1394 static const struct mrs_field_value id_aa64pfr0_sve[] = {
1395 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1396 MRS_FIELD_VALUE_END,
1397 };
1398
1399 static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1400 MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1401 MRS_HWCAP_END
1402 };
1403
1404 static const struct mrs_field_value id_aa64pfr0_ras[] = {
1405 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1406 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1407 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1408 MRS_FIELD_VALUE_END,
1409 };
1410
1411 static const struct mrs_field_value id_aa64pfr0_gic[] = {
1412 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1413 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1414 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1415 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1416 MRS_FIELD_VALUE_END,
1417 };
1418
1419 static const struct mrs_field_value id_aa64pfr0_advsimd[] = {
1420 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1421 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1422 MRS_FIELD_VALUE_END,
1423 };
1424
1425 static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1426 MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1427 MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1428 MRS_HWCAP_END
1429 };
1430
1431 static const struct mrs_field_value id_aa64pfr0_fp[] = {
1432 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1433 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1434 MRS_FIELD_VALUE_END,
1435 };
1436
1437 static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1438 MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1439 MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1440 MRS_HWCAP_END
1441 };
1442
1443 static const struct mrs_field_value id_aa64pfr0_el3[] = {
1444 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1445 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1446 MRS_FIELD_VALUE_END,
1447 };
1448
1449 static const struct mrs_field_value id_aa64pfr0_el2[] = {
1450 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1451 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1452 MRS_FIELD_VALUE_END,
1453 };
1454
1455 static const struct mrs_field_value id_aa64pfr0_el1[] = {
1456 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1457 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1458 MRS_FIELD_VALUE_END,
1459 };
1460
1461 static const struct mrs_field_value id_aa64pfr0_el0[] = {
1462 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1463 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1464 MRS_FIELD_VALUE_END,
1465 };
1466
1467 static const struct mrs_field id_aa64pfr0_fields[] = {
1468 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1469 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1470 MRS_FIELD(ID_AA64PFR0, RME, false, MRS_EXACT, id_aa64pfr0_rme),
1471 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1472 id_aa64pfr0_dit_caps),
1473 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1474 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1475 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1476 MRS_FIELD_HWCAP_SPLIT(ID_AA64PFR0, SVE, false, MRS_LOWER, MRS_EXACT,
1477 id_aa64pfr0_sve, id_aa64pfr0_sve_caps),
1478 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1479 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1480 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1481 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1482 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, id_aa64pfr0_fp,
1483 id_aa64pfr0_fp_caps),
1484 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1485 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1486 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1487 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1488 MRS_FIELD_END,
1489 };
1490
1491
1492 /* ID_AA64PFR1_EL1 */
1493 static const struct mrs_field_value id_aa64pfr1_nmi[] = {
1494 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL),
1495 MRS_FIELD_VALUE_END,
1496 };
1497
1498 static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = {
1499 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""),
1500 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"),
1501 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"),
1502 MRS_FIELD_VALUE_END,
1503 };
1504
1505 static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = {
1506 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL),
1507 MRS_FIELD_VALUE_END,
1508 };
1509
1510 static const struct mrs_field_value id_aa64pfr1_sme[] = {
1511 MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""),
1512 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"),
1513 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"),
1514 MRS_FIELD_VALUE_END,
1515 };
1516
1517 static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = {
1518 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""),
1519 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"),
1520 MRS_FIELD_VALUE_END,
1521 };
1522
1523 static const struct mrs_field_value id_aa64pfr1_ras_frac[] = {
1524 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""),
1525 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"),
1526 MRS_FIELD_VALUE_END,
1527 };
1528
1529 static const struct mrs_field_value id_aa64pfr1_mte[] = {
1530 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1531 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"),
1532 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"),
1533 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"),
1534 MRS_FIELD_VALUE_END,
1535 };
1536
1537 static const struct mrs_field_value id_aa64pfr1_ssbs[] = {
1538 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1539 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1540 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1541 MRS_FIELD_VALUE_END,
1542 };
1543
1544 static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1545 MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1546 MRS_HWCAP_END
1547 };
1548
1549 static const struct mrs_field_value id_aa64pfr1_bt[] = {
1550 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1551 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1552 MRS_FIELD_VALUE_END,
1553 };
1554
1555 #if 0
1556 /* Enable when we add BTI support */
1557 static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1558 MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1559 MRS_HWCAP_END
1560 };
1561 #endif
1562
1563 static const struct mrs_field id_aa64pfr1_fields[] = {
1564 MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_EXACT, id_aa64pfr1_nmi),
1565 MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_EXACT,
1566 id_aa64pfr1_csv2_frac),
1567 MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_EXACT,
1568 id_aa64pfr1_rndr_trap),
1569 MRS_FIELD(ID_AA64PFR1, SME, false, MRS_EXACT, id_aa64pfr1_sme),
1570 MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_EXACT,
1571 id_aa64pfr1_mpam_frac),
1572 MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_EXACT,
1573 id_aa64pfr1_ras_frac),
1574 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1575 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1576 id_aa64pfr1_ssbs_caps),
1577 MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
1578 MRS_FIELD_END,
1579 };
1580
1581
1582 /* ID_AA64PFR2_EL1 */
1583 static const struct mrs_field id_aa64pfr2_fields[] = {
1584 MRS_FIELD_END,
1585 };
1586
1587
1588 /* ID_AA64ZFR0_EL1 */
1589 static const struct mrs_field_value id_aa64zfr0_f64mm[] = {
1590 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1591 MRS_FIELD_VALUE_END,
1592 };
1593
1594 static const struct mrs_field_hwcap id_aa64zfr0_f64mm_caps[] = {
1595 MRS_HWCAP(2, HWCAP2_SVEF64MM, ID_AA64ZFR0_F64MM_IMPL),
1596 MRS_HWCAP_END,
1597 };
1598
1599 static const struct mrs_field_value id_aa64zfr0_f32mm[] = {
1600 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1601 MRS_FIELD_VALUE_END,
1602 };
1603
1604 static const struct mrs_field_hwcap id_aa64zfr0_f32mm_caps[] = {
1605 MRS_HWCAP(2, HWCAP2_SVEF32MM, ID_AA64ZFR0_F32MM_IMPL),
1606 MRS_HWCAP_END,
1607 };
1608
1609 static const struct mrs_field_value id_aa64zfr0_i8mm[] = {
1610 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1611 MRS_FIELD_VALUE_END,
1612 };
1613
1614 static const struct mrs_field_hwcap id_aa64zfr0_i8mm_caps[] = {
1615 MRS_HWCAP(2, HWCAP2_SVEI8MM, ID_AA64ZFR0_I8MM_IMPL),
1616 MRS_HWCAP_END,
1617 };
1618
1619 static const struct mrs_field_value id_aa64zfr0_sm4[] = {
1620 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1621 MRS_FIELD_VALUE_END,
1622 };
1623
1624 static const struct mrs_field_hwcap id_aa64zfr0_sm4_caps[] = {
1625 MRS_HWCAP(2, HWCAP2_SVESM4, ID_AA64ZFR0_SM4_IMPL),
1626 MRS_HWCAP_END,
1627 };
1628
1629 static const struct mrs_field_value id_aa64zfr0_sha3[] = {
1630 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1631 MRS_FIELD_VALUE_END,
1632 };
1633
1634 static const struct mrs_field_hwcap id_aa64zfr0_sha3_caps[] = {
1635 MRS_HWCAP(2, HWCAP2_SVESHA3, ID_AA64ZFR0_SHA3_IMPL),
1636 MRS_HWCAP_END,
1637 };
1638
1639 static const struct mrs_field_value id_aa64zfr0_bf16[] = {
1640 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1641 MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1642 MRS_FIELD_VALUE_END,
1643 };
1644
1645 static const struct mrs_field_hwcap id_aa64zfr0_bf16_caps[] = {
1646 MRS_HWCAP(2, HWCAP2_SVEBF16, ID_AA64ZFR0_BF16_BASE),
1647 MRS_HWCAP(2, HWCAP2_SVE_EBF16, ID_AA64ZFR0_BF16_EBF),
1648 MRS_HWCAP_END,
1649 };
1650
1651 static const struct mrs_field_value id_aa64zfr0_bitperm[] = {
1652 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1653 MRS_FIELD_VALUE_END,
1654 };
1655
1656 static const struct mrs_field_hwcap id_aa64zfr0_bitperm_caps[] = {
1657 MRS_HWCAP(2, HWCAP2_SVEBITPERM, ID_AA64ZFR0_BitPerm_IMPL),
1658 MRS_HWCAP_END,
1659 };
1660
1661 static const struct mrs_field_value id_aa64zfr0_aes[] = {
1662 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1663 MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1664 MRS_FIELD_VALUE_END,
1665 };
1666
1667 static const struct mrs_field_hwcap id_aa64zfr0_aes_caps[] = {
1668 MRS_HWCAP(2, HWCAP2_SVEAES, ID_AA64ZFR0_AES_BASE),
1669 MRS_HWCAP(2, HWCAP2_SVEPMULL, ID_AA64ZFR0_AES_PMULL),
1670 MRS_HWCAP_END,
1671 };
1672
1673 static const struct mrs_field_value id_aa64zfr0_svever[] = {
1674 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1675 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1676 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2P1, "SVE2P1"),
1677 MRS_FIELD_VALUE_END,
1678 };
1679
1680 static const struct mrs_field_hwcap id_aa64zfr0_svever_caps[] = {
1681 MRS_HWCAP(2, HWCAP2_SVE2, ID_AA64ZFR0_SVEver_SVE2),
1682 MRS_HWCAP(2, HWCAP2_SVE2P1, ID_AA64ZFR0_SVEver_SVE2P1),
1683 MRS_HWCAP_END,
1684 };
1685
1686 static const struct mrs_field id_aa64zfr0_fields[] = {
1687 MRS_FIELD_HWCAP(ID_AA64ZFR0, F64MM, false, MRS_LOWER,
1688 id_aa64zfr0_f64mm, id_aa64zfr0_f64mm_caps),
1689 MRS_FIELD_HWCAP(ID_AA64ZFR0, F32MM, false, MRS_LOWER,
1690 id_aa64zfr0_f32mm, id_aa64zfr0_f32mm_caps),
1691 MRS_FIELD_HWCAP(ID_AA64ZFR0, I8MM, false, MRS_LOWER, id_aa64zfr0_i8mm,
1692 id_aa64zfr0_i8mm_caps),
1693 MRS_FIELD_HWCAP(ID_AA64ZFR0, SM4, false, MRS_LOWER, id_aa64zfr0_sm4,
1694 id_aa64zfr0_sm4_caps),
1695 MRS_FIELD_HWCAP(ID_AA64ZFR0, SHA3, false, MRS_LOWER, id_aa64zfr0_sha3,
1696 id_aa64zfr0_sha3_caps),
1697 MRS_FIELD_HWCAP(ID_AA64ZFR0, BF16, false, MRS_LOWER, id_aa64zfr0_bf16,
1698 id_aa64zfr0_bf16_caps),
1699 MRS_FIELD_HWCAP(ID_AA64ZFR0, BitPerm, false, MRS_LOWER,
1700 id_aa64zfr0_bitperm, id_aa64zfr0_bitperm_caps),
1701 MRS_FIELD_HWCAP(ID_AA64ZFR0, AES, false, MRS_LOWER, id_aa64zfr0_aes,
1702 id_aa64zfr0_aes_caps),
1703 MRS_FIELD_HWCAP(ID_AA64ZFR0, SVEver, false, MRS_LOWER,
1704 id_aa64zfr0_svever, id_aa64zfr0_svever_caps),
1705 MRS_FIELD_END,
1706 };
1707
1708
1709 #ifdef COMPAT_FREEBSD32
1710 /* ID_ISAR5_EL1 */
1711 static const struct mrs_field_value id_isar5_vcma[] = {
1712 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1713 MRS_FIELD_VALUE_END,
1714 };
1715
1716 static const struct mrs_field_value id_isar5_rdm[] = {
1717 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1718 MRS_FIELD_VALUE_END,
1719 };
1720
1721 static const struct mrs_field_value id_isar5_crc32[] = {
1722 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1723 MRS_FIELD_VALUE_END,
1724 };
1725
1726 static const struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1727 MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1728 MRS_HWCAP_END
1729 };
1730
1731 static const struct mrs_field_value id_isar5_sha2[] = {
1732 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1733 MRS_FIELD_VALUE_END,
1734 };
1735
1736 static const struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1737 MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1738 MRS_HWCAP_END
1739 };
1740
1741 static const struct mrs_field_value id_isar5_sha1[] = {
1742 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1743 MRS_FIELD_VALUE_END,
1744 };
1745
1746 static const struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1747 MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1748 MRS_HWCAP_END
1749 };
1750
1751 static const struct mrs_field_value id_isar5_aes[] = {
1752 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1753 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1754 MRS_FIELD_VALUE_END,
1755 };
1756
1757 static const struct mrs_field_hwcap id_isar5_aes_caps[] = {
1758 MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1759 MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1760 MRS_HWCAP_END
1761 };
1762
1763 static const struct mrs_field_value id_isar5_sevl[] = {
1764 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1765 MRS_FIELD_VALUE_END,
1766 };
1767
1768 static const struct mrs_field id_isar5_fields[] = {
1769 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1770 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1771 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1772 id_isar5_crc32_caps),
1773 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1774 id_isar5_sha2_caps),
1775 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1776 id_isar5_sha1_caps),
1777 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1778 id_isar5_aes_caps),
1779 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1780 MRS_FIELD_END,
1781 };
1782
1783 /* MVFR0 */
1784 static const struct mrs_field_value mvfr0_fpround[] = {
1785 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1786 MRS_FIELD_VALUE_END,
1787 };
1788
1789 static const struct mrs_field_value mvfr0_fpsqrt[] = {
1790 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1791 MRS_FIELD_VALUE_END,
1792 };
1793
1794 static const struct mrs_field_value mvfr0_fpdivide[] = {
1795 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1796 MRS_FIELD_VALUE_END,
1797 };
1798
1799 static const struct mrs_field_value mvfr0_fptrap[] = {
1800 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1801 MRS_FIELD_VALUE_END,
1802 };
1803
1804 static const struct mrs_field_value mvfr0_fpdp[] = {
1805 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1806 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1807 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1808 MRS_FIELD_VALUE_END,
1809 };
1810
1811 static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1812 MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1813 MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1814 MRS_HWCAP_END
1815 };
1816
1817 static const struct mrs_field_value mvfr0_fpsp[] = {
1818 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1819 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1820 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1821 MRS_FIELD_VALUE_END,
1822 };
1823
1824 static const struct mrs_field_value mvfr0_simdreg[] = {
1825 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1826 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1827 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1828 MRS_FIELD_VALUE_END,
1829 };
1830
1831 static const struct mrs_field mvfr0_fields[] = {
1832 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1833 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1834 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1835 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1836 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1837 mvfr0_fpdp_caps),
1838 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1839 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1840 MRS_FIELD_END,
1841 };
1842
1843 /* MVFR1 */
1844 static const struct mrs_field_value mvfr1_simdfmac[] = {
1845 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1846 MRS_FIELD_VALUE_END,
1847 };
1848
1849 static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1850 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1851 MRS_HWCAP_END
1852 };
1853
1854 static const struct mrs_field_value mvfr1_fphp[] = {
1855 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1856 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1857 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1858 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1859 MRS_FIELD_VALUE_END,
1860 };
1861
1862 static const struct mrs_field_value mvfr1_simdhp[] = {
1863 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1864 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1865 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1866 MRS_FIELD_VALUE_END,
1867 };
1868
1869 static const struct mrs_field_value mvfr1_simdsp[] = {
1870 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1871 MRS_FIELD_VALUE_END,
1872 };
1873
1874 static const struct mrs_field_value mvfr1_simdint[] = {
1875 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1876 MRS_FIELD_VALUE_END,
1877 };
1878
1879 static const struct mrs_field_value mvfr1_simdls[] = {
1880 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1881 MRS_FIELD_VALUE_END,
1882 };
1883
1884 static const struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1885 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1886 MRS_HWCAP_END
1887 };
1888
1889 static const struct mrs_field_value mvfr1_fpdnan[] = {
1890 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1891 MRS_FIELD_VALUE_END,
1892 };
1893
1894 static const struct mrs_field_value mvfr1_fpftz[] = {
1895 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1896 MRS_FIELD_VALUE_END,
1897 };
1898
1899 static const struct mrs_field mvfr1_fields[] = {
1900 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1901 mvfr1_simdfmac_caps),
1902 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1903 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1904 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1905 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1906 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1907 mvfr1_simdls_caps),
1908 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1909 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1910 MRS_FIELD_END,
1911 };
1912 #endif /* COMPAT_FREEBSD32 */
1913
1914 struct mrs_user_reg {
1915 u_int reg;
1916 u_int CRm;
1917 u_int Op2;
1918 bool is64bit;
1919 size_t offset;
1920 const struct mrs_field *fields;
1921 };
1922
1923 #define USER_REG(name, field_name, _is64bit) \
1924 { \
1925 .reg = name, \
1926 .CRm = name##_CRm, \
1927 .Op2 = name##_op2, \
1928 .offset = __offsetof(struct cpu_desc, field_name), \
1929 .fields = field_name##_fields, \
1930 .is64bit = _is64bit, \
1931 }
1932 static const struct mrs_user_reg user_regs[] = {
1933 USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true),
1934 USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true),
1935
1936 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true),
1937 USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true),
1938
1939 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true),
1940 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true),
1941 USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true),
1942
1943 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true),
1944 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true),
1945 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true),
1946 USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true),
1947 USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
1948
1949 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
1950 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
1951 USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
1952
1953 USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
1954
1955 #ifdef COMPAT_FREEBSD32
1956 USER_REG(ID_ISAR5_EL1, id_isar5, false),
1957
1958 USER_REG(MVFR0_EL1, mvfr0, false),
1959 USER_REG(MVFR1_EL1, mvfr1, false),
1960 #endif /* COMPAT_FREEBSD32 */
1961 };
1962
1963 #define CPU_DESC_FIELD(desc, idx) \
1964 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1965
1966 static int
user_mrs_handler(vm_offset_t va,uint32_t insn,struct trapframe * frame,uint32_t esr)1967 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1968 uint32_t esr)
1969 {
1970 uint64_t value;
1971 int CRm, Op2, i, reg;
1972
1973 if ((insn & MRS_MASK) != MRS_VALUE)
1974 return (0);
1975
1976 /*
1977 * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1978 * These are in the EL1 CPU identification space.
1979 * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1980 * CRm == {4-7} holds the ID_AA64 registers.
1981 *
1982 * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1983 * Table D9-2 System instruction encodings for non-Debug System
1984 * register accesses.
1985 */
1986 if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1987 return (0);
1988
1989 CRm = mrs_CRm(insn);
1990 if (CRm > 7 || (CRm < 4 && CRm != 0))
1991 return (0);
1992
1993 Op2 = mrs_Op2(insn);
1994 value = 0;
1995
1996 for (i = 0; i < nitems(user_regs); i++) {
1997 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1998 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
1999 value = CPU_DESC_FIELD(user_cpu_desc, i);
2000 else
2001 value = CPU_DESC_FIELD(l_user_cpu_desc, i);
2002 break;
2003 }
2004 }
2005
2006 if (CRm == 0) {
2007 switch (Op2) {
2008 case 0:
2009 value = READ_SPECIALREG(midr_el1);
2010 break;
2011 case 5:
2012 value = READ_SPECIALREG(mpidr_el1);
2013 break;
2014 case 6:
2015 value = READ_SPECIALREG(revidr_el1);
2016 break;
2017 default:
2018 return (0);
2019 }
2020 }
2021
2022 /*
2023 * We will handle this instruction, move to the next so we
2024 * don't trap here again.
2025 */
2026 frame->tf_elr += INSN_SIZE;
2027
2028 reg = MRS_REGISTER(insn);
2029 /* If reg is 31 then write to xzr, i.e. do nothing */
2030 if (reg == 31)
2031 return (1);
2032
2033 if (reg < nitems(frame->tf_x))
2034 frame->tf_x[reg] = value;
2035 else if (reg == 30)
2036 frame->tf_lr = value;
2037
2038 return (1);
2039 }
2040
2041 /*
2042 * Compares two field values that may be signed or unsigned.
2043 * Returns:
2044 * < 0 when a is less than b
2045 * = 0 when a equals b
2046 * > 0 when a is greater than b
2047 */
2048 static int
mrs_field_cmp(uint64_t a,uint64_t b,u_int shift,int width,bool sign)2049 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
2050 {
2051 uint64_t mask;
2052
2053 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
2054 width));
2055
2056 mask = (1ul << width) - 1;
2057 /* Move the field to the lower bits */
2058 a = (a >> shift) & mask;
2059 b = (b >> shift) & mask;
2060
2061 if (sign) {
2062 /*
2063 * The field is signed. Toggle the upper bit so the comparison
2064 * works on unsigned values as this makes positive numbers,
2065 * i.e. those with a 0 bit, larger than negative numbers,
2066 * i.e. those with a 1 bit, in an unsigned comparison.
2067 */
2068 a ^= 1ul << (width - 1);
2069 b ^= 1ul << (width - 1);
2070 }
2071
2072 return (a - b);
2073 }
2074
2075 static uint64_t
update_lower_register(uint64_t val,uint64_t new_val,u_int shift,int width,bool sign)2076 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
2077 int width, bool sign)
2078 {
2079 uint64_t mask;
2080
2081 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
2082 width));
2083
2084 /*
2085 * If the new value is less than the existing value update it.
2086 */
2087 if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
2088 mask = (1ul << width) - 1;
2089 val &= ~(mask << shift);
2090 val |= new_val & (mask << shift);
2091 }
2092
2093 return (val);
2094 }
2095
2096 bool
extract_user_id_field(u_int reg,u_int field_shift,uint8_t * val)2097 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
2098 {
2099 uint64_t value;
2100 int i;
2101
2102 for (i = 0; i < nitems(user_regs); i++) {
2103 if (user_regs[i].reg == reg) {
2104 value = CPU_DESC_FIELD(user_cpu_desc, i);
2105 *val = value >> field_shift;
2106 return (true);
2107 }
2108 }
2109
2110 return (false);
2111 }
2112
2113 bool
get_kernel_reg(u_int reg,uint64_t * val)2114 get_kernel_reg(u_int reg, uint64_t *val)
2115 {
2116 int i;
2117
2118 for (i = 0; i < nitems(user_regs); i++) {
2119 if (user_regs[i].reg == reg) {
2120 *val = CPU_DESC_FIELD(kern_cpu_desc, i);
2121 return (true);
2122 }
2123 }
2124
2125 return (false);
2126 }
2127
2128 /*
2129 * Fetch the specified register's value, ensuring that individual field values
2130 * do not exceed those in the mask.
2131 */
2132 bool
get_kernel_reg_masked(u_int reg,uint64_t * valp,uint64_t mask)2133 get_kernel_reg_masked(u_int reg, uint64_t *valp, uint64_t mask)
2134 {
2135 const struct mrs_field *fields;
2136 uint64_t val;
2137
2138 for (int i = 0; i < nitems(user_regs); i++) {
2139 if (user_regs[i].reg == reg) {
2140 val = CPU_DESC_FIELD(kern_cpu_desc, i);
2141 fields = user_regs[i].fields;
2142 for (int j = 0; fields[j].type != 0; j++) {
2143 mask = update_lower_register(mask, val,
2144 fields[j].shift, 4, fields[j].sign);
2145 }
2146 *valp = mask;
2147 return (true);
2148 }
2149 }
2150
2151 return (false);
2152 }
2153
2154 static uint64_t
update_special_reg_field(uint64_t user_reg,u_int type,uint64_t value,u_int shift,bool sign)2155 update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value,
2156 u_int shift, bool sign)
2157 {
2158 switch (type & MRS_TYPE_MASK) {
2159 case MRS_EXACT:
2160 user_reg &= ~(0xful << shift);
2161 user_reg |= (uint64_t)MRS_EXACT_FIELD(type) << shift;
2162 break;
2163 case MRS_LOWER:
2164 user_reg = update_lower_register(user_reg, value, shift, 4,
2165 sign);
2166 break;
2167 default:
2168 panic("Invalid field type: %d", type);
2169 }
2170
2171 return (user_reg);
2172 }
2173
2174 void
update_special_regs(u_int cpu)2175 update_special_regs(u_int cpu)
2176 {
2177 struct cpu_desc *desc;
2178 const struct mrs_field *fields;
2179 uint64_t l_user_reg, user_reg, kern_reg, value;
2180 int i, j;
2181
2182 if (cpu == 0) {
2183 /* Create a user visible cpu description with safe values */
2184 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
2185 /* Safe values for these registers */
2186 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
2187 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
2188 ID_AA64PFR0_EL0_64;
2189 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
2190 /* Create the Linux user visible cpu description */
2191 memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc));
2192 }
2193
2194 desc = get_cpu_desc(cpu);
2195 for (i = 0; i < nitems(user_regs); i++) {
2196 value = CPU_DESC_FIELD(*desc, i);
2197 if (cpu == 0) {
2198 kern_reg = value;
2199 user_reg = value;
2200 l_user_reg = value;
2201 } else {
2202 kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
2203 user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
2204 l_user_reg = CPU_DESC_FIELD(l_user_cpu_desc, i);
2205 }
2206
2207 fields = user_regs[i].fields;
2208 for (j = 0; fields[j].type != 0; j++) {
2209 /* Update the FreeBSD userspace ID register view */
2210 user_reg = update_special_reg_field(user_reg,
2211 fields[j].type >> MRS_TYPE_FBSD_SHIFT, value,
2212 fields[j].shift, fields[j].sign);
2213
2214 /* Update the Linux userspace ID register view */
2215 l_user_reg = update_special_reg_field(l_user_reg,
2216 fields[j].type >> MRS_TYPE_LNX_SHIFT, value,
2217 fields[j].shift, fields[j].sign);
2218
2219 /* Update the kernel ID register view */
2220 kern_reg = update_lower_register(kern_reg, value,
2221 fields[j].shift, 4, fields[j].sign);
2222 }
2223
2224 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
2225 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
2226 CPU_DESC_FIELD(l_user_cpu_desc, i) = l_user_reg;
2227 }
2228 }
2229
2230 void
cpu_desc_init(void)2231 cpu_desc_init(void)
2232 {
2233 if (mp_ncpus == 1)
2234 return;
2235
2236 /*
2237 * Allocate memory for the non-boot CPUs to store their registers.
2238 * As this is indexed by CPU ID we need to allocate space for CPUs
2239 * 1 to mp_maxid. Because of this mp_maxid is already the correct
2240 * number of elements.
2241 */
2242 cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU,
2243 M_ZERO | M_WAITOK);
2244 }
2245
2246 /* HWCAP */
2247 bool __read_frequently lse_supported = false;
2248
2249 bool __read_frequently icache_aliasing = false;
2250 bool __read_frequently icache_vmid = false;
2251
2252 int64_t dcache_line_size; /* The minimum D cache line size */
2253 int64_t icache_line_size; /* The minimum I cache line size */
2254 int64_t idcache_line_size; /* The minimum cache line size */
2255
2256 /*
2257 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
2258 */
2259 static void
parse_cpu_features(bool is64bit,struct cpu_desc * cpu_desc,u_long * hwcap,u_long * hwcap2)2260 parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap,
2261 u_long *hwcap2)
2262 {
2263 const struct mrs_field_hwcap *hwcaps;
2264 const struct mrs_field *fields;
2265 uint64_t min, reg;
2266 u_long *cur_hwcap;
2267 int i, j, k;
2268
2269 for (i = 0; i < nitems(user_regs); i++) {
2270 if (user_regs[i].is64bit != is64bit)
2271 continue;
2272
2273 reg = CPU_DESC_FIELD(*cpu_desc, i);
2274 fields = user_regs[i].fields;
2275 for (j = 0; fields[j].type != 0; j++) {
2276 hwcaps = fields[j].hwcaps;
2277 if (hwcaps == NULL)
2278 continue;
2279
2280 for (k = 0; hwcaps[k].hwcap_id != 0; k++) {
2281 KASSERT(hwcaps[k].hwcap_id == 1 ||
2282 hwcaps[k].hwcap_id == 2,
2283 ("%s: Invalid HWCAP ID %d", __func__,
2284 hwcaps[k].hwcap_id));
2285
2286 cur_hwcap = hwcaps[k].hwcap_id == 1 ?
2287 hwcap : hwcap2;
2288 min = hwcaps[k].min;
2289
2290 /*
2291 * If the field is greater than the minimum
2292 * value we can set the hwcap;
2293 */
2294 if (mrs_field_cmp(reg, min, fields[j].shift,
2295 4, fields[j].sign) >= 0) {
2296 *cur_hwcap |= hwcaps[k].hwcap_val;
2297 }
2298 }
2299 }
2300 }
2301 }
2302
2303 static void
identify_cpu_sysinit(void * dummy __unused)2304 identify_cpu_sysinit(void *dummy __unused)
2305 {
2306 struct cpu_desc *desc, *prev_desc;
2307 int cpu;
2308 bool dic, idc;
2309
2310 dic = (allow_dic != 0);
2311 idc = (allow_idc != 0);
2312
2313 prev_desc = NULL;
2314 CPU_FOREACH(cpu) {
2315 desc = get_cpu_desc(cpu);
2316 if (cpu != 0) {
2317 check_cpu_regs(cpu, desc, prev_desc);
2318 update_special_regs(cpu);
2319 }
2320
2321 if (CTR_DIC_VAL(desc->ctr) == 0)
2322 dic = false;
2323 if (CTR_IDC_VAL(desc->ctr) == 0)
2324 idc = false;
2325 prev_desc = desc;
2326 }
2327
2328 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
2329 parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2);
2330 parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap,
2331 &linux_elf_hwcap2);
2332 #ifdef COMPAT_FREEBSD32
2333 parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2);
2334 #endif
2335
2336 /* We export the CPUID registers */
2337 elf_hwcap |= HWCAP_CPUID;
2338 linux_elf_hwcap |= HWCAP_CPUID;
2339
2340 #ifdef COMPAT_FREEBSD32
2341 /* Set the default caps and any that need to check multiple fields */
2342 elf32_hwcap |= parse_cpu_features_hwcap32();
2343 #endif
2344
2345 if (dic && idc) {
2346 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
2347 if (bootverbose)
2348 printf("Enabling DIC & IDC ICache sync\n");
2349 } else if (idc) {
2350 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
2351 if (bootverbose)
2352 printf("Enabling IDC ICache sync\n");
2353 }
2354
2355 if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
2356 lse_supported = true;
2357 if (bootverbose)
2358 printf("Enabling LSE atomics in the kernel\n");
2359 }
2360 #ifdef LSE_ATOMICS
2361 if (!lse_supported)
2362 panic("CPU does not support LSE atomic instructions");
2363 #endif
2364
2365 install_undef_handler(true, user_mrs_handler);
2366 }
2367 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
2368
2369 static void
cpu_features_sysinit(void * dummy __unused)2370 cpu_features_sysinit(void *dummy __unused)
2371 {
2372 struct sbuf sb;
2373 struct cpu_desc *desc, *prev_desc;
2374 u_int cpu;
2375
2376 prev_desc = NULL;
2377 CPU_FOREACH(cpu) {
2378 desc = get_cpu_desc(cpu);
2379 print_cpu_features(cpu, desc, prev_desc);
2380 prev_desc = desc;
2381 }
2382
2383 /* Fill in cpu_model for the hw.model sysctl */
2384 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
2385 print_cpu_midr(&sb, 0);
2386
2387 sbuf_finish(&sb);
2388 sbuf_delete(&sb);
2389
2390 free(cpu_desc, M_IDENTCPU);
2391 }
2392 /* Log features before APs are released and start printing to the dmesg. */
2393 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
2394
2395 static void
tcr_set_e0pd1(void * arg __unused)2396 tcr_set_e0pd1(void *arg __unused)
2397 {
2398 uint64_t tcr;
2399
2400 tcr = READ_SPECIALREG(tcr_el1);
2401 tcr |= TCR_E0PD1;
2402 WRITE_SPECIALREG(tcr_el1, tcr);
2403 isb();
2404 }
2405
2406 /* Enable support for more recent architecture features */
2407 static void
cpu_feat_support(void * arg __unused)2408 cpu_feat_support(void *arg __unused)
2409 {
2410 /*
2411 * If FEAT_E0PD is supported use it to cause faults without a page
2412 * table walk if userspace tries to access kernel memory.
2413 */
2414 if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) !=
2415 ID_AA64MMFR2_E0PD_NONE)
2416 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL);
2417 }
2418 SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL);
2419
2420 #ifdef COMPAT_FREEBSD32
2421 static u_long
parse_cpu_features_hwcap32(void)2422 parse_cpu_features_hwcap32(void)
2423 {
2424 u_long hwcap = HWCAP32_DEFAULT;
2425
2426 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
2427 MVFR1_SIMDLS_IMPL) &&
2428 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
2429 MVFR1_SIMDInt_IMPL) &&
2430 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
2431 MVFR1_SIMDSP_IMPL))
2432 hwcap |= HWCAP32_NEON;
2433
2434 return (hwcap);
2435 }
2436 #endif /* COMPAT_FREEBSD32 */
2437
2438 static void
print_ctr_fields(struct sbuf * sb,uint64_t reg,const void * arg __unused)2439 print_ctr_fields(struct sbuf *sb, uint64_t reg, const void *arg __unused)
2440 {
2441
2442 sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
2443 sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
2444 reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
2445
2446 switch(CTR_L1IP_VAL(reg)) {
2447 case CTR_L1IP_VPIPT:
2448 sbuf_printf(sb, "VPIPT");
2449 break;
2450 case CTR_L1IP_AIVIVT:
2451 sbuf_printf(sb, "AIVIVT");
2452 break;
2453 case CTR_L1IP_VIPT:
2454 sbuf_printf(sb, "VIPT");
2455 break;
2456 case CTR_L1IP_PIPT:
2457 sbuf_printf(sb, "PIPT");
2458 break;
2459 }
2460 sbuf_printf(sb, " ICache,");
2461 reg &= ~CTR_L1IP_MASK;
2462
2463 sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
2464 sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
2465 reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
2466
2467 if (CTR_IDC_VAL(reg) != 0)
2468 sbuf_printf(sb, ",IDC");
2469 if (CTR_DIC_VAL(reg) != 0)
2470 sbuf_printf(sb, ",DIC");
2471 reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
2472 reg &= ~CTR_RES1;
2473
2474 if (reg != 0)
2475 sbuf_printf(sb, ",%lx", reg);
2476 }
2477
2478 static void
print_register(struct sbuf * sb,const char * reg_name,uint64_t reg,void (* print_fields)(struct sbuf *,uint64_t,const void *),const void * arg)2479 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2480 void (*print_fields)(struct sbuf *, uint64_t, const void *),
2481 const void *arg)
2482 {
2483
2484 sbuf_printf(sb, "%29s = <", reg_name);
2485
2486 print_fields(sb, reg, arg);
2487
2488 sbuf_finish(sb);
2489 printf("%s>\n", sbuf_data(sb));
2490 sbuf_clear(sb);
2491 }
2492
2493 static void
print_id_fields(struct sbuf * sb,uint64_t reg,const void * arg)2494 print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg)
2495 {
2496 const struct mrs_field *fields = arg;
2497 const struct mrs_field_value *fv;
2498 int field, i, j, printed;
2499
2500 #define SEP_STR ((printed++) == 0) ? "" : ","
2501 printed = 0;
2502 for (i = 0; fields[i].type != 0; i++) {
2503 fv = fields[i].values;
2504
2505 /* TODO: Handle with an unknown message */
2506 if (fv == NULL)
2507 continue;
2508
2509 field = (reg & fields[i].mask) >> fields[i].shift;
2510 for (j = 0; fv[j].desc != NULL; j++) {
2511 if ((fv[j].value >> fields[i].shift) != field)
2512 continue;
2513
2514 if (fv[j].desc[0] != '\0')
2515 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2516 break;
2517 }
2518 if (fv[j].desc == NULL)
2519 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2520 fields[i].name, field);
2521
2522 reg &= ~(0xful << fields[i].shift);
2523 }
2524
2525 if (reg != 0)
2526 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2527 #undef SEP_STR
2528 }
2529
2530 static void
print_id_register(struct sbuf * sb,const char * reg_name,uint64_t reg,const struct mrs_field * fields)2531 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2532 const struct mrs_field *fields)
2533 {
2534
2535 print_register(sb, reg_name, reg, print_id_fields, fields);
2536 }
2537
2538 static void
print_cpu_midr(struct sbuf * sb,u_int cpu)2539 print_cpu_midr(struct sbuf *sb, u_int cpu)
2540 {
2541 const struct cpu_parts *cpu_partsp;
2542 const char *cpu_impl_name;
2543 const char *cpu_part_name;
2544 u_int midr;
2545 u_int impl_id;
2546 u_int part_id;
2547
2548 midr = pcpu_find(cpu)->pc_midr;
2549
2550 cpu_impl_name = NULL;
2551 cpu_partsp = NULL;
2552 impl_id = CPU_IMPL(midr);
2553 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2554 if (impl_id == cpu_implementers[i].impl_id) {
2555 cpu_impl_name = cpu_implementers[i].impl_name;
2556 cpu_partsp = cpu_implementers[i].cpu_parts;
2557 break;
2558 }
2559 }
2560 /* Unknown implementer, so unknown part */
2561 if (cpu_impl_name == NULL) {
2562 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2563 return;
2564 }
2565
2566 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2567 __func__, cpu_impl_name));
2568
2569 cpu_part_name = NULL;
2570 part_id = CPU_PART(midr);
2571 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2572 if (part_id == cpu_partsp[i].part_id) {
2573 cpu_part_name = cpu_partsp[i].part_name;
2574 break;
2575 }
2576 }
2577 /* Known Implementer, Unknown part */
2578 if (cpu_part_name == NULL) {
2579 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2580 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2581 return;
2582 }
2583
2584 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2585 cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2586 }
2587
2588 static void
print_cpu_cache(struct cpu_desc * desc,struct sbuf * sb,uint64_t ccs,bool icache,bool unified)2589 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs,
2590 bool icache, bool unified)
2591 {
2592 size_t cache_size;
2593 size_t line_size;
2594
2595 /* LineSize is Log2(S) - 4. */
2596 line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2597 /*
2598 * Calculate cache size (sets * ways * line size). There are different
2599 * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2600 * register.
2601 */
2602 if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2603 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2604 (CCSIDR_ASSOC_64(ccs) + 1);
2605 else
2606 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2607
2608 cache_size *= line_size;
2609 sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
2610 icache ? "instruction" : unified ? "unified" : "data");
2611 }
2612
2613 static void
print_cpu_caches(struct sbuf * sb,struct cpu_desc * desc)2614 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc)
2615 {
2616 /* Print out each cache combination */
2617 uint64_t clidr;
2618 int i = 1;
2619 clidr = desc->clidr;
2620
2621 for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2622 int j = 0;
2623 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
2624
2625 sbuf_printf(sb, " L%d cache: ", i + 1);
2626 if ((clidr & CLIDR_CTYPE_IO)) {
2627 print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true,
2628 false);
2629 /* If there's more, add to the line. */
2630 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
2631 sbuf_printf(sb, ", ");
2632 }
2633 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
2634 print_cpu_cache(desc, sb, desc->ccsidr[i][j], false,
2635 (clidr & CLIDR_CTYPE_UNIFIED));
2636 }
2637 sbuf_printf(sb, "\n");
2638
2639 }
2640 sbuf_finish(sb);
2641 printf("%s", sbuf_data(sb));
2642 }
2643
2644 static void
print_cpu_features(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)2645 print_cpu_features(u_int cpu, struct cpu_desc *desc,
2646 struct cpu_desc *prev_desc)
2647 {
2648 struct sbuf *sb;
2649
2650 sb = sbuf_new_auto();
2651 sbuf_printf(sb, "CPU%3u: ", cpu);
2652 print_cpu_midr(sb, cpu);
2653
2654 sbuf_cat(sb, " affinity:");
2655 switch(cpu_aff_levels) {
2656 default:
2657 case 4:
2658 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr));
2659 /* FALLTHROUGH */
2660 case 3:
2661 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr));
2662 /* FALLTHROUGH */
2663 case 2:
2664 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr));
2665 /* FALLTHROUGH */
2666 case 1:
2667 case 0: /* On UP this will be zero */
2668 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr));
2669 break;
2670 }
2671 sbuf_finish(sb);
2672 printf("%s\n", sbuf_data(sb));
2673 sbuf_clear(sb);
2674
2675 /*
2676 * There is a hardware errata where, if one CPU is performing a TLB
2677 * invalidation while another is performing a store-exclusive the
2678 * store-exclusive may return the wrong status. A workaround seems
2679 * to be to use an IPI to invalidate on each CPU, however given the
2680 * limited number of affected units (pass 1.1 is the evaluation
2681 * hardware revision), and the lack of information from Cavium
2682 * this has not been implemented.
2683 *
2684 * At the time of writing this the only information is from:
2685 * https://lkml.org/lkml/2016/8/4/722
2686 */
2687 /*
2688 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
2689 * triggers on pass 2.0+.
2690 */
2691 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
2692 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
2693 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
2694 "hardware bugs that may cause the incorrect operation of "
2695 "atomic operations.\n");
2696
2697 #define SHOULD_PRINT_REG(_reg) \
2698 (prev_desc == NULL || desc->_reg != prev_desc->_reg)
2699
2700 /* Cache Type Register */
2701 if (SHOULD_PRINT_REG(ctr)) {
2702 print_register(sb, "Cache Type",
2703 desc->ctr, print_ctr_fields, NULL);
2704 }
2705
2706 /* AArch64 Instruction Set Attribute Register 0 */
2707 if (SHOULD_PRINT_REG(id_aa64isar0))
2708 print_id_register(sb, "Instruction Set Attributes 0",
2709 desc->id_aa64isar0, id_aa64isar0_fields);
2710
2711 /* AArch64 Instruction Set Attribute Register 1 */
2712 if (SHOULD_PRINT_REG(id_aa64isar1))
2713 print_id_register(sb, "Instruction Set Attributes 1",
2714 desc->id_aa64isar1, id_aa64isar1_fields);
2715
2716 /* AArch64 Instruction Set Attribute Register 2 */
2717 if (SHOULD_PRINT_REG(id_aa64isar2))
2718 print_id_register(sb, "Instruction Set Attributes 2",
2719 desc->id_aa64isar2, id_aa64isar2_fields);
2720
2721 /* AArch64 Processor Feature Register 0 */
2722 if (SHOULD_PRINT_REG(id_aa64pfr0))
2723 print_id_register(sb, "Processor Features 0",
2724 desc->id_aa64pfr0, id_aa64pfr0_fields);
2725
2726 /* AArch64 Processor Feature Register 1 */
2727 if (SHOULD_PRINT_REG(id_aa64pfr1))
2728 print_id_register(sb, "Processor Features 1",
2729 desc->id_aa64pfr1, id_aa64pfr1_fields);
2730
2731 /* AArch64 Processor Feature Register 2 */
2732 if (SHOULD_PRINT_REG(id_aa64pfr2))
2733 print_id_register(sb, "Processor Features 2",
2734 desc->id_aa64pfr2, id_aa64pfr2_fields);
2735
2736 /* AArch64 Memory Model Feature Register 0 */
2737 if (SHOULD_PRINT_REG(id_aa64mmfr0))
2738 print_id_register(sb, "Memory Model Features 0",
2739 desc->id_aa64mmfr0, id_aa64mmfr0_fields);
2740
2741 /* AArch64 Memory Model Feature Register 1 */
2742 if (SHOULD_PRINT_REG(id_aa64mmfr1))
2743 print_id_register(sb, "Memory Model Features 1",
2744 desc->id_aa64mmfr1, id_aa64mmfr1_fields);
2745
2746 /* AArch64 Memory Model Feature Register 2 */
2747 if (SHOULD_PRINT_REG(id_aa64mmfr2))
2748 print_id_register(sb, "Memory Model Features 2",
2749 desc->id_aa64mmfr2, id_aa64mmfr2_fields);
2750
2751 /* AArch64 Memory Model Feature Register 3 */
2752 if (SHOULD_PRINT_REG(id_aa64mmfr3))
2753 print_id_register(sb, "Memory Model Features 3",
2754 desc->id_aa64mmfr3, id_aa64mmfr3_fields);
2755
2756 /* AArch64 Memory Model Feature Register 4 */
2757 if (SHOULD_PRINT_REG(id_aa64mmfr4))
2758 print_id_register(sb, "Memory Model Features 4",
2759 desc->id_aa64mmfr4, id_aa64mmfr4_fields);
2760
2761 /* AArch64 Debug Feature Register 0 */
2762 if (SHOULD_PRINT_REG(id_aa64dfr0))
2763 print_id_register(sb, "Debug Features 0",
2764 desc->id_aa64dfr0, id_aa64dfr0_fields);
2765
2766 /* AArch64 Memory Model Feature Register 1 */
2767 if (SHOULD_PRINT_REG(id_aa64dfr1))
2768 print_id_register(sb, "Debug Features 1",
2769 desc->id_aa64dfr1, id_aa64dfr1_fields);
2770
2771 /* AArch64 Auxiliary Feature Register 0 */
2772 if (SHOULD_PRINT_REG(id_aa64afr0))
2773 print_id_register(sb, "Auxiliary Features 0",
2774 desc->id_aa64afr0, id_aa64afr0_fields);
2775
2776 /* AArch64 Auxiliary Feature Register 1 */
2777 if (SHOULD_PRINT_REG(id_aa64afr1))
2778 print_id_register(sb, "Auxiliary Features 1",
2779 desc->id_aa64afr1, id_aa64afr1_fields);
2780
2781 /* AArch64 SVE Feature Register 0 */
2782 if (desc->have_sve) {
2783 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
2784 !prev_desc->have_sve) {
2785 print_id_register(sb, "SVE Features 0",
2786 desc->id_aa64zfr0, id_aa64zfr0_fields);
2787 }
2788 }
2789
2790 #ifdef COMPAT_FREEBSD32
2791 /* AArch32 Instruction Set Attribute Register 5 */
2792 if (SHOULD_PRINT_REG(id_isar5))
2793 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2794 desc->id_isar5, id_isar5_fields);
2795
2796 /* AArch32 Media and VFP Feature Register 0 */
2797 if (SHOULD_PRINT_REG(mvfr0))
2798 print_id_register(sb, "AArch32 Media and VFP Features 0",
2799 desc->mvfr0, mvfr0_fields);
2800
2801 /* AArch32 Media and VFP Feature Register 1 */
2802 if (SHOULD_PRINT_REG(mvfr1))
2803 print_id_register(sb, "AArch32 Media and VFP Features 1",
2804 desc->mvfr1, mvfr1_fields);
2805 #endif
2806 if (bootverbose)
2807 print_cpu_caches(sb, desc);
2808
2809 sbuf_delete(sb);
2810 sb = NULL;
2811 #undef SHOULD_PRINT_REG
2812 #undef SEP_STR
2813 }
2814
2815 void
identify_cache(uint64_t ctr)2816 identify_cache(uint64_t ctr)
2817 {
2818
2819 /* Identify the L1 cache type */
2820 switch (CTR_L1IP_VAL(ctr)) {
2821 case CTR_L1IP_PIPT:
2822 break;
2823 case CTR_L1IP_VPIPT:
2824 icache_vmid = true;
2825 break;
2826 default:
2827 case CTR_L1IP_VIPT:
2828 icache_aliasing = true;
2829 break;
2830 }
2831
2832 if (dcache_line_size == 0) {
2833 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2834 __func__, icache_line_size));
2835
2836 /* Get the D cache line size */
2837 dcache_line_size = CTR_DLINE_SIZE(ctr);
2838 /* And the same for the I cache */
2839 icache_line_size = CTR_ILINE_SIZE(ctr);
2840
2841 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2842 }
2843
2844 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2845 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2846 dcache_line_size, CTR_DLINE_SIZE(ctr));
2847 }
2848
2849 if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2850 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2851 icache_line_size, CTR_ILINE_SIZE(ctr));
2852 }
2853 }
2854
2855 void
identify_cpu(u_int cpu)2856 identify_cpu(u_int cpu)
2857 {
2858 struct cpu_desc *desc;
2859 uint64_t clidr;
2860
2861 desc = get_cpu_desc(cpu);
2862 /* Save affinity for current CPU */
2863 desc->mpidr = get_mpidr();
2864 CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK;
2865
2866 desc->ctr = READ_SPECIALREG(ctr_el0);
2867 desc->id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1_REG);
2868 desc->id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1_REG);
2869 desc->id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1_REG);
2870 desc->id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1_REG);
2871 desc->id_aa64isar2 = READ_SPECIALREG(ID_AA64ISAR2_EL1_REG);
2872 desc->id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1_REG);
2873 desc->id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1_REG);
2874 desc->id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1_REG);
2875 desc->id_aa64mmfr3 = READ_SPECIALREG(ID_AA64MMFR3_EL1_REG);
2876 desc->id_aa64mmfr4 = READ_SPECIALREG(ID_AA64MMFR4_EL1_REG);
2877 desc->id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1_REG);
2878 desc->id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1_REG);
2879 desc->id_aa64pfr2 = READ_SPECIALREG(ID_AA64PFR2_EL1_REG);
2880
2881 /*
2882 * ID_AA64ZFR0_EL1 is only valid when at least one of:
2883 * - ID_AA64PFR0_EL1.SVE is non-zero
2884 * - ID_AA64PFR1_EL1.SME is non-zero
2885 * In other cases it is zero, but still safe to read
2886 */
2887 desc->have_sve =
2888 (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0);
2889 desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
2890
2891 desc->clidr = READ_SPECIALREG(clidr_el1);
2892
2893 clidr = desc->clidr;
2894
2895 for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2896 int j = 0;
2897 if ((clidr & CLIDR_CTYPE_IO)) {
2898 WRITE_SPECIALREG(csselr_el1,
2899 CSSELR_Level(i) | CSSELR_InD);
2900 desc->ccsidr[i][j++] =
2901 READ_SPECIALREG(ccsidr_el1);
2902 }
2903 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
2904 continue;
2905 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
2906 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
2907 }
2908
2909 #ifdef COMPAT_FREEBSD32
2910 /* Only read aarch32 SRs if EL0-32 is available */
2911 if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) {
2912 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1);
2913 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1);
2914 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1);
2915 }
2916 #endif
2917 }
2918
2919 static void
check_cpu_regs(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)2920 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc)
2921 {
2922 switch (cpu_aff_levels) {
2923 case 0:
2924 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr))
2925 cpu_aff_levels = 1;
2926 /* FALLTHROUGH */
2927 case 1:
2928 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr))
2929 cpu_aff_levels = 2;
2930 /* FALLTHROUGH */
2931 case 2:
2932 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr))
2933 cpu_aff_levels = 3;
2934 /* FALLTHROUGH */
2935 case 3:
2936 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
2937 cpu_aff_levels = 4;
2938 break;
2939 }
2940
2941 if (desc->ctr != prev_desc->ctr) {
2942 /*
2943 * If the cache type register is different we may
2944 * have a different l1 cache type.
2945 */
2946 identify_cache(desc->ctr);
2947 }
2948 }
2949