1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Semihalf
7 * under sponsorship of the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include <sys/param.h>
36 #include <sys/kernel.h>
37 #include <sys/pcpu.h>
38 #include <sys/sbuf.h>
39 #include <sys/smp.h>
40 #include <sys/sysctl.h>
41 #include <sys/systm.h>
42
43 #include <machine/atomic.h>
44 #include <machine/cpu.h>
45 #include <machine/cpufunc.h>
46 #include <machine/elf.h>
47 #include <machine/md_var.h>
48 #include <machine/undefined.h>
49
50 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
51 static void print_cpu_features(u_int cpu);
52 #ifdef COMPAT_FREEBSD32
53 static u_long parse_cpu_features_hwcap32(void);
54 #endif
55
56 char machine[] = "arm64";
57
58 #ifdef SCTL_MASK32
59 extern int adaptive_machine_arch;
60 #endif
61
62 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
63 "Cache management tuning");
64
65 static int allow_dic = 1;
66 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
67 "Allow optimizations based on the DIC cache bit");
68
69 static int allow_idc = 1;
70 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
71 "Allow optimizations based on the IDC cache bit");
72
73 static void check_cpu_regs(u_int cpu);
74
75 /*
76 * The default implementation of I-cache sync assumes we have an
77 * aliasing cache until we know otherwise.
78 */
79 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
80 &arm64_aliasing_icache_sync_range;
81
82 static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)83 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
84 {
85 #ifdef SCTL_MASK32
86 static const char machine32[] = "arm";
87 #endif
88 int error;
89
90 #ifdef SCTL_MASK32
91 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
92 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
93 else
94 #endif
95 error = SYSCTL_OUT(req, machine, sizeof(machine));
96 return (error);
97 }
98
99 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
100 CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", "Machine class");
101
102 static char cpu_model[64];
103 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
104 cpu_model, sizeof(cpu_model), "Machine model");
105
106 /*
107 * Per-CPU affinity as provided in MPIDR_EL1
108 * Indexed by CPU number in logical order selected by the system.
109 * Relevant fields can be extracted using CPU_AFFn macros,
110 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
111 *
112 * Fields used by us:
113 * Aff1 - Cluster number
114 * Aff0 - CPU number in Aff1 cluster
115 */
116 uint64_t __cpu_affinity[MAXCPU];
117 static u_int cpu_aff_levels;
118
119 struct cpu_desc {
120 uint64_t mpidr;
121 uint64_t id_aa64afr0;
122 uint64_t id_aa64afr1;
123 uint64_t id_aa64dfr0;
124 uint64_t id_aa64dfr1;
125 uint64_t id_aa64isar0;
126 uint64_t id_aa64isar1;
127 uint64_t id_aa64mmfr0;
128 uint64_t id_aa64mmfr1;
129 uint64_t id_aa64mmfr2;
130 uint64_t id_aa64pfr0;
131 uint64_t id_aa64pfr1;
132 uint64_t ctr;
133 #ifdef COMPAT_FREEBSD32
134 uint64_t id_isar5;
135 uint64_t mvfr0;
136 uint64_t mvfr1;
137 #endif
138 };
139
140 static struct cpu_desc cpu_desc[MAXCPU];
141 static struct cpu_desc kern_cpu_desc;
142 static struct cpu_desc user_cpu_desc;
143 static u_int cpu_print_regs;
144 #define PRINT_ID_AA64_AFR0 0x00000001
145 #define PRINT_ID_AA64_AFR1 0x00000002
146 #define PRINT_ID_AA64_DFR0 0x00000010
147 #define PRINT_ID_AA64_DFR1 0x00000020
148 #define PRINT_ID_AA64_ISAR0 0x00000100
149 #define PRINT_ID_AA64_ISAR1 0x00000200
150 #define PRINT_ID_AA64_MMFR0 0x00001000
151 #define PRINT_ID_AA64_MMFR1 0x00002000
152 #define PRINT_ID_AA64_MMFR2 0x00004000
153 #define PRINT_ID_AA64_PFR0 0x00010000
154 #define PRINT_ID_AA64_PFR1 0x00020000
155 #ifdef COMPAT_FREEBSD32
156 #define PRINT_ID_ISAR5 0x01000000
157 #define PRINT_MVFR0 0x02000000
158 #define PRINT_MVFR1 0x04000000
159 #endif
160 #define PRINT_CTR_EL0 0x10000000
161
162 struct cpu_parts {
163 u_int part_id;
164 const char *part_name;
165 };
166 #define CPU_PART_NONE { 0, NULL }
167
168 struct cpu_implementers {
169 u_int impl_id;
170 const char *impl_name;
171 /*
172 * Part number is implementation defined
173 * so each vendor will have its own set of values and names.
174 */
175 const struct cpu_parts *cpu_parts;
176 };
177 #define CPU_IMPLEMENTER_NONE { 0, NULL, NULL }
178
179 /*
180 * Per-implementer table of (PartNum, CPU Name) pairs.
181 */
182 /* ARM Ltd. */
183 static const struct cpu_parts cpu_parts_arm[] = {
184 { CPU_PART_AEM_V8, "AEMv8" },
185 { CPU_PART_FOUNDATION, "Foundation-Model" },
186 { CPU_PART_CORTEX_A35, "Cortex-A35" },
187 { CPU_PART_CORTEX_A53, "Cortex-A53" },
188 { CPU_PART_CORTEX_A55, "Cortex-A55" },
189 { CPU_PART_CORTEX_A57, "Cortex-A57" },
190 { CPU_PART_CORTEX_A65, "Cortex-A65" },
191 { CPU_PART_CORTEX_A72, "Cortex-A72" },
192 { CPU_PART_CORTEX_A73, "Cortex-A73" },
193 { CPU_PART_CORTEX_A75, "Cortex-A75" },
194 { CPU_PART_CORTEX_A76, "Cortex-A76" },
195 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
196 { CPU_PART_CORTEX_A77, "Cortex-A77" },
197 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
198 CPU_PART_NONE,
199 };
200
201 /* Cavium */
202 static const struct cpu_parts cpu_parts_cavium[] = {
203 { CPU_PART_THUNDERX, "ThunderX" },
204 { CPU_PART_THUNDERX2, "ThunderX2" },
205 CPU_PART_NONE,
206 };
207
208 /* APM / Ampere */
209 static const struct cpu_parts cpu_parts_apm[] = {
210 { CPU_PART_EMAG8180, "eMAG 8180" },
211 CPU_PART_NONE,
212 };
213
214 /* Unknown */
215 static const struct cpu_parts cpu_parts_none[] = {
216 CPU_PART_NONE,
217 };
218
219 /*
220 * Implementers table.
221 */
222 const struct cpu_implementers cpu_implementers[] = {
223 { CPU_IMPL_APPLE, "Apple", cpu_parts_none },
224 { CPU_IMPL_APM, "APM", cpu_parts_apm },
225 { CPU_IMPL_ARM, "ARM", cpu_parts_arm },
226 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none },
227 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium },
228 { CPU_IMPL_DEC, "DEC", cpu_parts_none },
229 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none },
230 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none },
231 { CPU_IMPL_INTEL, "Intel", cpu_parts_none },
232 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none },
233 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none },
234 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_none },
235 CPU_IMPLEMENTER_NONE,
236 };
237
238 #define MRS_TYPE_MASK 0xf
239 #define MRS_INVALID 0
240 #define MRS_EXACT 1
241 #define MRS_EXACT_VAL(x) (MRS_EXACT | ((x) << 4))
242 #define MRS_EXACT_FIELD(x) ((x) >> 4)
243 #define MRS_LOWER 2
244
245 struct mrs_field_value {
246 uint64_t value;
247 const char *desc;
248 };
249
250 #define MRS_FIELD_VALUE(_value, _desc) \
251 { \
252 .value = (_value), \
253 .desc = (_desc), \
254 }
255
256 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \
257 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \
258 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
259
260 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \
261 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
262 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
263 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
264 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
265 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
266 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
267 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
268 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
269 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
270 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
271 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
272 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
273 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
274 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
275 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
276 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
277
278 #define MRS_FIELD_VALUE_END { .desc = NULL }
279
280 struct mrs_field_hwcap {
281 u_long *hwcap;
282 uint64_t min;
283 u_long hwcap_val;
284 };
285
286 #define MRS_HWCAP(_hwcap, _val, _min) \
287 { \
288 .hwcap = (_hwcap), \
289 .hwcap_val = (_val), \
290 .min = (_min), \
291 }
292
293 #define MRS_HWCAP_END { .hwcap = NULL }
294
295 struct mrs_field {
296 const char *name;
297 struct mrs_field_value *values;
298 struct mrs_field_hwcap *hwcaps;
299 uint64_t mask;
300 bool sign;
301 u_int type;
302 u_int shift;
303 };
304
305 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
306 { \
307 .name = #_name, \
308 .sign = (_sign), \
309 .type = (_type), \
310 .shift = _register ## _ ## _name ## _SHIFT, \
311 .mask = _register ## _ ## _name ## _MASK, \
312 .values = (_values), \
313 .hwcaps = (_hwcap), \
314 }
315
316 #define MRS_FIELD(_register, _name, _sign, _type, _values) \
317 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
318
319 #define MRS_FIELD_END { .type = MRS_INVALID, }
320
321 /* ID_AA64AFR0_EL1 */
322 static struct mrs_field id_aa64afr0_fields[] = {
323 MRS_FIELD_END,
324 };
325
326
327 /* ID_AA64AFR1_EL1 */
328 static struct mrs_field id_aa64afr1_fields[] = {
329 MRS_FIELD_END,
330 };
331
332
333 /* ID_AA64DFR0_EL1 */
334 static struct mrs_field_value id_aa64dfr0_tracefilt[] = {
335 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
336 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
337 MRS_FIELD_VALUE_END,
338 };
339
340 static struct mrs_field_value id_aa64dfr0_doublelock[] = {
341 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
342 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
343 MRS_FIELD_VALUE_END,
344 };
345
346 static struct mrs_field_value id_aa64dfr0_pmsver[] = {
347 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
348 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
349 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_8_3, "SPE v8.3"),
350 MRS_FIELD_VALUE_END,
351 };
352
353 static struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
354 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
355 MRS_FIELD_VALUE_END,
356 };
357
358 static struct mrs_field_value id_aa64dfr0_wrps[] = {
359 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
360 MRS_FIELD_VALUE_END,
361 };
362
363 static struct mrs_field_value id_aa64dfr0_brps[] = {
364 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
365 MRS_FIELD_VALUE_END,
366 };
367
368 static struct mrs_field_value id_aa64dfr0_pmuver[] = {
369 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
370 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
371 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3 v8.1"),
372 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3 v8.4"),
373 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3 v8.5"),
374 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
375 MRS_FIELD_VALUE_END,
376 };
377
378 static struct mrs_field_value id_aa64dfr0_tracever[] = {
379 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
380 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
381 MRS_FIELD_VALUE_END,
382 };
383
384 static struct mrs_field_value id_aa64dfr0_debugver[] = {
385 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
386 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
387 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8.2"),
388 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8.4"),
389 MRS_FIELD_VALUE_END,
390 };
391
392 static struct mrs_field id_aa64dfr0_fields[] = {
393 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
394 id_aa64dfr0_tracefilt),
395 MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
396 id_aa64dfr0_doublelock),
397 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
398 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
399 id_aa64dfr0_ctx_cmps),
400 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
401 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
402 MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
403 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
404 id_aa64dfr0_tracever),
405 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
406 id_aa64dfr0_debugver),
407 MRS_FIELD_END,
408 };
409
410
411 /* ID_AA64DFR1_EL1 */
412 static struct mrs_field id_aa64dfr1_fields[] = {
413 MRS_FIELD_END,
414 };
415
416
417 /* ID_AA64ISAR0_EL1 */
418 static struct mrs_field_value id_aa64isar0_rndr[] = {
419 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
420 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
421 MRS_FIELD_VALUE_END,
422 };
423
424 static struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
425 MRS_HWCAP(&elf_hwcap2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
426 MRS_HWCAP_END
427 };
428
429 static struct mrs_field_value id_aa64isar0_tlb[] = {
430 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
431 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
432 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
433 MRS_FIELD_VALUE_END,
434 };
435
436 static struct mrs_field_value id_aa64isar0_ts[] = {
437 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
438 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
439 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
440 MRS_FIELD_VALUE_END,
441 };
442
443 static struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
444 MRS_HWCAP(&elf_hwcap, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
445 MRS_HWCAP(&elf_hwcap2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
446 MRS_HWCAP_END
447 };
448
449 static struct mrs_field_value id_aa64isar0_fhm[] = {
450 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
451 MRS_FIELD_VALUE_END,
452 };
453
454 static struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
455 MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
456 MRS_HWCAP_END
457 };
458
459 static struct mrs_field_value id_aa64isar0_dp[] = {
460 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
461 MRS_FIELD_VALUE_END,
462 };
463
464 static struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
465 MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
466 MRS_HWCAP_END
467 };
468
469 static struct mrs_field_value id_aa64isar0_sm4[] = {
470 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
471 MRS_FIELD_VALUE_END,
472 };
473
474 static struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
475 MRS_HWCAP(&elf_hwcap, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
476 MRS_HWCAP_END
477 };
478
479 static struct mrs_field_value id_aa64isar0_sm3[] = {
480 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
481 MRS_FIELD_VALUE_END,
482 };
483
484 static struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
485 MRS_HWCAP(&elf_hwcap, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
486 MRS_HWCAP_END
487 };
488
489 static struct mrs_field_value id_aa64isar0_sha3[] = {
490 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
491 MRS_FIELD_VALUE_END,
492 };
493
494 static struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
495 MRS_HWCAP(&elf_hwcap, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
496 MRS_HWCAP_END
497 };
498
499 static struct mrs_field_value id_aa64isar0_rdm[] = {
500 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
501 MRS_FIELD_VALUE_END,
502 };
503
504 static struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
505 MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
506 MRS_HWCAP_END
507 };
508
509 static struct mrs_field_value id_aa64isar0_atomic[] = {
510 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
511 MRS_FIELD_VALUE_END,
512 };
513
514 static struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
515 MRS_HWCAP(&elf_hwcap, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
516 MRS_HWCAP_END
517 };
518
519 static struct mrs_field_value id_aa64isar0_crc32[] = {
520 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
521 MRS_FIELD_VALUE_END,
522 };
523
524 static struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
525 MRS_HWCAP(&elf_hwcap, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
526 MRS_HWCAP_END
527 };
528
529 static struct mrs_field_value id_aa64isar0_sha2[] = {
530 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
531 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
532 MRS_FIELD_VALUE_END,
533 };
534
535 static struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
536 MRS_HWCAP(&elf_hwcap, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
537 MRS_HWCAP(&elf_hwcap, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
538 MRS_HWCAP_END
539 };
540
541 static struct mrs_field_value id_aa64isar0_sha1[] = {
542 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
543 MRS_FIELD_VALUE_END,
544 };
545
546 static struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
547 MRS_HWCAP(&elf_hwcap, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
548 MRS_HWCAP_END
549 };
550
551 static struct mrs_field_value id_aa64isar0_aes[] = {
552 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
553 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
554 MRS_FIELD_VALUE_END,
555 };
556
557 static struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
558 MRS_HWCAP(&elf_hwcap, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
559 MRS_HWCAP(&elf_hwcap, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
560 MRS_HWCAP_END
561 };
562
563 static struct mrs_field id_aa64isar0_fields[] = {
564 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
565 id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
566 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
567 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
568 id_aa64isar0_ts_caps),
569 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
570 id_aa64isar0_fhm_caps),
571 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
572 id_aa64isar0_dp_caps),
573 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
574 id_aa64isar0_sm4_caps),
575 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
576 id_aa64isar0_sm3_caps),
577 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
578 id_aa64isar0_sha3_caps),
579 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
580 id_aa64isar0_rdm_caps),
581 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
582 id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
583 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
584 id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
585 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
586 id_aa64isar0_sha2_caps),
587 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
588 id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
589 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
590 id_aa64isar0_aes_caps),
591 MRS_FIELD_END,
592 };
593
594
595 /* ID_AA64ISAR1_EL1 */
596 static struct mrs_field_value id_aa64isar1_i8mm[] = {
597 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
598 MRS_FIELD_VALUE_END,
599 };
600
601 static struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
602 MRS_HWCAP(&elf_hwcap2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
603 MRS_HWCAP_END
604 };
605
606 static struct mrs_field_value id_aa64isar1_dgh[] = {
607 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
608 MRS_FIELD_VALUE_END,
609 };
610
611 static struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
612 MRS_HWCAP(&elf_hwcap2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
613 MRS_HWCAP_END
614 };
615
616 static struct mrs_field_value id_aa64isar1_bf16[] = {
617 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
618 MRS_FIELD_VALUE_END,
619 };
620
621 static struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
622 MRS_HWCAP(&elf_hwcap2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
623 MRS_HWCAP_END
624 };
625
626 static struct mrs_field_value id_aa64isar1_specres[] = {
627 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
628 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
629 MRS_FIELD_VALUE_END,
630 };
631
632 static struct mrs_field_value id_aa64isar1_sb[] = {
633 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
634 MRS_FIELD_VALUE_END,
635 };
636
637 static struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
638 MRS_HWCAP(&elf_hwcap, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
639 MRS_HWCAP_END
640 };
641
642 static struct mrs_field_value id_aa64isar1_frintts[] = {
643 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
644 MRS_FIELD_VALUE_END,
645 };
646
647 static struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
648 MRS_HWCAP(&elf_hwcap2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
649 MRS_HWCAP_END
650 };
651
652 static struct mrs_field_value id_aa64isar1_gpi[] = {
653 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
654 MRS_FIELD_VALUE_END,
655 };
656
657 static struct mrs_field_value id_aa64isar1_gpa[] = {
658 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
659 MRS_FIELD_VALUE_END,
660 };
661
662 static struct mrs_field_value id_aa64isar1_lrcpc[] = {
663 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
664 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
665 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
666 MRS_FIELD_VALUE_END,
667 };
668
669 static struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
670 MRS_HWCAP(&elf_hwcap, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
671 MRS_HWCAP(&elf_hwcap, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
672 MRS_HWCAP_END
673 };
674
675 static struct mrs_field_value id_aa64isar1_fcma[] = {
676 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
677 MRS_FIELD_VALUE_END,
678 };
679
680 static struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
681 MRS_HWCAP(&elf_hwcap, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
682 MRS_HWCAP_END
683 };
684
685 static struct mrs_field_value id_aa64isar1_jscvt[] = {
686 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
687 MRS_FIELD_VALUE_END,
688 };
689
690 static struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
691 MRS_HWCAP(&elf_hwcap, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
692 MRS_HWCAP_END
693 };
694
695 static struct mrs_field_value id_aa64isar1_api[] = {
696 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
697 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
698 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
699 MRS_FIELD_VALUE_END,
700 };
701
702 static struct mrs_field_value id_aa64isar1_apa[] = {
703 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
704 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
705 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
706 MRS_FIELD_VALUE_END,
707 };
708
709 static struct mrs_field_value id_aa64isar1_dpb[] = {
710 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
711 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
712 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
713 MRS_FIELD_VALUE_END,
714 };
715
716 static struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
717 MRS_HWCAP(&elf_hwcap, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
718 MRS_HWCAP(&elf_hwcap2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
719 MRS_HWCAP_END
720 };
721
722 static struct mrs_field id_aa64isar1_fields[] = {
723 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
724 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
725 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
726 id_aa64isar1_dgh_caps),
727 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
728 id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
729 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
730 id_aa64isar1_specres),
731 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
732 id_aa64isar1_sb_caps),
733 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
734 id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
735 MRS_FIELD(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi),
736 MRS_FIELD(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa),
737 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
738 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
739 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
740 id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
741 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
742 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
743 MRS_FIELD(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api),
744 MRS_FIELD(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa),
745 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
746 id_aa64isar1_dpb_caps),
747 MRS_FIELD_END,
748 };
749
750
751 /* ID_AA64MMFR0_EL1 */
752 static struct mrs_field_value id_aa64mmfr0_exs[] = {
753 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
754 MRS_FIELD_VALUE_END,
755 };
756
757 static struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
758 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
759 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
760 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
761 MRS_FIELD_VALUE_END,
762 };
763
764 static struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
765 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
766 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
767 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
768 MRS_FIELD_VALUE_END,
769 };
770
771 static struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
772 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
773 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
774 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
775 MRS_FIELD_VALUE_END,
776 };
777
778 static struct mrs_field_value id_aa64mmfr0_tgran4[] = {
779 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4,NONE, IMPL),
780 MRS_FIELD_VALUE_END,
781 };
782
783 static struct mrs_field_value id_aa64mmfr0_tgran64[] = {
784 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
785 MRS_FIELD_VALUE_END,
786 };
787
788 static struct mrs_field_value id_aa64mmfr0_tgran16[] = {
789 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
790 MRS_FIELD_VALUE_END,
791 };
792
793 static struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
794 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
795 MRS_FIELD_VALUE_END,
796 };
797
798 static struct mrs_field_value id_aa64mmfr0_snsmem[] = {
799 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
800 MRS_FIELD_VALUE_END,
801 };
802
803 static struct mrs_field_value id_aa64mmfr0_bigend[] = {
804 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
805 MRS_FIELD_VALUE_END,
806 };
807
808 static struct mrs_field_value id_aa64mmfr0_asidbits[] = {
809 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
810 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
811 MRS_FIELD_VALUE_END,
812 };
813
814 static struct mrs_field_value id_aa64mmfr0_parange[] = {
815 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
816 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
817 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
818 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
819 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
820 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
821 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
822 MRS_FIELD_VALUE_END,
823 };
824
825 static struct mrs_field id_aa64mmfr0_fields[] = {
826 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
827 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
828 id_aa64mmfr0_tgran4_2),
829 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
830 id_aa64mmfr0_tgran64_2),
831 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
832 id_aa64mmfr0_tgran16_2),
833 MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
834 MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
835 id_aa64mmfr0_tgran64),
836 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
837 id_aa64mmfr0_tgran16),
838 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
839 id_aa64mmfr0_bigendel0),
840 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
841 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
842 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
843 id_aa64mmfr0_asidbits),
844 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
845 id_aa64mmfr0_parange),
846 MRS_FIELD_END,
847 };
848
849
850 /* ID_AA64MMFR1_EL1 */
851 static struct mrs_field_value id_aa64mmfr1_xnx[] = {
852 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
853 MRS_FIELD_VALUE_END,
854 };
855
856 static struct mrs_field_value id_aa64mmfr1_specsei[] = {
857 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
858 MRS_FIELD_VALUE_END,
859 };
860
861 static struct mrs_field_value id_aa64mmfr1_pan[] = {
862 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
863 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
864 MRS_FIELD_VALUE_END,
865 };
866
867 static struct mrs_field_value id_aa64mmfr1_lo[] = {
868 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
869 MRS_FIELD_VALUE_END,
870 };
871
872 static struct mrs_field_value id_aa64mmfr1_hpds[] = {
873 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
874 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
875 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
876 MRS_FIELD_VALUE_END,
877 };
878
879 static struct mrs_field_value id_aa64mmfr1_vh[] = {
880 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
881 MRS_FIELD_VALUE_END,
882 };
883
884 static struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
885 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
886 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
887 MRS_FIELD_VALUE_END,
888 };
889
890 static struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
891 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
892 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
893 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
894 MRS_FIELD_VALUE_END,
895 };
896
897 static struct mrs_field id_aa64mmfr1_fields[] = {
898 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
899 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
900 id_aa64mmfr1_specsei),
901 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
902 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
903 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
904 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
905 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
906 id_aa64mmfr1_vmidbits),
907 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
908 MRS_FIELD_END,
909 };
910
911
912 /* ID_AA64MMFR2_EL1 */
913 static struct mrs_field_value id_aa64mmfr2_e0pd[] = {
914 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
915 MRS_FIELD_VALUE_END,
916 };
917
918 static struct mrs_field_value id_aa64mmfr2_evt[] = {
919 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
920 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
921 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
922 MRS_FIELD_VALUE_END,
923 };
924
925 static struct mrs_field_value id_aa64mmfr2_bbm[] = {
926 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
927 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
928 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
929 MRS_FIELD_VALUE_END,
930 };
931
932 static struct mrs_field_value id_aa64mmfr2_ttl[] = {
933 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
934 MRS_FIELD_VALUE_END,
935 };
936
937 static struct mrs_field_value id_aa64mmfr2_fwb[] = {
938 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
939 MRS_FIELD_VALUE_END,
940 };
941
942 static struct mrs_field_value id_aa64mmfr2_ids[] = {
943 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
944 MRS_FIELD_VALUE_END,
945 };
946
947 static struct mrs_field_value id_aa64mmfr2_at[] = {
948 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
949 MRS_FIELD_VALUE_END,
950 };
951
952 static struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
953 MRS_HWCAP(&elf_hwcap, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
954 MRS_HWCAP_END
955 };
956
957 static struct mrs_field_value id_aa64mmfr2_st[] = {
958 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
959 MRS_FIELD_VALUE_END,
960 };
961
962 static struct mrs_field_value id_aa64mmfr2_nv[] = {
963 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
964 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
965 MRS_FIELD_VALUE_END,
966 };
967
968 static struct mrs_field_value id_aa64mmfr2_ccidx[] = {
969 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
970 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
971 MRS_FIELD_VALUE_END,
972 };
973
974 static struct mrs_field_value id_aa64mmfr2_varange[] = {
975 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
976 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
977 MRS_FIELD_VALUE_END,
978 };
979
980 static struct mrs_field_value id_aa64mmfr2_iesb[] = {
981 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
982 MRS_FIELD_VALUE_END,
983 };
984
985 static struct mrs_field_value id_aa64mmfr2_lsm[] = {
986 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
987 MRS_FIELD_VALUE_END,
988 };
989
990 static struct mrs_field_value id_aa64mmfr2_uao[] = {
991 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
992 MRS_FIELD_VALUE_END,
993 };
994
995 static struct mrs_field_value id_aa64mmfr2_cnp[] = {
996 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
997 MRS_FIELD_VALUE_END,
998 };
999
1000 static struct mrs_field id_aa64mmfr2_fields[] = {
1001 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1002 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1003 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1004 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1005 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1006 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1007 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1008 id_aa64mmfr2_at_caps),
1009 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1010 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1011 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1012 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1013 id_aa64mmfr2_varange),
1014 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1015 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1016 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1017 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1018 MRS_FIELD_END,
1019 };
1020
1021
1022 /* ID_AA64PFR0_EL1 */
1023 static struct mrs_field_value id_aa64pfr0_csv3[] = {
1024 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1025 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1026 MRS_FIELD_VALUE_END,
1027 };
1028
1029 static struct mrs_field_value id_aa64pfr0_csv2[] = {
1030 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1031 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1032 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "SCXTNUM"),
1033 MRS_FIELD_VALUE_END,
1034 };
1035
1036 static struct mrs_field_value id_aa64pfr0_dit[] = {
1037 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1038 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1039 MRS_FIELD_VALUE_END,
1040 };
1041
1042 static struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1043 MRS_HWCAP(&elf_hwcap, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1044 MRS_HWCAP_END
1045 };
1046
1047 static struct mrs_field_value id_aa64pfr0_amu[] = {
1048 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1049 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1050 MRS_FIELD_VALUE_END,
1051 };
1052
1053 static struct mrs_field_value id_aa64pfr0_mpam[] = {
1054 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1055 MRS_FIELD_VALUE_END,
1056 };
1057
1058 static struct mrs_field_value id_aa64pfr0_sel2[] = {
1059 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1060 MRS_FIELD_VALUE_END,
1061 };
1062
1063 static struct mrs_field_value id_aa64pfr0_sve[] = {
1064 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1065 MRS_FIELD_VALUE_END,
1066 };
1067
1068 #if 0
1069 /* Enable when we add SVE support */
1070 static struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1071 MRS_HWCAP(&elf_hwcap, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1072 MRS_HWCAP_END
1073 };
1074 #endif
1075
1076 static struct mrs_field_value id_aa64pfr0_ras[] = {
1077 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1078 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1079 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1080 MRS_FIELD_VALUE_END,
1081 };
1082
1083 static struct mrs_field_value id_aa64pfr0_gic[] = {
1084 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1085 MRS_FIELD_VALUE_END,
1086 };
1087
1088 static struct mrs_field_value id_aa64pfr0_advsimd[] = {
1089 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1090 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1091 MRS_FIELD_VALUE_END,
1092 };
1093
1094 static struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1095 MRS_HWCAP(&elf_hwcap, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1096 MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1097 MRS_HWCAP_END
1098 };
1099
1100 static struct mrs_field_value id_aa64pfr0_fp[] = {
1101 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1102 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1103 MRS_FIELD_VALUE_END,
1104 };
1105
1106 static struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1107 MRS_HWCAP(&elf_hwcap, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1108 MRS_HWCAP(&elf_hwcap, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1109 MRS_HWCAP_END
1110 };
1111
1112 static struct mrs_field_value id_aa64pfr0_el3[] = {
1113 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1114 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1115 MRS_FIELD_VALUE_END,
1116 };
1117
1118 static struct mrs_field_value id_aa64pfr0_el2[] = {
1119 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1120 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1121 MRS_FIELD_VALUE_END,
1122 };
1123
1124 static struct mrs_field_value id_aa64pfr0_el1[] = {
1125 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1126 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1127 MRS_FIELD_VALUE_END,
1128 };
1129
1130 static struct mrs_field_value id_aa64pfr0_el0[] = {
1131 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1132 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1133 MRS_FIELD_VALUE_END,
1134 };
1135
1136 static struct mrs_field id_aa64pfr0_fields[] = {
1137 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1138 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1139 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1140 id_aa64pfr0_dit_caps),
1141 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1142 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1143 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1144 MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
1145 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1146 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1147 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1148 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1149 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, id_aa64pfr0_fp,
1150 id_aa64pfr0_fp_caps),
1151 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1152 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1153 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1154 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1155 MRS_FIELD_END,
1156 };
1157
1158
1159 /* ID_AA64PFR1_EL1 */
1160 static struct mrs_field_value id_aa64pfr1_mte[] = {
1161 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1162 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL_EL0, "MTE EL0"),
1163 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL, "MTE"),
1164 MRS_FIELD_VALUE_END,
1165 };
1166
1167 static struct mrs_field_value id_aa64pfr1_ssbs[] = {
1168 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1169 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1170 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1171 MRS_FIELD_VALUE_END,
1172 };
1173
1174 static struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1175 MRS_HWCAP(&elf_hwcap, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1176 MRS_HWCAP_END
1177 };
1178
1179 static struct mrs_field_value id_aa64pfr1_bt[] = {
1180 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1181 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1182 MRS_FIELD_VALUE_END,
1183 };
1184
1185 #if 0
1186 /* Enable when we add BTI support */
1187 static struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1188 MRS_HWCAP(&elf_hwcap2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1189 MRS_HWCAP_END
1190 };
1191 #endif
1192
1193 static struct mrs_field id_aa64pfr1_fields[] = {
1194 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1195 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1196 id_aa64pfr1_ssbs_caps),
1197 MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
1198 MRS_FIELD_END,
1199 };
1200
1201 #ifdef COMPAT_FREEBSD32
1202 /* ID_ISAR5_EL1 */
1203 static struct mrs_field_value id_isar5_vcma[] = {
1204 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1205 MRS_FIELD_VALUE_END,
1206 };
1207
1208 static struct mrs_field_value id_isar5_rdm[] = {
1209 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1210 MRS_FIELD_VALUE_END,
1211 };
1212
1213 static struct mrs_field_value id_isar5_crc32[] = {
1214 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1215 MRS_FIELD_VALUE_END,
1216 };
1217
1218 static struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1219 MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1220 MRS_HWCAP_END
1221 };
1222
1223 static struct mrs_field_value id_isar5_sha2[] = {
1224 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1225 MRS_FIELD_VALUE_END,
1226 };
1227
1228 static struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1229 MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1230 MRS_HWCAP_END
1231 };
1232
1233 static struct mrs_field_value id_isar5_sha1[] = {
1234 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1235 MRS_FIELD_VALUE_END,
1236 };
1237
1238 static struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1239 MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1240 MRS_HWCAP_END
1241 };
1242
1243 static struct mrs_field_value id_isar5_aes[] = {
1244 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1245 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1246 MRS_FIELD_VALUE_END,
1247 };
1248
1249 static struct mrs_field_hwcap id_isar5_aes_caps[] = {
1250 MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1251 MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1252 MRS_HWCAP_END
1253 };
1254
1255 static struct mrs_field_value id_isar5_sevl[] = {
1256 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1257 MRS_FIELD_VALUE_END,
1258 };
1259
1260 static struct mrs_field id_isar5_fields[] = {
1261 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1262 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1263 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1264 id_isar5_crc32_caps),
1265 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1266 id_isar5_sha2_caps),
1267 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1268 id_isar5_sha1_caps),
1269 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1270 id_isar5_aes_caps),
1271 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1272 MRS_FIELD_END,
1273 };
1274
1275 /* MVFR0 */
1276 static struct mrs_field_value mvfr0_fpround[] = {
1277 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1278 MRS_FIELD_VALUE_END,
1279 };
1280
1281 static struct mrs_field_value mvfr0_fpsqrt[] = {
1282 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1283 MRS_FIELD_VALUE_END,
1284 };
1285
1286 static struct mrs_field_value mvfr0_fpdivide[] = {
1287 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1288 MRS_FIELD_VALUE_END,
1289 };
1290
1291 static struct mrs_field_value mvfr0_fptrap[] = {
1292 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1293 MRS_FIELD_VALUE_END,
1294 };
1295
1296 static struct mrs_field_value mvfr0_fpdp[] = {
1297 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1298 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1299 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1300 MRS_FIELD_VALUE_END,
1301 };
1302
1303 static struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1304 MRS_HWCAP(&elf32_hwcap, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1305 MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1306 };
1307
1308 static struct mrs_field_value mvfr0_fpsp[] = {
1309 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1310 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1311 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1312 MRS_FIELD_VALUE_END,
1313 };
1314
1315 static struct mrs_field_value mvfr0_simdreg[] = {
1316 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1317 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1318 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1319 MRS_FIELD_VALUE_END,
1320 };
1321
1322 static struct mrs_field mvfr0_fields[] = {
1323 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1324 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1325 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1326 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1327 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1328 mvfr0_fpdp_caps),
1329 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1330 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1331 MRS_FIELD_END,
1332 };
1333
1334 /* MVFR1 */
1335 static struct mrs_field_value mvfr1_simdfmac[] = {
1336 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1337 MRS_FIELD_VALUE_END,
1338 };
1339
1340 static struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1341 MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1342 MRS_HWCAP_END
1343 };
1344
1345 static struct mrs_field_value mvfr1_fphp[] = {
1346 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1347 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1348 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1349 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1350 MRS_FIELD_VALUE_END,
1351 };
1352
1353 static struct mrs_field_value mvfr1_simdhp[] = {
1354 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1355 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1356 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1357 MRS_FIELD_VALUE_END,
1358 };
1359
1360 static struct mrs_field_value mvfr1_simdsp[] = {
1361 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1362 MRS_FIELD_VALUE_END,
1363 };
1364
1365 static struct mrs_field_value mvfr1_simdint[] = {
1366 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1367 MRS_FIELD_VALUE_END,
1368 };
1369
1370 static struct mrs_field_value mvfr1_simdls[] = {
1371 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1372 MRS_FIELD_VALUE_END,
1373 };
1374
1375 static struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1376 MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1377 MRS_HWCAP_END
1378 };
1379
1380 static struct mrs_field_value mvfr1_fpdnan[] = {
1381 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1382 MRS_FIELD_VALUE_END,
1383 };
1384
1385 static struct mrs_field_value mvfr1_fpftz[] = {
1386 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1387 MRS_FIELD_VALUE_END,
1388 };
1389
1390 static struct mrs_field mvfr1_fields[] = {
1391 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1392 mvfr1_simdfmac_caps),
1393 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1394 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1395 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1396 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1397 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1398 mvfr1_simdls_caps),
1399 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1400 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1401 MRS_FIELD_END,
1402 };
1403 #endif /* COMPAT_FREEBSD32 */
1404
1405 struct mrs_user_reg {
1406 u_int reg;
1407 u_int CRm;
1408 u_int Op2;
1409 size_t offset;
1410 struct mrs_field *fields;
1411 };
1412
1413 #define USER_REG(name, field_name) \
1414 { \
1415 .reg = name, \
1416 .CRm = name##_CRm, \
1417 .Op2 = name##_op2, \
1418 .offset = __offsetof(struct cpu_desc, field_name), \
1419 .fields = field_name##_fields, \
1420 }
1421 static struct mrs_user_reg user_regs[] = {
1422 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0),
1423
1424 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0),
1425 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1),
1426
1427 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0),
1428 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1),
1429 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2),
1430
1431 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0),
1432 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1),
1433 #ifdef COMPAT_FREEBSD32
1434 USER_REG(ID_ISAR5_EL1, id_isar5),
1435
1436 USER_REG(MVFR0_EL1, mvfr0),
1437 USER_REG(MVFR1_EL1, mvfr1),
1438 #endif /* COMPAT_FREEBSD32 */
1439 };
1440
1441 #define CPU_DESC_FIELD(desc, idx) \
1442 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1443
1444 static int
user_mrs_handler(vm_offset_t va,uint32_t insn,struct trapframe * frame,uint32_t esr)1445 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1446 uint32_t esr)
1447 {
1448 uint64_t value;
1449 int CRm, Op2, i, reg;
1450
1451 if ((insn & MRS_MASK) != MRS_VALUE)
1452 return (0);
1453
1454 /*
1455 * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1456 * These are in the EL1 CPU identification space.
1457 * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1458 * CRm == {4-7} holds the ID_AA64 registers.
1459 *
1460 * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1461 * Table D9-2 System instruction encodings for non-Debug System
1462 * register accesses.
1463 */
1464 if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1465 return (0);
1466
1467 CRm = mrs_CRm(insn);
1468 if (CRm > 7 || (CRm < 4 && CRm != 0))
1469 return (0);
1470
1471 Op2 = mrs_Op2(insn);
1472 value = 0;
1473
1474 for (i = 0; i < nitems(user_regs); i++) {
1475 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1476 value = CPU_DESC_FIELD(user_cpu_desc, i);
1477 break;
1478 }
1479 }
1480
1481 if (CRm == 0) {
1482 switch (Op2) {
1483 case 0:
1484 value = READ_SPECIALREG(midr_el1);
1485 break;
1486 case 5:
1487 value = READ_SPECIALREG(mpidr_el1);
1488 break;
1489 case 6:
1490 value = READ_SPECIALREG(revidr_el1);
1491 break;
1492 default:
1493 return (0);
1494 }
1495 }
1496
1497 /*
1498 * We will handle this instruction, move to the next so we
1499 * don't trap here again.
1500 */
1501 frame->tf_elr += INSN_SIZE;
1502
1503 reg = MRS_REGISTER(insn);
1504 /* If reg is 31 then write to xzr, i.e. do nothing */
1505 if (reg == 31)
1506 return (1);
1507
1508 if (reg < nitems(frame->tf_x))
1509 frame->tf_x[reg] = value;
1510 else if (reg == 30)
1511 frame->tf_lr = value;
1512
1513 return (1);
1514 }
1515
1516 bool
extract_user_id_field(u_int reg,u_int field_shift,uint8_t * val)1517 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
1518 {
1519 uint64_t value;
1520 int i;
1521
1522 for (i = 0; i < nitems(user_regs); i++) {
1523 if (user_regs[i].reg == reg) {
1524 value = CPU_DESC_FIELD(user_cpu_desc, i);
1525 *val = value >> field_shift;
1526 return (true);
1527 }
1528 }
1529
1530 return (false);
1531 }
1532
1533 bool
get_kernel_reg(u_int reg,uint64_t * val)1534 get_kernel_reg(u_int reg, uint64_t *val)
1535 {
1536 int i;
1537
1538 for (i = 0; i < nitems(user_regs); i++) {
1539 if (user_regs[i].reg == reg) {
1540 *val = CPU_DESC_FIELD(kern_cpu_desc, i);
1541 return (true);
1542 }
1543 }
1544
1545 return (false);
1546 }
1547
1548 /*
1549 * Compares two field values that may be signed or unsigned.
1550 * Returns:
1551 * < 0 when a is less than b
1552 * = 0 when a equals b
1553 * > 0 when a is greater than b
1554 */
1555 static int
mrs_field_cmp(uint64_t a,uint64_t b,u_int shift,int width,bool sign)1556 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
1557 {
1558 uint64_t mask;
1559
1560 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1561 width));
1562
1563 mask = (1ul << width) - 1;
1564 /* Move the field to the lower bits */
1565 a = (a >> shift) & mask;
1566 b = (b >> shift) & mask;
1567
1568 if (sign) {
1569 /*
1570 * The field is signed. Toggle the upper bit so the comparison
1571 * works on unsigned values as this makes positive numbers,
1572 * i.e. those with a 0 bit, larger than negative numbers,
1573 * i.e. those with a 1 bit, in an unsigned comparison.
1574 */
1575 a ^= 1ul << (width - 1);
1576 b ^= 1ul << (width - 1);
1577 }
1578
1579 return (a - b);
1580 }
1581
1582 static uint64_t
update_lower_register(uint64_t val,uint64_t new_val,u_int shift,int width,bool sign)1583 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
1584 int width, bool sign)
1585 {
1586 uint64_t mask;
1587
1588 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1589 width));
1590
1591 /*
1592 * If the new value is less than the existing value update it.
1593 */
1594 if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
1595 mask = (1ul << width) - 1;
1596 val &= ~(mask << shift);
1597 val |= new_val & (mask << shift);
1598 }
1599
1600 return (val);
1601 }
1602
1603 void
update_special_regs(u_int cpu)1604 update_special_regs(u_int cpu)
1605 {
1606 struct mrs_field *fields;
1607 uint64_t user_reg, kern_reg, value;
1608 int i, j;
1609
1610 if (cpu == 0) {
1611 /* Create a user visible cpu description with safe values */
1612 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
1613 /* Safe values for these registers */
1614 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
1615 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
1616 ID_AA64PFR0_EL0_64;
1617 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
1618 }
1619
1620 for (i = 0; i < nitems(user_regs); i++) {
1621 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
1622 if (cpu == 0) {
1623 kern_reg = value;
1624 user_reg = value;
1625 } else {
1626 kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
1627 user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
1628 }
1629
1630 fields = user_regs[i].fields;
1631 for (j = 0; fields[j].type != 0; j++) {
1632 switch (fields[j].type & MRS_TYPE_MASK) {
1633 case MRS_EXACT:
1634 user_reg &= ~(0xful << fields[j].shift);
1635 user_reg |=
1636 (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
1637 fields[j].shift;
1638 break;
1639 case MRS_LOWER:
1640 user_reg = update_lower_register(user_reg,
1641 value, fields[j].shift, 4, fields[j].sign);
1642 break;
1643 default:
1644 panic("Invalid field type: %d", fields[j].type);
1645 }
1646 kern_reg = update_lower_register(kern_reg, value,
1647 fields[j].shift, 4, fields[j].sign);
1648 }
1649
1650 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
1651 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
1652 }
1653 }
1654
1655 /* HWCAP */
1656 bool __read_frequently lse_supported = false;
1657
1658 bool __read_frequently icache_aliasing = false;
1659 bool __read_frequently icache_vmid = false;
1660
1661 int64_t dcache_line_size; /* The minimum D cache line size */
1662 int64_t icache_line_size; /* The minimum I cache line size */
1663 int64_t idcache_line_size; /* The minimum cache line size */
1664
1665 /*
1666 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
1667 */
1668 static void
parse_cpu_features(void)1669 parse_cpu_features(void)
1670 {
1671 struct mrs_field_hwcap *hwcaps;
1672 struct mrs_field *fields;
1673 uint64_t min, reg;
1674 int i, j, k;
1675
1676 for (i = 0; i < nitems(user_regs); i++) {
1677 reg = CPU_DESC_FIELD(user_cpu_desc, i);
1678 fields = user_regs[i].fields;
1679 for (j = 0; fields[j].type != 0; j++) {
1680 hwcaps = fields[j].hwcaps;
1681 if (hwcaps == NULL)
1682 continue;
1683
1684 for (k = 0; hwcaps[k].hwcap != NULL; k++) {
1685 min = hwcaps[k].min;
1686
1687 /*
1688 * If the field is greater than the minimum
1689 * value we can set the hwcap;
1690 */
1691 if (mrs_field_cmp(reg, min, fields[j].shift,
1692 4, fields[j].sign) >= 0) {
1693 *hwcaps[k].hwcap |= hwcaps[k].hwcap_val;
1694 }
1695 }
1696 }
1697 }
1698 }
1699
1700 static void
identify_cpu_sysinit(void * dummy __unused)1701 identify_cpu_sysinit(void *dummy __unused)
1702 {
1703 int cpu;
1704 bool dic, idc;
1705
1706 dic = (allow_dic != 0);
1707 idc = (allow_idc != 0);
1708
1709 CPU_FOREACH(cpu) {
1710 check_cpu_regs(cpu);
1711 if (cpu != 0)
1712 update_special_regs(cpu);
1713
1714 if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
1715 dic = false;
1716 if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
1717 idc = false;
1718 }
1719
1720 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
1721 parse_cpu_features();
1722
1723 #ifdef COMPAT_FREEBSD32
1724 /* Set the default caps and any that need to check multiple fields */
1725 elf32_hwcap |= parse_cpu_features_hwcap32();
1726 #endif
1727
1728 if (dic && idc) {
1729 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
1730 if (bootverbose)
1731 printf("Enabling DIC & IDC ICache sync\n");
1732 }
1733
1734 if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
1735 lse_supported = true;
1736 if (bootverbose)
1737 printf("Enabling LSE atomics in the kernel\n");
1738 }
1739 #ifdef LSE_ATOMICS
1740 if (!lse_supported)
1741 panic("CPU does not support LSE atomic instructions");
1742 #endif
1743
1744 install_undef_handler(true, user_mrs_handler);
1745 }
1746 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
1747
1748 static void
cpu_features_sysinit(void * dummy __unused)1749 cpu_features_sysinit(void *dummy __unused)
1750 {
1751 struct sbuf sb;
1752 u_int cpu;
1753
1754 CPU_FOREACH(cpu)
1755 print_cpu_features(cpu);
1756
1757 /* Fill in cpu_model for the hw.model sysctl */
1758 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
1759 print_cpu_midr(&sb, 0);
1760 sbuf_finish(&sb);
1761 sbuf_delete(&sb);
1762 }
1763 /* Log features before APs are released and start printing to the dmesg. */
1764 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
1765
1766 #ifdef COMPAT_FREEBSD32
1767 static u_long
parse_cpu_features_hwcap32(void)1768 parse_cpu_features_hwcap32(void)
1769 {
1770 u_long hwcap = HWCAP32_DEFAULT;
1771
1772 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
1773 MVFR1_SIMDLS_IMPL) &&
1774 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
1775 MVFR1_SIMDInt_IMPL) &&
1776 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
1777 MVFR1_SIMDSP_IMPL))
1778 hwcap |= HWCAP32_NEON;
1779
1780 return (hwcap);
1781 }
1782 #endif /* COMPAT_FREEBSD32 */
1783
1784 static void
print_ctr_fields(struct sbuf * sb,uint64_t reg,void * arg)1785 print_ctr_fields(struct sbuf *sb, uint64_t reg, void *arg)
1786 {
1787
1788 sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
1789 sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
1790 reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
1791
1792 switch(CTR_L1IP_VAL(reg)) {
1793 case CTR_L1IP_VPIPT:
1794 sbuf_printf(sb, "VPIPT");
1795 break;
1796 case CTR_L1IP_AIVIVT:
1797 sbuf_printf(sb, "AIVIVT");
1798 break;
1799 case CTR_L1IP_VIPT:
1800 sbuf_printf(sb, "VIPT");
1801 break;
1802 case CTR_L1IP_PIPT:
1803 sbuf_printf(sb, "PIPT");
1804 break;
1805 }
1806 sbuf_printf(sb, " ICache,");
1807 reg &= ~CTR_L1IP_MASK;
1808
1809 sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
1810 sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
1811 reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
1812
1813 if (CTR_IDC_VAL(reg) != 0)
1814 sbuf_printf(sb, ",IDC");
1815 if (CTR_DIC_VAL(reg) != 0)
1816 sbuf_printf(sb, ",DIC");
1817 reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
1818 reg &= ~CTR_RES1;
1819
1820 if (reg != 0)
1821 sbuf_printf(sb, ",%lx", reg);
1822 }
1823
1824 static void
print_register(struct sbuf * sb,const char * reg_name,uint64_t reg,void (* print_fields)(struct sbuf *,uint64_t,void *),void * arg)1825 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
1826 void (*print_fields)(struct sbuf *, uint64_t, void *), void *arg)
1827 {
1828
1829 sbuf_printf(sb, "%29s = <", reg_name);
1830
1831 print_fields(sb, reg, arg);
1832
1833 sbuf_finish(sb);
1834 printf("%s>\n", sbuf_data(sb));
1835 sbuf_clear(sb);
1836 }
1837
1838 static void
print_id_fields(struct sbuf * sb,uint64_t reg,void * arg)1839 print_id_fields(struct sbuf *sb, uint64_t reg, void *arg)
1840 {
1841 struct mrs_field *fields = arg;
1842 struct mrs_field_value *fv;
1843 int field, i, j, printed;
1844
1845 #define SEP_STR ((printed++) == 0) ? "" : ","
1846 printed = 0;
1847 for (i = 0; fields[i].type != 0; i++) {
1848 fv = fields[i].values;
1849
1850 /* TODO: Handle with an unknown message */
1851 if (fv == NULL)
1852 continue;
1853
1854 field = (reg & fields[i].mask) >> fields[i].shift;
1855 for (j = 0; fv[j].desc != NULL; j++) {
1856 if ((fv[j].value >> fields[i].shift) != field)
1857 continue;
1858
1859 if (fv[j].desc[0] != '\0')
1860 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
1861 break;
1862 }
1863 if (fv[j].desc == NULL)
1864 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
1865 fields[i].name, field);
1866
1867 reg &= ~(0xful << fields[i].shift);
1868 }
1869
1870 if (reg != 0)
1871 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
1872 #undef SEP_STR
1873 }
1874
1875 static void
print_id_register(struct sbuf * sb,const char * reg_name,uint64_t reg,struct mrs_field * fields)1876 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
1877 struct mrs_field *fields)
1878 {
1879
1880 print_register(sb, reg_name, reg, print_id_fields, fields);
1881 }
1882
1883 static void
print_cpu_midr(struct sbuf * sb,u_int cpu)1884 print_cpu_midr(struct sbuf *sb, u_int cpu)
1885 {
1886 const struct cpu_parts *cpu_partsp;
1887 const char *cpu_impl_name;
1888 const char *cpu_part_name;
1889 u_int midr;
1890 u_int impl_id;
1891 u_int part_id;
1892
1893 midr = pcpu_find(cpu)->pc_midr;
1894
1895 cpu_impl_name = NULL;
1896 cpu_partsp = NULL;
1897 impl_id = CPU_IMPL(midr);
1898 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
1899 if (impl_id == cpu_implementers[i].impl_id) {
1900 cpu_impl_name = cpu_implementers[i].impl_name;
1901 cpu_partsp = cpu_implementers[i].cpu_parts;
1902 break;
1903 }
1904 }
1905 /* Unknown implementer, so unknown part */
1906 if (cpu_impl_name == NULL) {
1907 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
1908 return;
1909 }
1910
1911 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
1912 __func__, cpu_impl_name));
1913
1914 cpu_part_name = NULL;
1915 part_id = CPU_PART(midr);
1916 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
1917 if (part_id == cpu_partsp[i].part_id) {
1918 cpu_part_name = cpu_partsp[i].part_name;
1919 break;
1920 }
1921 }
1922 /* Known Implementer, Unknown part */
1923 if (cpu_part_name == NULL) {
1924 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
1925 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
1926 return;
1927 }
1928
1929 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
1930 cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
1931 }
1932
1933 static void
print_cpu_features(u_int cpu)1934 print_cpu_features(u_int cpu)
1935 {
1936 struct sbuf *sb;
1937
1938 sb = sbuf_new_auto();
1939 sbuf_printf(sb, "CPU%3u: ", cpu);
1940 print_cpu_midr(sb, cpu);
1941
1942 sbuf_cat(sb, " affinity:");
1943 switch(cpu_aff_levels) {
1944 default:
1945 case 4:
1946 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
1947 /* FALLTHROUGH */
1948 case 3:
1949 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
1950 /* FALLTHROUGH */
1951 case 2:
1952 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
1953 /* FALLTHROUGH */
1954 case 1:
1955 case 0: /* On UP this will be zero */
1956 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
1957 break;
1958 }
1959 sbuf_finish(sb);
1960 printf("%s\n", sbuf_data(sb));
1961 sbuf_clear(sb);
1962
1963 /*
1964 * There is a hardware errata where, if one CPU is performing a TLB
1965 * invalidation while another is performing a store-exclusive the
1966 * store-exclusive may return the wrong status. A workaround seems
1967 * to be to use an IPI to invalidate on each CPU, however given the
1968 * limited number of affected units (pass 1.1 is the evaluation
1969 * hardware revision), and the lack of information from Cavium
1970 * this has not been implemented.
1971 *
1972 * At the time of writing this the only information is from:
1973 * https://lkml.org/lkml/2016/8/4/722
1974 */
1975 /*
1976 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
1977 * triggers on pass 2.0+.
1978 */
1979 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
1980 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
1981 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
1982 "hardware bugs that may cause the incorrect operation of "
1983 "atomic operations.\n");
1984
1985 /* Cache Type Register */
1986 if (cpu == 0 || (cpu_print_regs & PRINT_CTR_EL0) != 0) {
1987 print_register(sb, "Cache Type",
1988 cpu_desc[cpu].ctr, print_ctr_fields, NULL);
1989 }
1990
1991 /* AArch64 Instruction Set Attribute Register 0 */
1992 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR0) != 0)
1993 print_id_register(sb, "Instruction Set Attributes 0",
1994 cpu_desc[cpu].id_aa64isar0, id_aa64isar0_fields);
1995
1996 /* AArch64 Instruction Set Attribute Register 1 */
1997 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR1) != 0)
1998 print_id_register(sb, "Instruction Set Attributes 1",
1999 cpu_desc[cpu].id_aa64isar1, id_aa64isar1_fields);
2000
2001 /* AArch64 Processor Feature Register 0 */
2002 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR0) != 0)
2003 print_id_register(sb, "Processor Features 0",
2004 cpu_desc[cpu].id_aa64pfr0, id_aa64pfr0_fields);
2005
2006 /* AArch64 Processor Feature Register 1 */
2007 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR1) != 0)
2008 print_id_register(sb, "Processor Features 1",
2009 cpu_desc[cpu].id_aa64pfr1, id_aa64pfr1_fields);
2010
2011 /* AArch64 Memory Model Feature Register 0 */
2012 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR0) != 0)
2013 print_id_register(sb, "Memory Model Features 0",
2014 cpu_desc[cpu].id_aa64mmfr0, id_aa64mmfr0_fields);
2015
2016 /* AArch64 Memory Model Feature Register 1 */
2017 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR1) != 0)
2018 print_id_register(sb, "Memory Model Features 1",
2019 cpu_desc[cpu].id_aa64mmfr1, id_aa64mmfr1_fields);
2020
2021 /* AArch64 Memory Model Feature Register 2 */
2022 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR2) != 0)
2023 print_id_register(sb, "Memory Model Features 2",
2024 cpu_desc[cpu].id_aa64mmfr2, id_aa64mmfr2_fields);
2025
2026 /* AArch64 Debug Feature Register 0 */
2027 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR0) != 0)
2028 print_id_register(sb, "Debug Features 0",
2029 cpu_desc[cpu].id_aa64dfr0, id_aa64dfr0_fields);
2030
2031 /* AArch64 Memory Model Feature Register 1 */
2032 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR1) != 0)
2033 print_id_register(sb, "Debug Features 1",
2034 cpu_desc[cpu].id_aa64dfr1, id_aa64dfr1_fields);
2035
2036 /* AArch64 Auxiliary Feature Register 0 */
2037 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR0) != 0)
2038 print_id_register(sb, "Auxiliary Features 0",
2039 cpu_desc[cpu].id_aa64afr0, id_aa64afr0_fields);
2040
2041 /* AArch64 Auxiliary Feature Register 1 */
2042 if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR1) != 0)
2043 print_id_register(sb, "Auxiliary Features 1",
2044 cpu_desc[cpu].id_aa64afr1, id_aa64afr1_fields);
2045
2046 #ifdef COMPAT_FREEBSD32
2047 /* AArch32 Instruction Set Attribute Register 5 */
2048 if (cpu == 0 || (cpu_print_regs & PRINT_ID_ISAR5) != 0)
2049 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2050 cpu_desc[cpu].id_isar5, id_isar5_fields);
2051
2052 /* AArch32 Media and VFP Feature Register 0 */
2053 if (cpu == 0 || (cpu_print_regs & PRINT_MVFR0) != 0)
2054 print_id_register(sb, "AArch32 Media and VFP Features 0",
2055 cpu_desc[cpu].mvfr0, mvfr0_fields);
2056
2057 /* AArch32 Media and VFP Feature Register 1 */
2058 if (cpu == 0 || (cpu_print_regs & PRINT_MVFR1) != 0)
2059 print_id_register(sb, "AArch32 Media and VFP Features 1",
2060 cpu_desc[cpu].mvfr1, mvfr1_fields);
2061 #endif
2062
2063 sbuf_delete(sb);
2064 sb = NULL;
2065 #undef SEP_STR
2066 }
2067
2068 void
identify_cache(uint64_t ctr)2069 identify_cache(uint64_t ctr)
2070 {
2071
2072 /* Identify the L1 cache type */
2073 switch (CTR_L1IP_VAL(ctr)) {
2074 case CTR_L1IP_PIPT:
2075 break;
2076 case CTR_L1IP_VPIPT:
2077 icache_vmid = true;
2078 break;
2079 default:
2080 case CTR_L1IP_VIPT:
2081 icache_aliasing = true;
2082 break;
2083 }
2084
2085 if (dcache_line_size == 0) {
2086 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2087 __func__, icache_line_size));
2088
2089 /* Get the D cache line size */
2090 dcache_line_size = CTR_DLINE_SIZE(ctr);
2091 /* And the same for the I cache */
2092 icache_line_size = CTR_ILINE_SIZE(ctr);
2093
2094 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2095 }
2096
2097 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2098 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2099 dcache_line_size, CTR_DLINE_SIZE(ctr));
2100 }
2101
2102 if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2103 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2104 icache_line_size, CTR_ILINE_SIZE(ctr));
2105 }
2106 }
2107
2108 void
identify_cpu(u_int cpu)2109 identify_cpu(u_int cpu)
2110 {
2111 /* Save affinity for current CPU */
2112 cpu_desc[cpu].mpidr = get_mpidr();
2113 CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
2114
2115 cpu_desc[cpu].ctr = READ_SPECIALREG(ctr_el0);
2116 cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
2117 cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
2118 cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
2119 cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
2120 cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
2121 cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
2122 cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
2123 cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
2124 cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
2125 #ifdef COMPAT_FREEBSD32
2126 /* Only read aarch32 SRs if EL0-32 is available */
2127 if (ID_AA64PFR0_EL0_VAL(cpu_desc[cpu].id_aa64pfr0) ==
2128 ID_AA64PFR0_EL0_64_32) {
2129 cpu_desc[cpu].id_isar5 = READ_SPECIALREG(id_isar5_el1);
2130 cpu_desc[cpu].mvfr0 = READ_SPECIALREG(mvfr0_el1);
2131 cpu_desc[cpu].mvfr1 = READ_SPECIALREG(mvfr1_el1);
2132 }
2133 #endif
2134 }
2135
2136 static void
check_cpu_regs(u_int cpu)2137 check_cpu_regs(u_int cpu)
2138 {
2139
2140 switch (cpu_aff_levels) {
2141 case 0:
2142 if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
2143 CPU_AFF0(cpu_desc[0].mpidr))
2144 cpu_aff_levels = 1;
2145 /* FALLTHROUGH */
2146 case 1:
2147 if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
2148 CPU_AFF1(cpu_desc[0].mpidr))
2149 cpu_aff_levels = 2;
2150 /* FALLTHROUGH */
2151 case 2:
2152 if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
2153 CPU_AFF2(cpu_desc[0].mpidr))
2154 cpu_aff_levels = 3;
2155 /* FALLTHROUGH */
2156 case 3:
2157 if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
2158 CPU_AFF3(cpu_desc[0].mpidr))
2159 cpu_aff_levels = 4;
2160 break;
2161 }
2162
2163 if (cpu_desc[cpu].id_aa64afr0 != cpu_desc[0].id_aa64afr0)
2164 cpu_print_regs |= PRINT_ID_AA64_AFR0;
2165 if (cpu_desc[cpu].id_aa64afr1 != cpu_desc[0].id_aa64afr1)
2166 cpu_print_regs |= PRINT_ID_AA64_AFR1;
2167
2168 if (cpu_desc[cpu].id_aa64dfr0 != cpu_desc[0].id_aa64dfr0)
2169 cpu_print_regs |= PRINT_ID_AA64_DFR0;
2170 if (cpu_desc[cpu].id_aa64dfr1 != cpu_desc[0].id_aa64dfr1)
2171 cpu_print_regs |= PRINT_ID_AA64_DFR1;
2172
2173 if (cpu_desc[cpu].id_aa64isar0 != cpu_desc[0].id_aa64isar0)
2174 cpu_print_regs |= PRINT_ID_AA64_ISAR0;
2175 if (cpu_desc[cpu].id_aa64isar1 != cpu_desc[0].id_aa64isar1)
2176 cpu_print_regs |= PRINT_ID_AA64_ISAR1;
2177
2178 if (cpu_desc[cpu].id_aa64mmfr0 != cpu_desc[0].id_aa64mmfr0)
2179 cpu_print_regs |= PRINT_ID_AA64_MMFR0;
2180 if (cpu_desc[cpu].id_aa64mmfr1 != cpu_desc[0].id_aa64mmfr1)
2181 cpu_print_regs |= PRINT_ID_AA64_MMFR1;
2182 if (cpu_desc[cpu].id_aa64mmfr2 != cpu_desc[0].id_aa64mmfr2)
2183 cpu_print_regs |= PRINT_ID_AA64_MMFR2;
2184
2185 if (cpu_desc[cpu].id_aa64pfr0 != cpu_desc[0].id_aa64pfr0)
2186 cpu_print_regs |= PRINT_ID_AA64_PFR0;
2187 if (cpu_desc[cpu].id_aa64pfr1 != cpu_desc[0].id_aa64pfr1)
2188 cpu_print_regs |= PRINT_ID_AA64_PFR1;
2189
2190 if (cpu_desc[cpu].ctr != cpu_desc[0].ctr) {
2191 /*
2192 * If the cache type register is different we may
2193 * have a different l1 cache type.
2194 */
2195 identify_cache(cpu_desc[cpu].ctr);
2196 cpu_print_regs |= PRINT_CTR_EL0;
2197 }
2198
2199 #ifdef COMPAT_FREEBSD32
2200 if (cpu_desc[cpu].id_isar5 != cpu_desc[0].id_isar5)
2201 cpu_print_regs |= PRINT_ID_ISAR5;
2202 if (cpu_desc[cpu].mvfr0 != cpu_desc[0].mvfr0)
2203 cpu_print_regs |= PRINT_MVFR0;
2204 if (cpu_desc[cpu].mvfr1 != cpu_desc[0].mvfr1)
2205 cpu_print_regs |= PRINT_MVFR1;
2206 #endif
2207 }
2208