1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2006-2008 Stanislav Sedov <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 */
29
30 #include <sys/cdefs.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/conf.h>
34 #include <sys/fcntl.h>
35 #include <sys/ioccom.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/mutex.h>
39 #include <sys/priv.h>
40 #include <sys/proc.h>
41 #include <sys/queue.h>
42 #include <sys/sched.h>
43 #include <sys/kernel.h>
44 #include <sys/sysctl.h>
45 #include <sys/uio.h>
46 #include <sys/pcpu.h>
47 #include <sys/smp.h>
48 #include <sys/pmckern.h>
49 #include <sys/cpuctl.h>
50
51 #include <vm/vm.h>
52 #include <vm/vm_param.h>
53 #include <vm/pmap.h>
54
55 #include <machine/cpufunc.h>
56 #include <machine/md_var.h>
57 #include <machine/specialreg.h>
58 #include <x86/ucode.h>
59
60 static d_open_t cpuctl_open;
61 static d_ioctl_t cpuctl_ioctl;
62
63 #define CPUCTL_VERSION 1
64
65 #ifdef CPUCTL_DEBUG
66 # define DPRINTF(format,...) printf(format, __VA_ARGS__);
67 #else
68 # define DPRINTF(...)
69 #endif
70
71 #define UCODE_SIZE_MAX (4 * 1024 * 1024)
72
73 static int cpuctl_do_msr(int cpu, cpuctl_msr_args_t *data, u_long cmd,
74 struct thread *td);
75 static int cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data,
76 struct thread *td);
77 static int cpuctl_do_cpuid_count(int cpu, cpuctl_cpuid_count_args_t *data,
78 struct thread *td);
79 static int cpuctl_do_eval_cpu_features(int cpu, struct thread *td);
80 static int cpuctl_do_update(int cpu, cpuctl_update_args_t *data,
81 struct thread *td);
82 static int update_intel(int cpu, cpuctl_update_args_t *args,
83 struct thread *td);
84 static int update_amd(int cpu, cpuctl_update_args_t *args, struct thread *td);
85 static int update_via(int cpu, cpuctl_update_args_t *args,
86 struct thread *td);
87
88 static struct cdev **cpuctl_devs;
89 static MALLOC_DEFINE(M_CPUCTL, "cpuctl", "CPUCTL buffer");
90
91 static struct cdevsw cpuctl_cdevsw = {
92 .d_version = D_VERSION,
93 .d_open = cpuctl_open,
94 .d_ioctl = cpuctl_ioctl,
95 .d_name = "cpuctl",
96 };
97
98 /*
99 * This function checks if specified cpu enabled or not.
100 */
101 static int
cpu_enabled(int cpu)102 cpu_enabled(int cpu)
103 {
104
105 return (pmc_cpu_is_disabled(cpu) == 0);
106 }
107
108 /*
109 * Check if the current thread is bound to a specific cpu.
110 */
111 static int
cpu_sched_is_bound(struct thread * td)112 cpu_sched_is_bound(struct thread *td)
113 {
114 int ret;
115
116 thread_lock(td);
117 ret = sched_is_bound(td);
118 thread_unlock(td);
119 return (ret);
120 }
121
122 /*
123 * Switch to target cpu to run.
124 */
125 static void
set_cpu(int cpu,struct thread * td)126 set_cpu(int cpu, struct thread *td)
127 {
128
129 KASSERT(cpu >= 0 && cpu <= mp_maxid && cpu_enabled(cpu),
130 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu));
131 thread_lock(td);
132 sched_bind(td, cpu);
133 thread_unlock(td);
134 KASSERT(td->td_oncpu == cpu,
135 ("[cpuctl,%d]: cannot bind to target cpu %d on cpu %d", __LINE__,
136 cpu, td->td_oncpu));
137 }
138
139 static void
restore_cpu(int oldcpu,int is_bound,struct thread * td)140 restore_cpu(int oldcpu, int is_bound, struct thread *td)
141 {
142
143 KASSERT(oldcpu >= 0 && oldcpu <= mp_maxid && cpu_enabled(oldcpu),
144 ("[cpuctl,%d]: bad cpu number %d", __LINE__, oldcpu));
145 thread_lock(td);
146 if (is_bound == 0)
147 sched_unbind(td);
148 else
149 sched_bind(td, oldcpu);
150 thread_unlock(td);
151 }
152
153 int
cpuctl_ioctl(struct cdev * dev,u_long cmd,caddr_t data,int flags,struct thread * td)154 cpuctl_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
155 int flags, struct thread *td)
156 {
157 int cpu, ret;
158
159 cpu = dev2unit(dev);
160 if (cpu > mp_maxid || !cpu_enabled(cpu)) {
161 DPRINTF("[cpuctl,%d]: bad cpu number %d\n", __LINE__, cpu);
162 return (ENXIO);
163 }
164 /* Require write flag for "write" requests. */
165 if ((cmd == CPUCTL_MSRCBIT || cmd == CPUCTL_MSRSBIT ||
166 cmd == CPUCTL_UPDATE || cmd == CPUCTL_WRMSR ||
167 cmd == CPUCTL_EVAL_CPU_FEATURES) &&
168 (flags & FWRITE) == 0)
169 return (EPERM);
170 switch (cmd) {
171 case CPUCTL_RDMSR:
172 ret = cpuctl_do_msr(cpu, (cpuctl_msr_args_t *)data, cmd, td);
173 break;
174 case CPUCTL_MSRSBIT:
175 case CPUCTL_MSRCBIT:
176 case CPUCTL_WRMSR:
177 ret = priv_check(td, PRIV_CPUCTL_WRMSR);
178 if (ret != 0)
179 goto fail;
180 ret = cpuctl_do_msr(cpu, (cpuctl_msr_args_t *)data, cmd, td);
181 break;
182 case CPUCTL_CPUID:
183 ret = cpuctl_do_cpuid(cpu, (cpuctl_cpuid_args_t *)data, td);
184 break;
185 case CPUCTL_UPDATE:
186 ret = priv_check(td, PRIV_CPUCTL_UPDATE);
187 if (ret != 0)
188 goto fail;
189 ret = cpuctl_do_update(cpu, (cpuctl_update_args_t *)data, td);
190 break;
191 case CPUCTL_CPUID_COUNT:
192 ret = cpuctl_do_cpuid_count(cpu,
193 (cpuctl_cpuid_count_args_t *)data, td);
194 break;
195 case CPUCTL_EVAL_CPU_FEATURES:
196 ret = cpuctl_do_eval_cpu_features(cpu, td);
197 break;
198 default:
199 ret = EINVAL;
200 break;
201 }
202 fail:
203 return (ret);
204 }
205
206 /*
207 * Actually perform cpuid operation.
208 */
209 static int
cpuctl_do_cpuid_count(int cpu,cpuctl_cpuid_count_args_t * data,struct thread * td)210 cpuctl_do_cpuid_count(int cpu, cpuctl_cpuid_count_args_t *data,
211 struct thread *td)
212 {
213 int is_bound = 0;
214 int oldcpu;
215
216 KASSERT(cpu >= 0 && cpu <= mp_maxid,
217 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu));
218
219 /* Explicitly clear cpuid data to avoid returning stale info. */
220 bzero(data->data, sizeof(data->data));
221 DPRINTF("[cpuctl,%d]: retrieving cpuid lev %#0x type %#0x for %d cpu\n",
222 __LINE__, data->level, data->level_type, cpu);
223 #ifdef __i386__
224 if (cpu_id == 0)
225 return (ENODEV);
226 #endif
227 oldcpu = td->td_oncpu;
228 is_bound = cpu_sched_is_bound(td);
229 set_cpu(cpu, td);
230 cpuid_count(data->level, data->level_type, data->data);
231 restore_cpu(oldcpu, is_bound, td);
232 return (0);
233 }
234
235 static int
cpuctl_do_cpuid(int cpu,cpuctl_cpuid_args_t * data,struct thread * td)236 cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data, struct thread *td)
237 {
238 cpuctl_cpuid_count_args_t cdata;
239 int error;
240
241 cdata.level = data->level;
242 /* Override the level type. */
243 cdata.level_type = 0;
244 error = cpuctl_do_cpuid_count(cpu, &cdata, td);
245 bcopy(cdata.data, data->data, sizeof(data->data)); /* Ignore error */
246 return (error);
247 }
248
249 /*
250 * Actually perform MSR operations.
251 */
252 static int
cpuctl_do_msr(int cpu,cpuctl_msr_args_t * data,u_long cmd,struct thread * td)253 cpuctl_do_msr(int cpu, cpuctl_msr_args_t *data, u_long cmd, struct thread *td)
254 {
255 uint64_t reg;
256 int is_bound = 0;
257 int oldcpu;
258 int ret;
259
260 KASSERT(cpu >= 0 && cpu <= mp_maxid,
261 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu));
262
263 /*
264 * Explicitly clear cpuid data to avoid returning stale
265 * info
266 */
267 DPRINTF("[cpuctl,%d]: operating on MSR %#0x for %d cpu\n", __LINE__,
268 data->msr, cpu);
269 #ifdef __i386__
270 if ((cpu_feature & CPUID_MSR) == 0)
271 return (ENODEV);
272 #endif
273 oldcpu = td->td_oncpu;
274 is_bound = cpu_sched_is_bound(td);
275 set_cpu(cpu, td);
276 if (cmd == CPUCTL_RDMSR) {
277 data->data = 0;
278 ret = rdmsr_safe(data->msr, &data->data);
279 } else if (cmd == CPUCTL_WRMSR) {
280 ret = wrmsr_safe(data->msr, data->data);
281 } else if (cmd == CPUCTL_MSRSBIT) {
282 critical_enter();
283 ret = rdmsr_safe(data->msr, ®);
284 if (ret == 0)
285 ret = wrmsr_safe(data->msr, reg | data->data);
286 critical_exit();
287 } else if (cmd == CPUCTL_MSRCBIT) {
288 critical_enter();
289 ret = rdmsr_safe(data->msr, ®);
290 if (ret == 0)
291 ret = wrmsr_safe(data->msr, reg & ~data->data);
292 critical_exit();
293 } else
294 panic("[cpuctl,%d]: unknown operation requested: %lu",
295 __LINE__, cmd);
296 restore_cpu(oldcpu, is_bound, td);
297 return (ret);
298 }
299
300 /*
301 * Actually perform microcode update.
302 */
303 static int
cpuctl_do_update(int cpu,cpuctl_update_args_t * data,struct thread * td)304 cpuctl_do_update(int cpu, cpuctl_update_args_t *data, struct thread *td)
305 {
306 cpuctl_cpuid_args_t args = {
307 .level = 0,
308 };
309 char vendor[13];
310 int ret;
311
312 KASSERT(cpu >= 0 && cpu <= mp_maxid,
313 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu));
314 DPRINTF("[cpuctl,%d]: XXX %d", __LINE__, cpu);
315
316 ret = cpuctl_do_cpuid(cpu, &args, td);
317 if (ret != 0)
318 return (ret);
319 ((uint32_t *)vendor)[0] = args.data[1];
320 ((uint32_t *)vendor)[1] = args.data[3];
321 ((uint32_t *)vendor)[2] = args.data[2];
322 vendor[12] = '\0';
323 if (strncmp(vendor, INTEL_VENDOR_ID, sizeof(INTEL_VENDOR_ID)) == 0)
324 ret = update_intel(cpu, data, td);
325 else if(strncmp(vendor, AMD_VENDOR_ID, sizeof(AMD_VENDOR_ID)) == 0)
326 ret = update_amd(cpu, data, td);
327 else if(strncmp(vendor, CENTAUR_VENDOR_ID, sizeof(CENTAUR_VENDOR_ID))
328 == 0)
329 ret = update_via(cpu, data, td);
330 else
331 ret = ENXIO;
332 return (ret);
333 }
334
335 struct ucode_update_data {
336 void *ptr;
337 int cpu;
338 int ret;
339 };
340
341 static void
ucode_intel_load_rv(void * arg)342 ucode_intel_load_rv(void *arg)
343 {
344 struct ucode_update_data *d;
345
346 d = arg;
347 if (PCPU_GET(cpuid) == d->cpu)
348 d->ret = ucode_intel_load(d->ptr, true, NULL, NULL);
349 }
350
351 static int
update_intel(int cpu,cpuctl_update_args_t * args,struct thread * td)352 update_intel(int cpu, cpuctl_update_args_t *args, struct thread *td)
353 {
354 struct ucode_update_data d;
355 void *ptr;
356 int is_bound, oldcpu, ret;
357
358 if (args->size == 0 || args->data == NULL) {
359 DPRINTF("[cpuctl,%d]: zero-sized firmware image", __LINE__);
360 return (EINVAL);
361 }
362 if (args->size > UCODE_SIZE_MAX) {
363 DPRINTF("[cpuctl,%d]: firmware image too large", __LINE__);
364 return (EINVAL);
365 }
366
367 /*
368 * 16 byte alignment required. Rely on the fact that
369 * malloc(9) always returns the pointer aligned at least on
370 * the size of the allocation.
371 */
372 ptr = malloc(args->size + 16, M_CPUCTL, M_WAITOK);
373 if (copyin(args->data, ptr, args->size) != 0) {
374 DPRINTF("[cpuctl,%d]: copyin %p->%p of %zd bytes failed",
375 __LINE__, args->data, ptr, args->size);
376 ret = EFAULT;
377 goto out;
378 }
379 oldcpu = td->td_oncpu;
380 is_bound = cpu_sched_is_bound(td);
381 set_cpu(cpu, td);
382 d.ptr = ptr;
383 d.cpu = cpu;
384 smp_rendezvous(NULL, ucode_intel_load_rv, NULL, &d);
385 restore_cpu(oldcpu, is_bound, td);
386 ret = d.ret;
387
388 /*
389 * Replace any existing update. This ensures that the new update
390 * will be reloaded automatically during ACPI resume.
391 */
392 if (ret == 0)
393 ptr = ucode_update(ptr);
394
395 out:
396 free(ptr, M_CPUCTL);
397 return (ret);
398 }
399
400 /*
401 * NB: MSR 0xc0010020, MSR_K8_UCODE_UPDATE, is not documented by AMD.
402 * Coreboot, illumos and Linux source code was used to understand
403 * its workings.
404 */
405 static void
amd_ucode_wrmsr(void * ucode_ptr)406 amd_ucode_wrmsr(void *ucode_ptr)
407 {
408 uint32_t tmp[4];
409
410 wrmsr_safe(MSR_K8_UCODE_UPDATE, (uintptr_t)ucode_ptr);
411 do_cpuid(0, tmp);
412 }
413
414 static int
update_amd(int cpu,cpuctl_update_args_t * args,struct thread * td)415 update_amd(int cpu, cpuctl_update_args_t *args, struct thread *td)
416 {
417 void *ptr;
418 int ret;
419
420 if (args->size == 0 || args->data == NULL) {
421 DPRINTF("[cpuctl,%d]: zero-sized firmware image", __LINE__);
422 return (EINVAL);
423 }
424 if (args->size > UCODE_SIZE_MAX) {
425 DPRINTF("[cpuctl,%d]: firmware image too large", __LINE__);
426 return (EINVAL);
427 }
428
429 /*
430 * 16 byte alignment required. Rely on the fact that
431 * malloc(9) always returns the pointer aligned at least on
432 * the size of the allocation.
433 */
434 ptr = malloc(args->size + 16, M_CPUCTL, M_ZERO | M_WAITOK);
435 if (copyin(args->data, ptr, args->size) != 0) {
436 DPRINTF("[cpuctl,%d]: copyin %p->%p of %zd bytes failed",
437 __LINE__, args->data, ptr, args->size);
438 ret = EFAULT;
439 goto fail;
440 }
441 smp_rendezvous(NULL, amd_ucode_wrmsr, NULL, ptr);
442 ret = 0;
443 fail:
444 free(ptr, M_CPUCTL);
445 return (ret);
446 }
447
448 static int
update_via(int cpu,cpuctl_update_args_t * args,struct thread * td)449 update_via(int cpu, cpuctl_update_args_t *args, struct thread *td)
450 {
451 void *ptr;
452 uint64_t rev0, rev1, res;
453 uint32_t tmp[4];
454 int is_bound;
455 int oldcpu;
456 int ret;
457
458 if (args->size == 0 || args->data == NULL) {
459 DPRINTF("[cpuctl,%d]: zero-sized firmware image", __LINE__);
460 return (EINVAL);
461 }
462 if (args->size > UCODE_SIZE_MAX) {
463 DPRINTF("[cpuctl,%d]: firmware image too large", __LINE__);
464 return (EINVAL);
465 }
466
467 /*
468 * 4 byte alignment required.
469 */
470 ptr = malloc(args->size, M_CPUCTL, M_WAITOK);
471 if (copyin(args->data, ptr, args->size) != 0) {
472 DPRINTF("[cpuctl,%d]: copyin %p->%p of %zd bytes failed",
473 __LINE__, args->data, ptr, args->size);
474 ret = EFAULT;
475 goto fail;
476 }
477 oldcpu = td->td_oncpu;
478 is_bound = cpu_sched_is_bound(td);
479 set_cpu(cpu, td);
480 critical_enter();
481 rdmsr_safe(MSR_BIOS_SIGN, &rev0); /* Get current microcode revision. */
482
483 /*
484 * Perform update.
485 */
486 wrmsr_safe(MSR_BIOS_UPDT_TRIG, (uintptr_t)(ptr));
487 do_cpuid(1, tmp);
488
489 /*
490 * Result are in low byte of MSR FCR5:
491 * 0x00: No update has been attempted since RESET.
492 * 0x01: The last attempted update was successful.
493 * 0x02: The last attempted update was unsuccessful due to a bad
494 * environment. No update was loaded and any preexisting
495 * patches are still active.
496 * 0x03: The last attempted update was not applicable to this processor.
497 * No update was loaded and any preexisting patches are still
498 * active.
499 * 0x04: The last attempted update was not successful due to an invalid
500 * update data block. No update was loaded and any preexisting
501 * patches are still active
502 */
503 rdmsr_safe(0x1205, &res);
504 res &= 0xff;
505 critical_exit();
506 rdmsr_safe(MSR_BIOS_SIGN, &rev1); /* Get new microcode revision. */
507 restore_cpu(oldcpu, is_bound, td);
508
509 DPRINTF("[cpu,%d]: rev0=%x rev1=%x res=%x\n", __LINE__,
510 (unsigned)(rev0 >> 32), (unsigned)(rev1 >> 32), (unsigned)res);
511
512 if (res != 0x01)
513 ret = EINVAL;
514 else
515 ret = 0;
516 fail:
517 free(ptr, M_CPUCTL);
518 return (ret);
519 }
520
521 static int
cpuctl_do_eval_cpu_features(int cpu,struct thread * td)522 cpuctl_do_eval_cpu_features(int cpu, struct thread *td)
523 {
524 int is_bound = 0;
525 int oldcpu;
526
527 KASSERT(cpu >= 0 && cpu <= mp_maxid,
528 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu));
529
530 #ifdef __i386__
531 if (cpu_id == 0)
532 return (ENODEV);
533 #endif
534 oldcpu = td->td_oncpu;
535 is_bound = cpu_sched_is_bound(td);
536 set_cpu(cpu, td);
537 identify_cpu1();
538 identify_cpu2();
539 restore_cpu(oldcpu, is_bound, td);
540 hw_ibrs_recalculate(true);
541 hw_ssb_recalculate(true);
542 #ifdef __amd64__
543 amd64_syscall_ret_flush_l1d_recalc();
544 pmap_allow_2m_x_ept_recalculate();
545 #endif
546 hw_mds_recalculate();
547 x86_taa_recalculate();
548 x86_rngds_mitg_recalculate(true);
549 zenbleed_check_and_apply(true);
550 printcpuinfo();
551 return (0);
552 }
553
554 int
cpuctl_open(struct cdev * dev,int flags,int fmt __unused,struct thread * td)555 cpuctl_open(struct cdev *dev, int flags, int fmt __unused, struct thread *td)
556 {
557 int ret = 0;
558 int cpu;
559
560 cpu = dev2unit(dev);
561 if (cpu > mp_maxid || !cpu_enabled(cpu)) {
562 DPRINTF("[cpuctl,%d]: incorrect cpu number %d\n", __LINE__,
563 cpu);
564 return (ENXIO);
565 }
566 if (flags & FWRITE)
567 ret = securelevel_gt(td->td_ucred, 0);
568 return (ret);
569 }
570
571 static int
cpuctl_modevent(module_t mod __unused,int type,void * data __unused)572 cpuctl_modevent(module_t mod __unused, int type, void *data __unused)
573 {
574 int cpu;
575
576 switch(type) {
577 case MOD_LOAD:
578 if (bootverbose)
579 printf("cpuctl: access to MSR registers/cpuid info.\n");
580 cpuctl_devs = malloc(sizeof(*cpuctl_devs) * (mp_maxid + 1), M_CPUCTL,
581 M_WAITOK | M_ZERO);
582 CPU_FOREACH(cpu)
583 if (cpu_enabled(cpu))
584 cpuctl_devs[cpu] = make_dev(&cpuctl_cdevsw, cpu,
585 UID_ROOT, GID_KMEM, 0640, "cpuctl%d", cpu);
586 break;
587 case MOD_UNLOAD:
588 CPU_FOREACH(cpu) {
589 if (cpuctl_devs[cpu] != NULL)
590 destroy_dev(cpuctl_devs[cpu]);
591 }
592 free(cpuctl_devs, M_CPUCTL);
593 break;
594 case MOD_SHUTDOWN:
595 break;
596 default:
597 return (EOPNOTSUPP);
598 }
599 return (0);
600 }
601
602 DEV_MODULE(cpuctl, cpuctl_modevent, NULL);
603 MODULE_VERSION(cpuctl, CPUCTL_VERSION);
604