1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Clocksource driver for the synthetic counter and timers 5 * provided by the Hyper-V hypervisor to guest VMs, as described 6 * in the Hyper-V Top Level Functional Spec (TLFS). This driver 7 * is instruction set architecture independent. 8 * 9 * Copyright (C) 2019, Microsoft, Inc. 10 * 11 * Author: Michael Kelley <[email protected]> 12 */ 13 14 #include <linux/percpu.h> 15 #include <linux/cpumask.h> 16 #include <linux/clockchips.h> 17 #include <linux/clocksource.h> 18 #include <linux/sched_clock.h> 19 #include <linux/mm.h> 20 #include <linux/cpuhotplug.h> 21 #include <linux/interrupt.h> 22 #include <linux/irq.h> 23 #include <linux/acpi.h> 24 #include <linux/hyperv.h> 25 #include <clocksource/hyperv_timer.h> 26 #include <asm/hyperv-tlfs.h> 27 #include <asm/mshyperv.h> 28 29 static struct clock_event_device __percpu *hv_clock_event; 30 static u64 hv_sched_clock_offset __ro_after_init; 31 32 /* 33 * If false, we're using the old mechanism for stimer0 interrupts 34 * where it sends a VMbus message when it expires. The old 35 * mechanism is used when running on older versions of Hyper-V 36 * that don't support Direct Mode. While Hyper-V provides 37 * four stimer's per CPU, Linux uses only stimer0. 38 * 39 * Because Direct Mode does not require processing a VMbus 40 * message, stimer interrupts can be enabled earlier in the 41 * process of booting a CPU, and consistent with when timer 42 * interrupts are enabled for other clocksource drivers. 43 * However, for legacy versions of Hyper-V when Direct Mode 44 * is not enabled, setting up stimer interrupts must be 45 * delayed until VMbus is initialized and can process the 46 * interrupt message. 47 */ 48 static bool direct_mode_enabled; 49 50 static int stimer0_irq = -1; 51 static int stimer0_message_sint; 52 static DEFINE_PER_CPU(long, stimer0_evt); 53 54 /* 55 * Common code for stimer0 interrupts coming via Direct Mode or 56 * as a VMbus message. 57 */ 58 void hv_stimer0_isr(void) 59 { 60 struct clock_event_device *ce; 61 62 ce = this_cpu_ptr(hv_clock_event); 63 ce->event_handler(ce); 64 } 65 EXPORT_SYMBOL_GPL(hv_stimer0_isr); 66 67 /* 68 * stimer0 interrupt handler for architectures that support 69 * per-cpu interrupts, which also implies Direct Mode. 70 */ 71 static irqreturn_t hv_stimer0_percpu_isr(int irq, void *dev_id) 72 { 73 hv_stimer0_isr(); 74 return IRQ_HANDLED; 75 } 76 77 static int hv_ce_set_next_event(unsigned long delta, 78 struct clock_event_device *evt) 79 { 80 u64 current_tick; 81 82 current_tick = hv_read_reference_counter(); 83 current_tick += delta; 84 hv_set_register(HV_REGISTER_STIMER0_COUNT, current_tick); 85 return 0; 86 } 87 88 static int hv_ce_shutdown(struct clock_event_device *evt) 89 { 90 hv_set_register(HV_REGISTER_STIMER0_COUNT, 0); 91 hv_set_register(HV_REGISTER_STIMER0_CONFIG, 0); 92 if (direct_mode_enabled && stimer0_irq >= 0) 93 disable_percpu_irq(stimer0_irq); 94 95 return 0; 96 } 97 98 static int hv_ce_set_oneshot(struct clock_event_device *evt) 99 { 100 union hv_stimer_config timer_cfg; 101 102 timer_cfg.as_uint64 = 0; 103 timer_cfg.enable = 1; 104 timer_cfg.auto_enable = 1; 105 if (direct_mode_enabled) { 106 /* 107 * When it expires, the timer will directly interrupt 108 * on the specified hardware vector/IRQ. 109 */ 110 timer_cfg.direct_mode = 1; 111 timer_cfg.apic_vector = HYPERV_STIMER0_VECTOR; 112 if (stimer0_irq >= 0) 113 enable_percpu_irq(stimer0_irq, IRQ_TYPE_NONE); 114 } else { 115 /* 116 * When it expires, the timer will generate a VMbus message, 117 * to be handled by the normal VMbus interrupt handler. 118 */ 119 timer_cfg.direct_mode = 0; 120 timer_cfg.sintx = stimer0_message_sint; 121 } 122 hv_set_register(HV_REGISTER_STIMER0_CONFIG, timer_cfg.as_uint64); 123 return 0; 124 } 125 126 /* 127 * hv_stimer_init - Per-cpu initialization of the clockevent 128 */ 129 static int hv_stimer_init(unsigned int cpu) 130 { 131 struct clock_event_device *ce; 132 133 if (!hv_clock_event) 134 return 0; 135 136 ce = per_cpu_ptr(hv_clock_event, cpu); 137 ce->name = "Hyper-V clockevent"; 138 ce->features = CLOCK_EVT_FEAT_ONESHOT; 139 ce->cpumask = cpumask_of(cpu); 140 ce->rating = 1000; 141 ce->set_state_shutdown = hv_ce_shutdown; 142 ce->set_state_oneshot = hv_ce_set_oneshot; 143 ce->set_next_event = hv_ce_set_next_event; 144 145 clockevents_config_and_register(ce, 146 HV_CLOCK_HZ, 147 HV_MIN_DELTA_TICKS, 148 HV_MAX_MAX_DELTA_TICKS); 149 return 0; 150 } 151 152 /* 153 * hv_stimer_cleanup - Per-cpu cleanup of the clockevent 154 */ 155 int hv_stimer_cleanup(unsigned int cpu) 156 { 157 struct clock_event_device *ce; 158 159 if (!hv_clock_event) 160 return 0; 161 162 /* 163 * In the legacy case where Direct Mode is not enabled 164 * (which can only be on x86/64), stimer cleanup happens 165 * relatively early in the CPU offlining process. We 166 * must unbind the stimer-based clockevent device so 167 * that the LAPIC timer can take over until clockevents 168 * are no longer needed in the offlining process. Note 169 * that clockevents_unbind_device() eventually calls 170 * hv_ce_shutdown(). 171 * 172 * The unbind should not be done when Direct Mode is 173 * enabled because we may be on an architecture where 174 * there are no other clockevent devices to fallback to. 175 */ 176 ce = per_cpu_ptr(hv_clock_event, cpu); 177 if (direct_mode_enabled) 178 hv_ce_shutdown(ce); 179 else 180 clockevents_unbind_device(ce, cpu); 181 182 return 0; 183 } 184 EXPORT_SYMBOL_GPL(hv_stimer_cleanup); 185 186 /* 187 * These placeholders are overridden by arch specific code on 188 * architectures that need special setup of the stimer0 IRQ because 189 * they don't support per-cpu IRQs (such as x86/x64). 190 */ 191 void __weak hv_setup_stimer0_handler(void (*handler)(void)) 192 { 193 }; 194 195 void __weak hv_remove_stimer0_handler(void) 196 { 197 }; 198 199 /* Called only on architectures with per-cpu IRQs (i.e., not x86/x64) */ 200 static int hv_setup_stimer0_irq(void) 201 { 202 int ret; 203 204 ret = acpi_register_gsi(NULL, HYPERV_STIMER0_VECTOR, 205 ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH); 206 if (ret < 0) { 207 pr_err("Can't register Hyper-V stimer0 GSI. Error %d", ret); 208 return ret; 209 } 210 stimer0_irq = ret; 211 212 ret = request_percpu_irq(stimer0_irq, hv_stimer0_percpu_isr, 213 "Hyper-V stimer0", &stimer0_evt); 214 if (ret) { 215 pr_err("Can't request Hyper-V stimer0 IRQ %d. Error %d", 216 stimer0_irq, ret); 217 acpi_unregister_gsi(stimer0_irq); 218 stimer0_irq = -1; 219 } 220 return ret; 221 } 222 223 static void hv_remove_stimer0_irq(void) 224 { 225 if (stimer0_irq == -1) { 226 hv_remove_stimer0_handler(); 227 } else { 228 free_percpu_irq(stimer0_irq, &stimer0_evt); 229 acpi_unregister_gsi(stimer0_irq); 230 stimer0_irq = -1; 231 } 232 } 233 234 /* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */ 235 int hv_stimer_alloc(bool have_percpu_irqs) 236 { 237 int ret; 238 239 /* 240 * Synthetic timers are always available except on old versions of 241 * Hyper-V on x86. In that case, return as error as Linux will use a 242 * clockevent based on emulated LAPIC timer hardware. 243 */ 244 if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) 245 return -EINVAL; 246 247 hv_clock_event = alloc_percpu(struct clock_event_device); 248 if (!hv_clock_event) 249 return -ENOMEM; 250 251 direct_mode_enabled = ms_hyperv.misc_features & 252 HV_STIMER_DIRECT_MODE_AVAILABLE; 253 254 /* 255 * If Direct Mode isn't enabled, the remainder of the initialization 256 * is done later by hv_stimer_legacy_init() 257 */ 258 if (!direct_mode_enabled) 259 return 0; 260 261 if (have_percpu_irqs) { 262 ret = hv_setup_stimer0_irq(); 263 if (ret) 264 goto free_clock_event; 265 } else { 266 hv_setup_stimer0_handler(hv_stimer0_isr); 267 } 268 269 /* 270 * Since we are in Direct Mode, stimer initialization 271 * can be done now with a CPUHP value in the same range 272 * as other clockevent devices. 273 */ 274 ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING, 275 "clockevents/hyperv/stimer:starting", 276 hv_stimer_init, hv_stimer_cleanup); 277 if (ret < 0) { 278 hv_remove_stimer0_irq(); 279 goto free_clock_event; 280 } 281 return ret; 282 283 free_clock_event: 284 free_percpu(hv_clock_event); 285 hv_clock_event = NULL; 286 return ret; 287 } 288 EXPORT_SYMBOL_GPL(hv_stimer_alloc); 289 290 /* 291 * hv_stimer_legacy_init -- Called from the VMbus driver to handle 292 * the case when Direct Mode is not enabled, and the stimer 293 * must be initialized late in the CPU onlining process. 294 * 295 */ 296 void hv_stimer_legacy_init(unsigned int cpu, int sint) 297 { 298 if (direct_mode_enabled) 299 return; 300 301 /* 302 * This function gets called by each vCPU, so setting the 303 * global stimer_message_sint value each time is conceptually 304 * not ideal, but the value passed in is always the same and 305 * it avoids introducing yet another interface into this 306 * clocksource driver just to set the sint in the legacy case. 307 */ 308 stimer0_message_sint = sint; 309 (void)hv_stimer_init(cpu); 310 } 311 EXPORT_SYMBOL_GPL(hv_stimer_legacy_init); 312 313 /* 314 * hv_stimer_legacy_cleanup -- Called from the VMbus driver to 315 * handle the case when Direct Mode is not enabled, and the 316 * stimer must be cleaned up early in the CPU offlining 317 * process. 318 */ 319 void hv_stimer_legacy_cleanup(unsigned int cpu) 320 { 321 if (direct_mode_enabled) 322 return; 323 (void)hv_stimer_cleanup(cpu); 324 } 325 EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup); 326 327 /* 328 * Do a global cleanup of clockevents for the cases of kexec and 329 * vmbus exit 330 */ 331 void hv_stimer_global_cleanup(void) 332 { 333 int cpu; 334 335 /* 336 * hv_stime_legacy_cleanup() will stop the stimer if Direct 337 * Mode is not enabled, and fallback to the LAPIC timer. 338 */ 339 for_each_present_cpu(cpu) { 340 hv_stimer_legacy_cleanup(cpu); 341 } 342 343 if (!hv_clock_event) 344 return; 345 346 if (direct_mode_enabled) { 347 cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING); 348 hv_remove_stimer0_irq(); 349 stimer0_irq = -1; 350 } 351 free_percpu(hv_clock_event); 352 hv_clock_event = NULL; 353 354 } 355 EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); 356 357 /* 358 * Code and definitions for the Hyper-V clocksources. Two 359 * clocksources are defined: one that reads the Hyper-V defined MSR, and 360 * the other that uses the TSC reference page feature as defined in the 361 * TLFS. The MSR version is for compatibility with old versions of 362 * Hyper-V and 32-bit x86. The TSC reference page version is preferred. 363 */ 364 365 static union { 366 struct ms_hyperv_tsc_page page; 367 u8 reserved[PAGE_SIZE]; 368 } tsc_pg __aligned(PAGE_SIZE); 369 370 static struct ms_hyperv_tsc_page *tsc_page = &tsc_pg.page; 371 static unsigned long tsc_pfn; 372 373 static unsigned long hv_get_tsc_pfn(void) 374 { 375 return tsc_pfn; 376 } 377 378 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 379 { 380 return tsc_page; 381 } 382 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 383 384 static u64 notrace read_hv_clock_tsc(void) 385 { 386 u64 current_tick = hv_read_tsc_page(hv_get_tsc_page()); 387 388 if (current_tick == U64_MAX) 389 current_tick = hv_get_register(HV_REGISTER_TIME_REF_COUNT); 390 391 return current_tick; 392 } 393 394 static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg) 395 { 396 return read_hv_clock_tsc(); 397 } 398 399 static u64 notrace read_hv_sched_clock_tsc(void) 400 { 401 return (read_hv_clock_tsc() - hv_sched_clock_offset) * 402 (NSEC_PER_SEC / HV_CLOCK_HZ); 403 } 404 405 static void suspend_hv_clock_tsc(struct clocksource *arg) 406 { 407 union hv_reference_tsc_msr tsc_msr; 408 409 /* Disable the TSC page */ 410 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 411 tsc_msr.enable = 0; 412 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 413 } 414 415 416 static void resume_hv_clock_tsc(struct clocksource *arg) 417 { 418 union hv_reference_tsc_msr tsc_msr; 419 420 /* Re-enable the TSC page */ 421 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 422 tsc_msr.enable = 1; 423 tsc_msr.pfn = tsc_pfn; 424 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 425 } 426 427 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 428 static int hv_cs_enable(struct clocksource *cs) 429 { 430 vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); 431 return 0; 432 } 433 #endif 434 435 static struct clocksource hyperv_cs_tsc = { 436 .name = "hyperv_clocksource_tsc_page", 437 .rating = 500, 438 .read = read_hv_clock_tsc_cs, 439 .mask = CLOCKSOURCE_MASK(64), 440 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 441 .suspend= suspend_hv_clock_tsc, 442 .resume = resume_hv_clock_tsc, 443 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 444 .enable = hv_cs_enable, 445 .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK, 446 #else 447 .vdso_clock_mode = VDSO_CLOCKMODE_NONE, 448 #endif 449 }; 450 451 static u64 notrace read_hv_clock_msr(void) 452 { 453 /* 454 * Read the partition counter to get the current tick count. This count 455 * is set to 0 when the partition is created and is incremented in 456 * 100 nanosecond units. 457 */ 458 return hv_get_register(HV_REGISTER_TIME_REF_COUNT); 459 } 460 461 static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg) 462 { 463 return read_hv_clock_msr(); 464 } 465 466 static u64 notrace read_hv_sched_clock_msr(void) 467 { 468 return (read_hv_clock_msr() - hv_sched_clock_offset) * 469 (NSEC_PER_SEC / HV_CLOCK_HZ); 470 } 471 472 static struct clocksource hyperv_cs_msr = { 473 .name = "hyperv_clocksource_msr", 474 .rating = 500, 475 .read = read_hv_clock_msr_cs, 476 .mask = CLOCKSOURCE_MASK(64), 477 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 478 }; 479 480 /* 481 * Reference to pv_ops must be inline so objtool 482 * detection of noinstr violations can work correctly. 483 */ 484 #ifdef CONFIG_GENERIC_SCHED_CLOCK 485 static __always_inline void hv_setup_sched_clock(void *sched_clock) 486 { 487 /* 488 * We're on an architecture with generic sched clock (not x86/x64). 489 * The Hyper-V sched clock read function returns nanoseconds, not 490 * the normal 100ns units of the Hyper-V synthetic clock. 491 */ 492 sched_clock_register(sched_clock, 64, NSEC_PER_SEC); 493 } 494 #elif defined CONFIG_PARAVIRT 495 static __always_inline void hv_setup_sched_clock(void *sched_clock) 496 { 497 /* We're on x86/x64 *and* using PV ops */ 498 paravirt_set_sched_clock(sched_clock); 499 } 500 #else /* !CONFIG_GENERIC_SCHED_CLOCK && !CONFIG_PARAVIRT */ 501 static __always_inline void hv_setup_sched_clock(void *sched_clock) {} 502 #endif /* CONFIG_GENERIC_SCHED_CLOCK */ 503 504 static bool __init hv_init_tsc_clocksource(void) 505 { 506 union hv_reference_tsc_msr tsc_msr; 507 508 if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) 509 return false; 510 511 if (hv_root_partition) 512 return false; 513 514 /* 515 * If Hyper-V offers TSC_INVARIANT, then the virtualized TSC correctly 516 * handles frequency and offset changes due to live migration, 517 * pause/resume, and other VM management operations. So lower the 518 * Hyper-V Reference TSC rating, causing the generic TSC to be used. 519 * TSC_INVARIANT is not offered on ARM64, so the Hyper-V Reference 520 * TSC will be preferred over the virtualized ARM64 arch counter. 521 * While the Hyper-V MSR clocksource won't be used since the 522 * Reference TSC clocksource is present, change its rating as 523 * well for consistency. 524 */ 525 if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) { 526 hyperv_cs_tsc.rating = 250; 527 hyperv_cs_msr.rating = 250; 528 } 529 530 hv_read_reference_counter = read_hv_clock_tsc; 531 tsc_pfn = HVPFN_DOWN(virt_to_phys(tsc_page)); 532 533 /* 534 * The Hyper-V TLFS specifies to preserve the value of reserved 535 * bits in registers. So read the existing value, preserve the 536 * low order 12 bits, and add in the guest physical address 537 * (which already has at least the low 12 bits set to zero since 538 * it is page aligned). Also set the "enable" bit, which is bit 0. 539 */ 540 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 541 tsc_msr.enable = 1; 542 tsc_msr.pfn = tsc_pfn; 543 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 544 545 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 546 547 hv_sched_clock_offset = hv_read_reference_counter(); 548 hv_setup_sched_clock(read_hv_sched_clock_tsc); 549 550 return true; 551 } 552 553 void __init hv_init_clocksource(void) 554 { 555 /* 556 * Try to set up the TSC page clocksource. If it succeeds, we're 557 * done. Otherwise, set up the MSR clocksource. At least one of 558 * these will always be available except on very old versions of 559 * Hyper-V on x86. In that case we won't have a Hyper-V 560 * clocksource, but Linux will still run with a clocksource based 561 * on the emulated PIT or LAPIC timer. 562 */ 563 if (hv_init_tsc_clocksource()) 564 return; 565 566 if (!(ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE)) 567 return; 568 569 hv_read_reference_counter = read_hv_clock_msr; 570 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 571 572 hv_sched_clock_offset = hv_read_reference_counter(); 573 hv_setup_sched_clock(read_hv_sched_clock_msr); 574 } 575