1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Clocksource driver for the synthetic counter and timers 5 * provided by the Hyper-V hypervisor to guest VMs, as described 6 * in the Hyper-V Top Level Functional Spec (TLFS). This driver 7 * is instruction set architecture independent. 8 * 9 * Copyright (C) 2019, Microsoft, Inc. 10 * 11 * Author: Michael Kelley <[email protected]> 12 */ 13 14 #include <linux/percpu.h> 15 #include <linux/cpumask.h> 16 #include <linux/clockchips.h> 17 #include <linux/clocksource.h> 18 #include <linux/sched_clock.h> 19 #include <linux/mm.h> 20 #include <linux/cpuhotplug.h> 21 #include <linux/interrupt.h> 22 #include <linux/irq.h> 23 #include <linux/acpi.h> 24 #include <linux/hyperv.h> 25 #include <clocksource/hyperv_timer.h> 26 #include <asm/hyperv-tlfs.h> 27 #include <asm/mshyperv.h> 28 29 static struct clock_event_device __percpu *hv_clock_event; 30 static u64 hv_sched_clock_offset __ro_after_init; 31 32 /* 33 * If false, we're using the old mechanism for stimer0 interrupts 34 * where it sends a VMbus message when it expires. The old 35 * mechanism is used when running on older versions of Hyper-V 36 * that don't support Direct Mode. While Hyper-V provides 37 * four stimer's per CPU, Linux uses only stimer0. 38 * 39 * Because Direct Mode does not require processing a VMbus 40 * message, stimer interrupts can be enabled earlier in the 41 * process of booting a CPU, and consistent with when timer 42 * interrupts are enabled for other clocksource drivers. 43 * However, for legacy versions of Hyper-V when Direct Mode 44 * is not enabled, setting up stimer interrupts must be 45 * delayed until VMbus is initialized and can process the 46 * interrupt message. 47 */ 48 static bool direct_mode_enabled; 49 50 static int stimer0_irq = -1; 51 static int stimer0_message_sint; 52 static DEFINE_PER_CPU(long, stimer0_evt); 53 54 /* 55 * Common code for stimer0 interrupts coming via Direct Mode or 56 * as a VMbus message. 57 */ 58 void hv_stimer0_isr(void) 59 { 60 struct clock_event_device *ce; 61 62 ce = this_cpu_ptr(hv_clock_event); 63 ce->event_handler(ce); 64 } 65 EXPORT_SYMBOL_GPL(hv_stimer0_isr); 66 67 /* 68 * stimer0 interrupt handler for architectures that support 69 * per-cpu interrupts, which also implies Direct Mode. 70 */ 71 static irqreturn_t hv_stimer0_percpu_isr(int irq, void *dev_id) 72 { 73 hv_stimer0_isr(); 74 return IRQ_HANDLED; 75 } 76 77 static int hv_ce_set_next_event(unsigned long delta, 78 struct clock_event_device *evt) 79 { 80 u64 current_tick; 81 82 current_tick = hv_read_reference_counter(); 83 current_tick += delta; 84 hv_set_register(HV_REGISTER_STIMER0_COUNT, current_tick); 85 return 0; 86 } 87 88 static int hv_ce_shutdown(struct clock_event_device *evt) 89 { 90 hv_set_register(HV_REGISTER_STIMER0_COUNT, 0); 91 hv_set_register(HV_REGISTER_STIMER0_CONFIG, 0); 92 if (direct_mode_enabled && stimer0_irq >= 0) 93 disable_percpu_irq(stimer0_irq); 94 95 return 0; 96 } 97 98 static int hv_ce_set_oneshot(struct clock_event_device *evt) 99 { 100 union hv_stimer_config timer_cfg; 101 102 timer_cfg.as_uint64 = 0; 103 timer_cfg.enable = 1; 104 timer_cfg.auto_enable = 1; 105 if (direct_mode_enabled) { 106 /* 107 * When it expires, the timer will directly interrupt 108 * on the specified hardware vector/IRQ. 109 */ 110 timer_cfg.direct_mode = 1; 111 timer_cfg.apic_vector = HYPERV_STIMER0_VECTOR; 112 if (stimer0_irq >= 0) 113 enable_percpu_irq(stimer0_irq, IRQ_TYPE_NONE); 114 } else { 115 /* 116 * When it expires, the timer will generate a VMbus message, 117 * to be handled by the normal VMbus interrupt handler. 118 */ 119 timer_cfg.direct_mode = 0; 120 timer_cfg.sintx = stimer0_message_sint; 121 } 122 hv_set_register(HV_REGISTER_STIMER0_CONFIG, timer_cfg.as_uint64); 123 return 0; 124 } 125 126 /* 127 * hv_stimer_init - Per-cpu initialization of the clockevent 128 */ 129 static int hv_stimer_init(unsigned int cpu) 130 { 131 struct clock_event_device *ce; 132 133 if (!hv_clock_event) 134 return 0; 135 136 ce = per_cpu_ptr(hv_clock_event, cpu); 137 ce->name = "Hyper-V clockevent"; 138 ce->features = CLOCK_EVT_FEAT_ONESHOT; 139 ce->cpumask = cpumask_of(cpu); 140 ce->rating = 1000; 141 ce->set_state_shutdown = hv_ce_shutdown; 142 ce->set_state_oneshot = hv_ce_set_oneshot; 143 ce->set_next_event = hv_ce_set_next_event; 144 145 clockevents_config_and_register(ce, 146 HV_CLOCK_HZ, 147 HV_MIN_DELTA_TICKS, 148 HV_MAX_MAX_DELTA_TICKS); 149 return 0; 150 } 151 152 /* 153 * hv_stimer_cleanup - Per-cpu cleanup of the clockevent 154 */ 155 int hv_stimer_cleanup(unsigned int cpu) 156 { 157 struct clock_event_device *ce; 158 159 if (!hv_clock_event) 160 return 0; 161 162 /* 163 * In the legacy case where Direct Mode is not enabled 164 * (which can only be on x86/64), stimer cleanup happens 165 * relatively early in the CPU offlining process. We 166 * must unbind the stimer-based clockevent device so 167 * that the LAPIC timer can take over until clockevents 168 * are no longer needed in the offlining process. Note 169 * that clockevents_unbind_device() eventually calls 170 * hv_ce_shutdown(). 171 * 172 * The unbind should not be done when Direct Mode is 173 * enabled because we may be on an architecture where 174 * there are no other clockevent devices to fallback to. 175 */ 176 ce = per_cpu_ptr(hv_clock_event, cpu); 177 if (direct_mode_enabled) 178 hv_ce_shutdown(ce); 179 else 180 clockevents_unbind_device(ce, cpu); 181 182 return 0; 183 } 184 EXPORT_SYMBOL_GPL(hv_stimer_cleanup); 185 186 /* 187 * These placeholders are overridden by arch specific code on 188 * architectures that need special setup of the stimer0 IRQ because 189 * they don't support per-cpu IRQs (such as x86/x64). 190 */ 191 void __weak hv_setup_stimer0_handler(void (*handler)(void)) 192 { 193 }; 194 195 void __weak hv_remove_stimer0_handler(void) 196 { 197 }; 198 199 /* Called only on architectures with per-cpu IRQs (i.e., not x86/x64) */ 200 static int hv_setup_stimer0_irq(void) 201 { 202 int ret; 203 204 ret = acpi_register_gsi(NULL, HYPERV_STIMER0_VECTOR, 205 ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH); 206 if (ret < 0) { 207 pr_err("Can't register Hyper-V stimer0 GSI. Error %d", ret); 208 return ret; 209 } 210 stimer0_irq = ret; 211 212 ret = request_percpu_irq(stimer0_irq, hv_stimer0_percpu_isr, 213 "Hyper-V stimer0", &stimer0_evt); 214 if (ret) { 215 pr_err("Can't request Hyper-V stimer0 IRQ %d. Error %d", 216 stimer0_irq, ret); 217 acpi_unregister_gsi(stimer0_irq); 218 stimer0_irq = -1; 219 } 220 return ret; 221 } 222 223 static void hv_remove_stimer0_irq(void) 224 { 225 if (stimer0_irq == -1) { 226 hv_remove_stimer0_handler(); 227 } else { 228 free_percpu_irq(stimer0_irq, &stimer0_evt); 229 acpi_unregister_gsi(stimer0_irq); 230 stimer0_irq = -1; 231 } 232 } 233 234 /* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */ 235 int hv_stimer_alloc(bool have_percpu_irqs) 236 { 237 int ret; 238 239 /* 240 * Synthetic timers are always available except on old versions of 241 * Hyper-V on x86. In that case, return as error as Linux will use a 242 * clockevent based on emulated LAPIC timer hardware. 243 */ 244 if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) 245 return -EINVAL; 246 247 hv_clock_event = alloc_percpu(struct clock_event_device); 248 if (!hv_clock_event) 249 return -ENOMEM; 250 251 direct_mode_enabled = ms_hyperv.misc_features & 252 HV_STIMER_DIRECT_MODE_AVAILABLE; 253 254 /* 255 * If Direct Mode isn't enabled, the remainder of the initialization 256 * is done later by hv_stimer_legacy_init() 257 */ 258 if (!direct_mode_enabled) 259 return 0; 260 261 if (have_percpu_irqs) { 262 ret = hv_setup_stimer0_irq(); 263 if (ret) 264 goto free_clock_event; 265 } else { 266 hv_setup_stimer0_handler(hv_stimer0_isr); 267 } 268 269 /* 270 * Since we are in Direct Mode, stimer initialization 271 * can be done now with a CPUHP value in the same range 272 * as other clockevent devices. 273 */ 274 ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING, 275 "clockevents/hyperv/stimer:starting", 276 hv_stimer_init, hv_stimer_cleanup); 277 if (ret < 0) { 278 hv_remove_stimer0_irq(); 279 goto free_clock_event; 280 } 281 return ret; 282 283 free_clock_event: 284 free_percpu(hv_clock_event); 285 hv_clock_event = NULL; 286 return ret; 287 } 288 EXPORT_SYMBOL_GPL(hv_stimer_alloc); 289 290 /* 291 * hv_stimer_legacy_init -- Called from the VMbus driver to handle 292 * the case when Direct Mode is not enabled, and the stimer 293 * must be initialized late in the CPU onlining process. 294 * 295 */ 296 void hv_stimer_legacy_init(unsigned int cpu, int sint) 297 { 298 if (direct_mode_enabled) 299 return; 300 301 /* 302 * This function gets called by each vCPU, so setting the 303 * global stimer_message_sint value each time is conceptually 304 * not ideal, but the value passed in is always the same and 305 * it avoids introducing yet another interface into this 306 * clocksource driver just to set the sint in the legacy case. 307 */ 308 stimer0_message_sint = sint; 309 (void)hv_stimer_init(cpu); 310 } 311 EXPORT_SYMBOL_GPL(hv_stimer_legacy_init); 312 313 /* 314 * hv_stimer_legacy_cleanup -- Called from the VMbus driver to 315 * handle the case when Direct Mode is not enabled, and the 316 * stimer must be cleaned up early in the CPU offlining 317 * process. 318 */ 319 void hv_stimer_legacy_cleanup(unsigned int cpu) 320 { 321 if (direct_mode_enabled) 322 return; 323 (void)hv_stimer_cleanup(cpu); 324 } 325 EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup); 326 327 /* 328 * Do a global cleanup of clockevents for the cases of kexec and 329 * vmbus exit 330 */ 331 void hv_stimer_global_cleanup(void) 332 { 333 int cpu; 334 335 /* 336 * hv_stime_legacy_cleanup() will stop the stimer if Direct 337 * Mode is not enabled, and fallback to the LAPIC timer. 338 */ 339 for_each_present_cpu(cpu) { 340 hv_stimer_legacy_cleanup(cpu); 341 } 342 343 if (!hv_clock_event) 344 return; 345 346 if (direct_mode_enabled) { 347 cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING); 348 hv_remove_stimer0_irq(); 349 stimer0_irq = -1; 350 } 351 free_percpu(hv_clock_event); 352 hv_clock_event = NULL; 353 354 } 355 EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); 356 357 /* 358 * Code and definitions for the Hyper-V clocksources. Two 359 * clocksources are defined: one that reads the Hyper-V defined MSR, and 360 * the other that uses the TSC reference page feature as defined in the 361 * TLFS. The MSR version is for compatibility with old versions of 362 * Hyper-V and 32-bit x86. The TSC reference page version is preferred. 363 */ 364 365 static union { 366 struct ms_hyperv_tsc_page page; 367 u8 reserved[PAGE_SIZE]; 368 } tsc_pg __aligned(PAGE_SIZE); 369 370 static struct ms_hyperv_tsc_page *tsc_page = &tsc_pg.page; 371 372 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 373 { 374 return tsc_page; 375 } 376 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 377 378 static u64 notrace read_hv_clock_tsc(void) 379 { 380 u64 current_tick = hv_read_tsc_page(hv_get_tsc_page()); 381 382 if (current_tick == U64_MAX) 383 current_tick = hv_get_register(HV_REGISTER_TIME_REF_COUNT); 384 385 return current_tick; 386 } 387 388 static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg) 389 { 390 return read_hv_clock_tsc(); 391 } 392 393 static u64 notrace read_hv_sched_clock_tsc(void) 394 { 395 return (read_hv_clock_tsc() - hv_sched_clock_offset) * 396 (NSEC_PER_SEC / HV_CLOCK_HZ); 397 } 398 399 static void suspend_hv_clock_tsc(struct clocksource *arg) 400 { 401 union hv_reference_tsc_msr tsc_msr; 402 403 /* Disable the TSC page */ 404 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 405 tsc_msr.enable = 0; 406 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 407 } 408 409 410 static void resume_hv_clock_tsc(struct clocksource *arg) 411 { 412 phys_addr_t phys_addr = virt_to_phys(tsc_page); 413 union hv_reference_tsc_msr tsc_msr; 414 415 /* Re-enable the TSC page */ 416 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 417 tsc_msr.enable = 1; 418 tsc_msr.pfn = HVPFN_DOWN(phys_addr); 419 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 420 } 421 422 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 423 static int hv_cs_enable(struct clocksource *cs) 424 { 425 vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); 426 return 0; 427 } 428 #endif 429 430 static struct clocksource hyperv_cs_tsc = { 431 .name = "hyperv_clocksource_tsc_page", 432 .rating = 500, 433 .read = read_hv_clock_tsc_cs, 434 .mask = CLOCKSOURCE_MASK(64), 435 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 436 .suspend= suspend_hv_clock_tsc, 437 .resume = resume_hv_clock_tsc, 438 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 439 .enable = hv_cs_enable, 440 .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK, 441 #else 442 .vdso_clock_mode = VDSO_CLOCKMODE_NONE, 443 #endif 444 }; 445 446 static u64 notrace read_hv_clock_msr(void) 447 { 448 /* 449 * Read the partition counter to get the current tick count. This count 450 * is set to 0 when the partition is created and is incremented in 451 * 100 nanosecond units. 452 */ 453 return hv_get_register(HV_REGISTER_TIME_REF_COUNT); 454 } 455 456 static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg) 457 { 458 return read_hv_clock_msr(); 459 } 460 461 static u64 notrace read_hv_sched_clock_msr(void) 462 { 463 return (read_hv_clock_msr() - hv_sched_clock_offset) * 464 (NSEC_PER_SEC / HV_CLOCK_HZ); 465 } 466 467 static struct clocksource hyperv_cs_msr = { 468 .name = "hyperv_clocksource_msr", 469 .rating = 500, 470 .read = read_hv_clock_msr_cs, 471 .mask = CLOCKSOURCE_MASK(64), 472 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 473 }; 474 475 /* 476 * Reference to pv_ops must be inline so objtool 477 * detection of noinstr violations can work correctly. 478 */ 479 #ifdef CONFIG_GENERIC_SCHED_CLOCK 480 static __always_inline void hv_setup_sched_clock(void *sched_clock) 481 { 482 /* 483 * We're on an architecture with generic sched clock (not x86/x64). 484 * The Hyper-V sched clock read function returns nanoseconds, not 485 * the normal 100ns units of the Hyper-V synthetic clock. 486 */ 487 sched_clock_register(sched_clock, 64, NSEC_PER_SEC); 488 } 489 #elif defined CONFIG_PARAVIRT 490 static __always_inline void hv_setup_sched_clock(void *sched_clock) 491 { 492 /* We're on x86/x64 *and* using PV ops */ 493 paravirt_set_sched_clock(sched_clock); 494 } 495 #else /* !CONFIG_GENERIC_SCHED_CLOCK && !CONFIG_PARAVIRT */ 496 static __always_inline void hv_setup_sched_clock(void *sched_clock) {} 497 #endif /* CONFIG_GENERIC_SCHED_CLOCK */ 498 499 static bool __init hv_init_tsc_clocksource(void) 500 { 501 union hv_reference_tsc_msr tsc_msr; 502 phys_addr_t phys_addr; 503 504 if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) 505 return false; 506 507 if (hv_root_partition) 508 return false; 509 510 /* 511 * If Hyper-V offers TSC_INVARIANT, then the virtualized TSC correctly 512 * handles frequency and offset changes due to live migration, 513 * pause/resume, and other VM management operations. So lower the 514 * Hyper-V Reference TSC rating, causing the generic TSC to be used. 515 * TSC_INVARIANT is not offered on ARM64, so the Hyper-V Reference 516 * TSC will be preferred over the virtualized ARM64 arch counter. 517 * While the Hyper-V MSR clocksource won't be used since the 518 * Reference TSC clocksource is present, change its rating as 519 * well for consistency. 520 */ 521 if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) { 522 hyperv_cs_tsc.rating = 250; 523 hyperv_cs_msr.rating = 250; 524 } 525 526 hv_read_reference_counter = read_hv_clock_tsc; 527 phys_addr = virt_to_phys(hv_get_tsc_page()); 528 529 /* 530 * The Hyper-V TLFS specifies to preserve the value of reserved 531 * bits in registers. So read the existing value, preserve the 532 * low order 12 bits, and add in the guest physical address 533 * (which already has at least the low 12 bits set to zero since 534 * it is page aligned). Also set the "enable" bit, which is bit 0. 535 */ 536 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 537 tsc_msr.enable = 1; 538 tsc_msr.pfn = HVPFN_DOWN(phys_addr); 539 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 540 541 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 542 543 hv_sched_clock_offset = hv_read_reference_counter(); 544 hv_setup_sched_clock(read_hv_sched_clock_tsc); 545 546 return true; 547 } 548 549 void __init hv_init_clocksource(void) 550 { 551 /* 552 * Try to set up the TSC page clocksource. If it succeeds, we're 553 * done. Otherwise, set up the MSR clocksource. At least one of 554 * these will always be available except on very old versions of 555 * Hyper-V on x86. In that case we won't have a Hyper-V 556 * clocksource, but Linux will still run with a clocksource based 557 * on the emulated PIT or LAPIC timer. 558 */ 559 if (hv_init_tsc_clocksource()) 560 return; 561 562 if (!(ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE)) 563 return; 564 565 hv_read_reference_counter = read_hv_clock_msr; 566 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 567 568 hv_sched_clock_offset = hv_read_reference_counter(); 569 hv_setup_sched_clock(read_hv_sched_clock_msr); 570 } 571