1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* 45 * Default maximum number of loops with no packets consumed i.e. stuck in a 46 * loop. 47 */ 48 #define INTEL_PT_MAX_LOOPS 100000 49 50 struct intel_pt_blk { 51 struct intel_pt_blk *prev; 52 uint64_t ip[INTEL_PT_BLK_SIZE]; 53 }; 54 55 struct intel_pt_stack { 56 struct intel_pt_blk *blk; 57 struct intel_pt_blk *spare; 58 int pos; 59 }; 60 61 enum intel_pt_p_once { 62 INTEL_PT_PRT_ONCE_UNK_VMCS, 63 INTEL_PT_PRT_ONCE_ERANGE, 64 }; 65 66 enum intel_pt_pkt_state { 67 INTEL_PT_STATE_NO_PSB, 68 INTEL_PT_STATE_NO_IP, 69 INTEL_PT_STATE_ERR_RESYNC, 70 INTEL_PT_STATE_IN_SYNC, 71 INTEL_PT_STATE_TNT_CONT, 72 INTEL_PT_STATE_TNT, 73 INTEL_PT_STATE_TIP, 74 INTEL_PT_STATE_TIP_PGD, 75 INTEL_PT_STATE_FUP, 76 INTEL_PT_STATE_FUP_NO_TIP, 77 INTEL_PT_STATE_FUP_IN_PSB, 78 INTEL_PT_STATE_RESAMPLE, 79 INTEL_PT_STATE_VM_TIME_CORRELATION, 80 }; 81 82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 83 { 84 switch (pkt_state) { 85 case INTEL_PT_STATE_NO_PSB: 86 case INTEL_PT_STATE_NO_IP: 87 case INTEL_PT_STATE_ERR_RESYNC: 88 case INTEL_PT_STATE_IN_SYNC: 89 case INTEL_PT_STATE_TNT_CONT: 90 case INTEL_PT_STATE_RESAMPLE: 91 case INTEL_PT_STATE_VM_TIME_CORRELATION: 92 return true; 93 case INTEL_PT_STATE_TNT: 94 case INTEL_PT_STATE_TIP: 95 case INTEL_PT_STATE_TIP_PGD: 96 case INTEL_PT_STATE_FUP: 97 case INTEL_PT_STATE_FUP_NO_TIP: 98 case INTEL_PT_STATE_FUP_IN_PSB: 99 return false; 100 default: 101 return true; 102 }; 103 } 104 105 #ifdef INTEL_PT_STRICT 106 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 107 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 108 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 109 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 110 #else 111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 112 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 113 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 114 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 115 #endif 116 117 struct intel_pt_decoder { 118 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 119 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 120 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 121 uint64_t max_insn_cnt, void *data); 122 bool (*pgd_ip)(uint64_t ip, void *data); 123 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 124 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 125 void *data; 126 struct intel_pt_state state; 127 const unsigned char *buf; 128 size_t len; 129 bool return_compression; 130 bool branch_enable; 131 bool mtc_insn; 132 bool pge; 133 bool have_tma; 134 bool have_cyc; 135 bool fixup_last_mtc; 136 bool have_last_ip; 137 bool in_psb; 138 bool hop; 139 bool leap; 140 bool vm_time_correlation; 141 bool vm_tm_corr_dry_run; 142 bool vm_tm_corr_reliable; 143 bool vm_tm_corr_same_buf; 144 bool vm_tm_corr_continuous; 145 bool nr; 146 bool next_nr; 147 enum intel_pt_param_flags flags; 148 uint64_t pos; 149 uint64_t last_ip; 150 uint64_t ip; 151 uint64_t pip_payload; 152 uint64_t timestamp; 153 uint64_t tsc_timestamp; 154 uint64_t ref_timestamp; 155 uint64_t buf_timestamp; 156 uint64_t sample_timestamp; 157 uint64_t ret_addr; 158 uint64_t ctc_timestamp; 159 uint64_t ctc_delta; 160 uint64_t cycle_cnt; 161 uint64_t cyc_ref_timestamp; 162 uint64_t first_timestamp; 163 uint64_t last_reliable_timestamp; 164 uint64_t vmcs; 165 uint64_t print_once; 166 uint64_t last_ctc; 167 uint32_t last_mtc; 168 uint32_t tsc_ctc_ratio_n; 169 uint32_t tsc_ctc_ratio_d; 170 uint32_t tsc_ctc_mult; 171 uint32_t tsc_slip; 172 uint32_t ctc_rem_mask; 173 int mtc_shift; 174 struct intel_pt_stack stack; 175 enum intel_pt_pkt_state pkt_state; 176 enum intel_pt_pkt_ctx pkt_ctx; 177 enum intel_pt_pkt_ctx prev_pkt_ctx; 178 enum intel_pt_blk_type blk_type; 179 int blk_type_pos; 180 struct intel_pt_pkt packet; 181 struct intel_pt_pkt tnt; 182 int pkt_step; 183 int pkt_len; 184 int last_packet_type; 185 unsigned int cbr; 186 unsigned int cbr_seen; 187 unsigned int max_non_turbo_ratio; 188 double max_non_turbo_ratio_fp; 189 double cbr_cyc_to_tsc; 190 double calc_cyc_to_tsc; 191 bool have_calc_cyc_to_tsc; 192 int exec_mode; 193 unsigned int insn_bytes; 194 uint64_t period; 195 enum intel_pt_period_type period_type; 196 uint64_t tot_insn_cnt; 197 uint64_t period_insn_cnt; 198 uint64_t period_mask; 199 uint64_t period_ticks; 200 uint64_t last_masked_timestamp; 201 uint64_t tot_cyc_cnt; 202 uint64_t sample_tot_cyc_cnt; 203 uint64_t base_cyc_cnt; 204 uint64_t cyc_cnt_timestamp; 205 uint64_t ctl; 206 uint64_t cyc_threshold; 207 double tsc_to_cyc; 208 bool continuous_period; 209 bool overflow; 210 bool set_fup_tx_flags; 211 bool set_fup_ptw; 212 bool set_fup_mwait; 213 bool set_fup_pwre; 214 bool set_fup_exstop; 215 bool set_fup_bep; 216 bool sample_cyc; 217 unsigned int fup_tx_flags; 218 unsigned int tx_flags; 219 uint64_t fup_ptw_payload; 220 uint64_t fup_mwait_payload; 221 uint64_t fup_pwre_payload; 222 uint64_t cbr_payload; 223 uint64_t timestamp_insn_cnt; 224 uint64_t sample_insn_cnt; 225 uint64_t stuck_ip; 226 int max_loops; 227 int no_progress; 228 int stuck_ip_prd; 229 int stuck_ip_cnt; 230 uint64_t psb_ip; 231 const unsigned char *next_buf; 232 size_t next_len; 233 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 234 }; 235 236 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 237 { 238 int i; 239 240 for (i = 0; x != 1; i++) 241 x >>= 1; 242 243 return x << i; 244 } 245 246 __printf(1, 2) 247 static void p_log(const char *fmt, ...) 248 { 249 char buf[512]; 250 va_list args; 251 252 va_start(args, fmt); 253 vsnprintf(buf, sizeof(buf), fmt, args); 254 va_end(args); 255 256 fprintf(stderr, "%s\n", buf); 257 intel_pt_log("%s\n", buf); 258 } 259 260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 261 enum intel_pt_p_once id) 262 { 263 uint64_t bit = 1ULL << id; 264 265 if (decoder->print_once & bit) 266 return false; 267 decoder->print_once |= bit; 268 return true; 269 } 270 271 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 272 { 273 if (!(ctl & INTEL_PT_CYC_ENABLE)) 274 return 0; 275 276 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 277 } 278 279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 280 { 281 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 282 uint64_t period; 283 284 period = intel_pt_lower_power_of_2(decoder->period); 285 decoder->period_mask = ~(period - 1); 286 decoder->period_ticks = period; 287 } 288 } 289 290 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 291 { 292 if (!d) 293 return 0; 294 return (t / d) * n + ((t % d) * n) / d; 295 } 296 297 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 298 { 299 struct intel_pt_decoder *decoder; 300 301 if (!params->get_trace || !params->walk_insn) 302 return NULL; 303 304 decoder = zalloc(sizeof(struct intel_pt_decoder)); 305 if (!decoder) 306 return NULL; 307 308 decoder->get_trace = params->get_trace; 309 decoder->walk_insn = params->walk_insn; 310 decoder->pgd_ip = params->pgd_ip; 311 decoder->lookahead = params->lookahead; 312 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 313 decoder->data = params->data; 314 decoder->return_compression = params->return_compression; 315 decoder->branch_enable = params->branch_enable; 316 decoder->hop = params->quick >= 1; 317 decoder->leap = params->quick >= 2; 318 decoder->vm_time_correlation = params->vm_time_correlation; 319 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 320 decoder->first_timestamp = params->first_timestamp; 321 decoder->last_reliable_timestamp = params->first_timestamp; 322 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; 323 324 decoder->flags = params->flags; 325 326 decoder->ctl = params->ctl; 327 decoder->period = params->period; 328 decoder->period_type = params->period_type; 329 330 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 331 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 332 333 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 334 335 intel_pt_setup_period(decoder); 336 337 decoder->mtc_shift = params->mtc_period; 338 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 339 340 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 341 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 342 343 if (!decoder->tsc_ctc_ratio_n) 344 decoder->tsc_ctc_ratio_d = 0; 345 346 if (decoder->tsc_ctc_ratio_d) { 347 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 348 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 349 decoder->tsc_ctc_ratio_d; 350 } 351 352 /* 353 * A TSC packet can slip past MTC packets so that the timestamp appears 354 * to go backwards. One estimate is that can be up to about 40 CPU 355 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 356 * slippage an order of magnitude more to be on the safe side. 357 */ 358 decoder->tsc_slip = 0x10000; 359 360 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 361 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 362 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 363 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 364 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 365 366 if (decoder->hop) 367 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 368 369 return decoder; 370 } 371 372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 373 uint64_t first_timestamp) 374 { 375 decoder->first_timestamp = first_timestamp; 376 } 377 378 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 379 { 380 struct intel_pt_blk *blk = stack->blk; 381 382 stack->blk = blk->prev; 383 if (!stack->spare) 384 stack->spare = blk; 385 else 386 free(blk); 387 } 388 389 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 390 { 391 if (!stack->pos) { 392 if (!stack->blk) 393 return 0; 394 intel_pt_pop_blk(stack); 395 if (!stack->blk) 396 return 0; 397 stack->pos = INTEL_PT_BLK_SIZE; 398 } 399 return stack->blk->ip[--stack->pos]; 400 } 401 402 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 403 { 404 struct intel_pt_blk *blk; 405 406 if (stack->spare) { 407 blk = stack->spare; 408 stack->spare = NULL; 409 } else { 410 blk = malloc(sizeof(struct intel_pt_blk)); 411 if (!blk) 412 return -ENOMEM; 413 } 414 415 blk->prev = stack->blk; 416 stack->blk = blk; 417 stack->pos = 0; 418 return 0; 419 } 420 421 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 422 { 423 int err; 424 425 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 426 err = intel_pt_alloc_blk(stack); 427 if (err) 428 return err; 429 } 430 431 stack->blk->ip[stack->pos++] = ip; 432 return 0; 433 } 434 435 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 436 { 437 while (stack->blk) 438 intel_pt_pop_blk(stack); 439 stack->pos = 0; 440 } 441 442 static void intel_pt_free_stack(struct intel_pt_stack *stack) 443 { 444 intel_pt_clear_stack(stack); 445 zfree(&stack->blk); 446 zfree(&stack->spare); 447 } 448 449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 450 { 451 intel_pt_free_stack(&decoder->stack); 452 free(decoder); 453 } 454 455 static int intel_pt_ext_err(int code) 456 { 457 switch (code) { 458 case -ENOMEM: 459 return INTEL_PT_ERR_NOMEM; 460 case -ENOSYS: 461 return INTEL_PT_ERR_INTERN; 462 case -EBADMSG: 463 return INTEL_PT_ERR_BADPKT; 464 case -ENODATA: 465 return INTEL_PT_ERR_NODATA; 466 case -EILSEQ: 467 return INTEL_PT_ERR_NOINSN; 468 case -ENOENT: 469 return INTEL_PT_ERR_MISMAT; 470 case -EOVERFLOW: 471 return INTEL_PT_ERR_OVR; 472 case -ENOSPC: 473 return INTEL_PT_ERR_LOST; 474 case -ELOOP: 475 return INTEL_PT_ERR_NELOOP; 476 default: 477 return INTEL_PT_ERR_UNK; 478 } 479 } 480 481 static const char *intel_pt_err_msgs[] = { 482 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 483 [INTEL_PT_ERR_INTERN] = "Internal error", 484 [INTEL_PT_ERR_BADPKT] = "Bad packet", 485 [INTEL_PT_ERR_NODATA] = "No more data", 486 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 487 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 488 [INTEL_PT_ERR_OVR] = "Overflow packet", 489 [INTEL_PT_ERR_LOST] = "Lost trace data", 490 [INTEL_PT_ERR_UNK] = "Unknown error!", 491 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", 492 }; 493 494 int intel_pt__strerror(int code, char *buf, size_t buflen) 495 { 496 if (code < 1 || code >= INTEL_PT_ERR_MAX) 497 code = INTEL_PT_ERR_UNK; 498 strlcpy(buf, intel_pt_err_msgs[code], buflen); 499 return 0; 500 } 501 502 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 503 uint64_t last_ip) 504 { 505 uint64_t ip; 506 507 switch (packet->count) { 508 case 1: 509 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 510 packet->payload; 511 break; 512 case 2: 513 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 514 packet->payload; 515 break; 516 case 3: 517 ip = packet->payload; 518 /* Sign-extend 6-byte ip */ 519 if (ip & (uint64_t)0x800000000000ULL) 520 ip |= (uint64_t)0xffff000000000000ULL; 521 break; 522 case 4: 523 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 524 packet->payload; 525 break; 526 case 6: 527 ip = packet->payload; 528 break; 529 default: 530 return 0; 531 } 532 533 return ip; 534 } 535 536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 537 { 538 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 539 decoder->have_last_ip = true; 540 } 541 542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 543 { 544 intel_pt_set_last_ip(decoder); 545 decoder->ip = decoder->last_ip; 546 } 547 548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 549 { 550 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 551 decoder->buf); 552 } 553 554 static int intel_pt_bug(struct intel_pt_decoder *decoder) 555 { 556 intel_pt_log("ERROR: Internal error\n"); 557 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 558 return -ENOSYS; 559 } 560 561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 562 { 563 decoder->tx_flags = 0; 564 } 565 566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 567 { 568 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 569 } 570 571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 572 { 573 decoder->pip_payload = decoder->packet.payload; 574 } 575 576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 577 { 578 decoder->next_nr = decoder->pip_payload & 1; 579 } 580 581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 582 { 583 decoder->nr = decoder->pip_payload & 1; 584 decoder->next_nr = decoder->nr; 585 } 586 587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 588 { 589 intel_pt_update_pip(decoder); 590 intel_pt_set_nr(decoder); 591 } 592 593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 594 { 595 intel_pt_clear_tx_flags(decoder); 596 decoder->have_tma = false; 597 decoder->pkt_len = 1; 598 decoder->pkt_step = 1; 599 intel_pt_decoder_log_packet(decoder); 600 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 601 intel_pt_log("ERROR: Bad packet\n"); 602 decoder->pkt_state = INTEL_PT_STATE_ERR1; 603 } 604 return -EBADMSG; 605 } 606 607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 608 { 609 decoder->sample_timestamp = decoder->timestamp; 610 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 611 decoder->state.cycles = decoder->tot_cyc_cnt; 612 } 613 614 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 615 { 616 decoder->ip = 0; 617 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 618 decoder->timestamp = 0; 619 decoder->have_tma = false; 620 } 621 622 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 623 { 624 struct intel_pt_buffer buffer = { .buf = 0, }; 625 int ret; 626 627 decoder->pkt_step = 0; 628 629 intel_pt_log("Getting more data\n"); 630 ret = decoder->get_trace(&buffer, decoder->data); 631 if (ret) 632 return ret; 633 decoder->buf = buffer.buf; 634 decoder->len = buffer.len; 635 if (!decoder->len) { 636 intel_pt_log("No more data\n"); 637 return -ENODATA; 638 } 639 decoder->buf_timestamp = buffer.ref_timestamp; 640 if (!buffer.consecutive || reposition) { 641 intel_pt_reposition(decoder); 642 decoder->ref_timestamp = buffer.ref_timestamp; 643 decoder->state.trace_nr = buffer.trace_nr; 644 decoder->vm_tm_corr_same_buf = false; 645 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 646 decoder->ref_timestamp); 647 return -ENOLINK; 648 } 649 650 return 0; 651 } 652 653 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 654 bool reposition) 655 { 656 if (!decoder->next_buf) 657 return intel_pt_get_data(decoder, reposition); 658 659 decoder->buf = decoder->next_buf; 660 decoder->len = decoder->next_len; 661 decoder->next_buf = 0; 662 decoder->next_len = 0; 663 return 0; 664 } 665 666 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 667 { 668 unsigned char *buf = decoder->temp_buf; 669 size_t old_len, len, n; 670 int ret; 671 672 old_len = decoder->len; 673 len = decoder->len; 674 memcpy(buf, decoder->buf, len); 675 676 ret = intel_pt_get_data(decoder, false); 677 if (ret) { 678 decoder->pos += old_len; 679 return ret < 0 ? ret : -EINVAL; 680 } 681 682 n = INTEL_PT_PKT_MAX_SZ - len; 683 if (n > decoder->len) 684 n = decoder->len; 685 memcpy(buf + len, decoder->buf, n); 686 len += n; 687 688 decoder->prev_pkt_ctx = decoder->pkt_ctx; 689 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 690 if (ret < (int)old_len) { 691 decoder->next_buf = decoder->buf; 692 decoder->next_len = decoder->len; 693 decoder->buf = buf; 694 decoder->len = old_len; 695 return intel_pt_bad_packet(decoder); 696 } 697 698 decoder->next_buf = decoder->buf + (ret - old_len); 699 decoder->next_len = decoder->len - (ret - old_len); 700 701 decoder->buf = buf; 702 decoder->len = ret; 703 704 return ret; 705 } 706 707 struct intel_pt_pkt_info { 708 struct intel_pt_decoder *decoder; 709 struct intel_pt_pkt packet; 710 uint64_t pos; 711 int pkt_len; 712 int last_packet_type; 713 void *data; 714 }; 715 716 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 717 718 /* Lookahead packets in current buffer */ 719 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 720 intel_pt_pkt_cb_t cb, void *data) 721 { 722 struct intel_pt_pkt_info pkt_info; 723 const unsigned char *buf = decoder->buf; 724 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 725 size_t len = decoder->len; 726 int ret; 727 728 pkt_info.decoder = decoder; 729 pkt_info.pos = decoder->pos; 730 pkt_info.pkt_len = decoder->pkt_step; 731 pkt_info.last_packet_type = decoder->last_packet_type; 732 pkt_info.data = data; 733 734 while (1) { 735 do { 736 pkt_info.pos += pkt_info.pkt_len; 737 buf += pkt_info.pkt_len; 738 len -= pkt_info.pkt_len; 739 740 if (!len) 741 return INTEL_PT_NEED_MORE_BYTES; 742 743 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 744 &pkt_ctx); 745 if (!ret) 746 return INTEL_PT_NEED_MORE_BYTES; 747 if (ret < 0) 748 return ret; 749 750 pkt_info.pkt_len = ret; 751 } while (pkt_info.packet.type == INTEL_PT_PAD); 752 753 ret = cb(&pkt_info); 754 if (ret) 755 return 0; 756 757 pkt_info.last_packet_type = pkt_info.packet.type; 758 } 759 } 760 761 struct intel_pt_calc_cyc_to_tsc_info { 762 uint64_t cycle_cnt; 763 unsigned int cbr; 764 uint32_t last_mtc; 765 uint64_t ctc_timestamp; 766 uint64_t ctc_delta; 767 uint64_t tsc_timestamp; 768 uint64_t timestamp; 769 bool have_tma; 770 bool fixup_last_mtc; 771 bool from_mtc; 772 double cbr_cyc_to_tsc; 773 }; 774 775 /* 776 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 777 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 778 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 779 * packet by copying the missing bits from the current MTC assuming the least 780 * difference between the two, and that the current MTC comes after last_mtc. 781 */ 782 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 783 uint32_t *last_mtc) 784 { 785 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 786 uint32_t mask = ~(first_missing_bit - 1); 787 788 *last_mtc |= mtc & mask; 789 if (*last_mtc >= mtc) { 790 *last_mtc -= first_missing_bit; 791 *last_mtc &= 0xff; 792 } 793 } 794 795 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 796 { 797 struct intel_pt_decoder *decoder = pkt_info->decoder; 798 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 799 uint64_t timestamp; 800 double cyc_to_tsc; 801 unsigned int cbr; 802 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 803 804 switch (pkt_info->packet.type) { 805 case INTEL_PT_TNT: 806 case INTEL_PT_TIP_PGE: 807 case INTEL_PT_TIP: 808 case INTEL_PT_FUP: 809 case INTEL_PT_PSB: 810 case INTEL_PT_PIP: 811 case INTEL_PT_MODE_EXEC: 812 case INTEL_PT_MODE_TSX: 813 case INTEL_PT_PSBEND: 814 case INTEL_PT_PAD: 815 case INTEL_PT_VMCS: 816 case INTEL_PT_MNT: 817 case INTEL_PT_PTWRITE: 818 case INTEL_PT_PTWRITE_IP: 819 case INTEL_PT_BBP: 820 case INTEL_PT_BIP: 821 case INTEL_PT_BEP: 822 case INTEL_PT_BEP_IP: 823 return 0; 824 825 case INTEL_PT_MTC: 826 if (!data->have_tma) 827 return 0; 828 829 mtc = pkt_info->packet.payload; 830 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 831 data->fixup_last_mtc = false; 832 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 833 &data->last_mtc); 834 } 835 if (mtc > data->last_mtc) 836 mtc_delta = mtc - data->last_mtc; 837 else 838 mtc_delta = mtc + 256 - data->last_mtc; 839 data->ctc_delta += mtc_delta << decoder->mtc_shift; 840 data->last_mtc = mtc; 841 842 if (decoder->tsc_ctc_mult) { 843 timestamp = data->ctc_timestamp + 844 data->ctc_delta * decoder->tsc_ctc_mult; 845 } else { 846 timestamp = data->ctc_timestamp + 847 multdiv(data->ctc_delta, 848 decoder->tsc_ctc_ratio_n, 849 decoder->tsc_ctc_ratio_d); 850 } 851 852 if (timestamp < data->timestamp) 853 return 1; 854 855 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 856 data->timestamp = timestamp; 857 return 0; 858 } 859 860 break; 861 862 case INTEL_PT_TSC: 863 /* 864 * For now, do not support using TSC packets - refer 865 * intel_pt_calc_cyc_to_tsc(). 866 */ 867 if (data->from_mtc) 868 return 1; 869 timestamp = pkt_info->packet.payload | 870 (data->timestamp & (0xffULL << 56)); 871 if (data->from_mtc && timestamp < data->timestamp && 872 data->timestamp - timestamp < decoder->tsc_slip) 873 return 1; 874 if (timestamp < data->timestamp) 875 timestamp += (1ULL << 56); 876 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 877 if (data->from_mtc) 878 return 1; 879 data->tsc_timestamp = timestamp; 880 data->timestamp = timestamp; 881 return 0; 882 } 883 break; 884 885 case INTEL_PT_TMA: 886 if (data->from_mtc) 887 return 1; 888 889 if (!decoder->tsc_ctc_ratio_d) 890 return 0; 891 892 ctc = pkt_info->packet.payload; 893 fc = pkt_info->packet.count; 894 ctc_rem = ctc & decoder->ctc_rem_mask; 895 896 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 897 898 data->ctc_timestamp = data->tsc_timestamp - fc; 899 if (decoder->tsc_ctc_mult) { 900 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 901 } else { 902 data->ctc_timestamp -= 903 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 904 decoder->tsc_ctc_ratio_d); 905 } 906 907 data->ctc_delta = 0; 908 data->have_tma = true; 909 data->fixup_last_mtc = true; 910 911 return 0; 912 913 case INTEL_PT_CYC: 914 data->cycle_cnt += pkt_info->packet.payload; 915 return 0; 916 917 case INTEL_PT_CBR: 918 cbr = pkt_info->packet.payload; 919 if (data->cbr && data->cbr != cbr) 920 return 1; 921 data->cbr = cbr; 922 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 923 return 0; 924 925 case INTEL_PT_TIP_PGD: 926 case INTEL_PT_TRACESTOP: 927 case INTEL_PT_EXSTOP: 928 case INTEL_PT_EXSTOP_IP: 929 case INTEL_PT_MWAIT: 930 case INTEL_PT_PWRE: 931 case INTEL_PT_PWRX: 932 case INTEL_PT_OVF: 933 case INTEL_PT_BAD: /* Does not happen */ 934 default: 935 return 1; 936 } 937 938 if (!data->cbr && decoder->cbr) { 939 data->cbr = decoder->cbr; 940 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 941 } 942 943 if (!data->cycle_cnt) 944 return 1; 945 946 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 947 948 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 949 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 950 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 951 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 952 return 1; 953 } 954 955 decoder->calc_cyc_to_tsc = cyc_to_tsc; 956 decoder->have_calc_cyc_to_tsc = true; 957 958 if (data->cbr) { 959 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 960 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 961 } else { 962 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 963 cyc_to_tsc, pkt_info->pos); 964 } 965 966 return 1; 967 } 968 969 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 970 bool from_mtc) 971 { 972 struct intel_pt_calc_cyc_to_tsc_info data = { 973 .cycle_cnt = 0, 974 .cbr = 0, 975 .last_mtc = decoder->last_mtc, 976 .ctc_timestamp = decoder->ctc_timestamp, 977 .ctc_delta = decoder->ctc_delta, 978 .tsc_timestamp = decoder->tsc_timestamp, 979 .timestamp = decoder->timestamp, 980 .have_tma = decoder->have_tma, 981 .fixup_last_mtc = decoder->fixup_last_mtc, 982 .from_mtc = from_mtc, 983 .cbr_cyc_to_tsc = 0, 984 }; 985 986 /* 987 * For now, do not support using TSC packets for at least the reasons: 988 * 1) timing might have stopped 989 * 2) TSC packets within PSB+ can slip against CYC packets 990 */ 991 if (!from_mtc) 992 return; 993 994 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 995 } 996 997 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 998 { 999 int ret; 1000 1001 decoder->last_packet_type = decoder->packet.type; 1002 1003 do { 1004 decoder->pos += decoder->pkt_step; 1005 decoder->buf += decoder->pkt_step; 1006 decoder->len -= decoder->pkt_step; 1007 1008 if (!decoder->len) { 1009 ret = intel_pt_get_next_data(decoder, false); 1010 if (ret) 1011 return ret; 1012 } 1013 1014 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1015 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1016 &decoder->packet, &decoder->pkt_ctx); 1017 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1018 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1019 ret = intel_pt_get_split_packet(decoder); 1020 if (ret < 0) 1021 return ret; 1022 } 1023 if (ret <= 0) 1024 return intel_pt_bad_packet(decoder); 1025 1026 decoder->pkt_len = ret; 1027 decoder->pkt_step = ret; 1028 intel_pt_decoder_log_packet(decoder); 1029 } while (decoder->packet.type == INTEL_PT_PAD); 1030 1031 return 0; 1032 } 1033 1034 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1035 { 1036 uint64_t timestamp, masked_timestamp; 1037 1038 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1039 masked_timestamp = timestamp & decoder->period_mask; 1040 if (decoder->continuous_period) { 1041 if (masked_timestamp > decoder->last_masked_timestamp) 1042 return 1; 1043 } else { 1044 timestamp += 1; 1045 masked_timestamp = timestamp & decoder->period_mask; 1046 if (masked_timestamp > decoder->last_masked_timestamp) { 1047 decoder->last_masked_timestamp = masked_timestamp; 1048 decoder->continuous_period = true; 1049 } 1050 } 1051 1052 if (masked_timestamp < decoder->last_masked_timestamp) 1053 return decoder->period_ticks; 1054 1055 return decoder->period_ticks - (timestamp - masked_timestamp); 1056 } 1057 1058 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1059 { 1060 switch (decoder->period_type) { 1061 case INTEL_PT_PERIOD_INSTRUCTIONS: 1062 return decoder->period - decoder->period_insn_cnt; 1063 case INTEL_PT_PERIOD_TICKS: 1064 return intel_pt_next_period(decoder); 1065 case INTEL_PT_PERIOD_NONE: 1066 case INTEL_PT_PERIOD_MTC: 1067 default: 1068 return 0; 1069 } 1070 } 1071 1072 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1073 { 1074 uint64_t timestamp, masked_timestamp; 1075 1076 switch (decoder->period_type) { 1077 case INTEL_PT_PERIOD_INSTRUCTIONS: 1078 decoder->period_insn_cnt = 0; 1079 break; 1080 case INTEL_PT_PERIOD_TICKS: 1081 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1082 masked_timestamp = timestamp & decoder->period_mask; 1083 if (masked_timestamp > decoder->last_masked_timestamp) 1084 decoder->last_masked_timestamp = masked_timestamp; 1085 else 1086 decoder->last_masked_timestamp += decoder->period_ticks; 1087 break; 1088 case INTEL_PT_PERIOD_NONE: 1089 case INTEL_PT_PERIOD_MTC: 1090 default: 1091 break; 1092 } 1093 1094 decoder->state.type |= INTEL_PT_INSTRUCTION; 1095 } 1096 1097 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1098 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1099 { 1100 uint64_t max_insn_cnt, insn_cnt = 0; 1101 int err; 1102 1103 if (!decoder->mtc_insn) 1104 decoder->mtc_insn = true; 1105 1106 max_insn_cnt = intel_pt_next_sample(decoder); 1107 1108 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1109 max_insn_cnt, decoder->data); 1110 1111 decoder->tot_insn_cnt += insn_cnt; 1112 decoder->timestamp_insn_cnt += insn_cnt; 1113 decoder->sample_insn_cnt += insn_cnt; 1114 decoder->period_insn_cnt += insn_cnt; 1115 1116 if (err) { 1117 decoder->no_progress = 0; 1118 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1119 intel_pt_log_at("ERROR: Failed to get instruction", 1120 decoder->ip); 1121 if (err == -ENOENT) 1122 return -ENOLINK; 1123 return -EILSEQ; 1124 } 1125 1126 if (ip && decoder->ip == ip) { 1127 err = -EAGAIN; 1128 goto out; 1129 } 1130 1131 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1132 intel_pt_sample_insn(decoder); 1133 1134 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1135 decoder->state.type = INTEL_PT_INSTRUCTION; 1136 decoder->state.from_ip = decoder->ip; 1137 decoder->state.to_ip = 0; 1138 decoder->ip += intel_pt_insn->length; 1139 err = INTEL_PT_RETURN; 1140 goto out; 1141 } 1142 1143 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1144 /* Zero-length calls are excluded */ 1145 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1146 intel_pt_insn->rel) { 1147 err = intel_pt_push(&decoder->stack, decoder->ip + 1148 intel_pt_insn->length); 1149 if (err) 1150 goto out; 1151 } 1152 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1153 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1154 } 1155 1156 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1157 int cnt = decoder->no_progress++; 1158 1159 decoder->state.from_ip = decoder->ip; 1160 decoder->ip += intel_pt_insn->length + 1161 intel_pt_insn->rel; 1162 decoder->state.to_ip = decoder->ip; 1163 err = INTEL_PT_RETURN; 1164 1165 /* 1166 * Check for being stuck in a loop. This can happen if a 1167 * decoder error results in the decoder erroneously setting the 1168 * ip to an address that is itself in an infinite loop that 1169 * consumes no packets. When that happens, there must be an 1170 * unconditional branch. 1171 */ 1172 if (cnt) { 1173 if (cnt == 1) { 1174 decoder->stuck_ip = decoder->state.to_ip; 1175 decoder->stuck_ip_prd = 1; 1176 decoder->stuck_ip_cnt = 1; 1177 } else if (cnt > decoder->max_loops || 1178 decoder->state.to_ip == decoder->stuck_ip) { 1179 intel_pt_log_at("ERROR: Never-ending loop", 1180 decoder->state.to_ip); 1181 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1182 err = -ELOOP; 1183 goto out; 1184 } else if (!--decoder->stuck_ip_cnt) { 1185 decoder->stuck_ip_prd += 1; 1186 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1187 decoder->stuck_ip = decoder->state.to_ip; 1188 } 1189 } 1190 goto out_no_progress; 1191 } 1192 out: 1193 decoder->no_progress = 0; 1194 out_no_progress: 1195 decoder->state.insn_op = intel_pt_insn->op; 1196 decoder->state.insn_len = intel_pt_insn->length; 1197 memcpy(decoder->state.insn, intel_pt_insn->buf, 1198 INTEL_PT_INSN_BUF_SZ); 1199 1200 if (decoder->tx_flags & INTEL_PT_IN_TX) 1201 decoder->state.flags |= INTEL_PT_IN_TX; 1202 1203 return err; 1204 } 1205 1206 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1207 { 1208 enum intel_pt_sample_type type = decoder->state.type; 1209 bool ret = false; 1210 1211 decoder->state.type &= ~INTEL_PT_BRANCH; 1212 1213 if (decoder->set_fup_tx_flags) { 1214 decoder->set_fup_tx_flags = false; 1215 decoder->tx_flags = decoder->fup_tx_flags; 1216 decoder->state.type |= INTEL_PT_TRANSACTION; 1217 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1218 decoder->state.type |= INTEL_PT_BRANCH; 1219 decoder->state.flags = decoder->fup_tx_flags; 1220 ret = true; 1221 } 1222 if (decoder->set_fup_ptw) { 1223 decoder->set_fup_ptw = false; 1224 decoder->state.type |= INTEL_PT_PTW; 1225 decoder->state.flags |= INTEL_PT_FUP_IP; 1226 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1227 ret = true; 1228 } 1229 if (decoder->set_fup_mwait) { 1230 decoder->set_fup_mwait = false; 1231 decoder->state.type |= INTEL_PT_MWAIT_OP; 1232 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1233 ret = true; 1234 } 1235 if (decoder->set_fup_pwre) { 1236 decoder->set_fup_pwre = false; 1237 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1238 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1239 ret = true; 1240 } 1241 if (decoder->set_fup_exstop) { 1242 decoder->set_fup_exstop = false; 1243 decoder->state.type |= INTEL_PT_EX_STOP; 1244 decoder->state.flags |= INTEL_PT_FUP_IP; 1245 ret = true; 1246 } 1247 if (decoder->set_fup_bep) { 1248 decoder->set_fup_bep = false; 1249 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1250 ret = true; 1251 } 1252 if (decoder->overflow) { 1253 decoder->overflow = false; 1254 if (!ret && !decoder->pge) { 1255 if (decoder->hop) { 1256 decoder->state.type = 0; 1257 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 1258 } 1259 decoder->pge = true; 1260 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; 1261 decoder->state.from_ip = 0; 1262 decoder->state.to_ip = decoder->ip; 1263 return true; 1264 } 1265 } 1266 if (ret) { 1267 decoder->state.from_ip = decoder->ip; 1268 decoder->state.to_ip = 0; 1269 } else { 1270 decoder->state.type = type; 1271 } 1272 return ret; 1273 } 1274 1275 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1276 struct intel_pt_insn *intel_pt_insn, 1277 uint64_t ip, int err) 1278 { 1279 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1280 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1281 ip == decoder->ip + intel_pt_insn->length; 1282 } 1283 1284 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1285 { 1286 struct intel_pt_insn intel_pt_insn; 1287 uint64_t ip; 1288 int err; 1289 1290 ip = decoder->last_ip; 1291 1292 while (1) { 1293 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1294 if (err == INTEL_PT_RETURN) 1295 return 0; 1296 if (err == -EAGAIN || 1297 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1298 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1299 1300 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1301 if (intel_pt_fup_event(decoder) && no_tip) 1302 return 0; 1303 return -EAGAIN; 1304 } 1305 decoder->set_fup_tx_flags = false; 1306 if (err) 1307 return err; 1308 1309 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1310 intel_pt_log_at("ERROR: Unexpected indirect branch", 1311 decoder->ip); 1312 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1313 return -ENOENT; 1314 } 1315 1316 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1317 intel_pt_log_at("ERROR: Unexpected conditional branch", 1318 decoder->ip); 1319 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1320 return -ENOENT; 1321 } 1322 1323 intel_pt_bug(decoder); 1324 } 1325 } 1326 1327 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1328 { 1329 struct intel_pt_insn intel_pt_insn; 1330 int err; 1331 1332 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1333 if (err == INTEL_PT_RETURN && 1334 decoder->pgd_ip && 1335 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1336 (decoder->state.type & INTEL_PT_BRANCH) && 1337 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1338 /* Unconditional branch leaving filter region */ 1339 decoder->no_progress = 0; 1340 decoder->pge = false; 1341 decoder->continuous_period = false; 1342 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1343 decoder->state.type |= INTEL_PT_TRACE_END; 1344 intel_pt_update_nr(decoder); 1345 return 0; 1346 } 1347 if (err == INTEL_PT_RETURN) 1348 return 0; 1349 if (err) 1350 return err; 1351 1352 intel_pt_update_nr(decoder); 1353 1354 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1355 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1356 decoder->pge = false; 1357 decoder->continuous_period = false; 1358 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1359 decoder->state.from_ip = decoder->ip; 1360 if (decoder->packet.count == 0) { 1361 decoder->state.to_ip = 0; 1362 } else { 1363 decoder->state.to_ip = decoder->last_ip; 1364 decoder->ip = decoder->last_ip; 1365 } 1366 decoder->state.type |= INTEL_PT_TRACE_END; 1367 } else { 1368 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1369 decoder->state.from_ip = decoder->ip; 1370 if (decoder->packet.count == 0) { 1371 decoder->state.to_ip = 0; 1372 } else { 1373 decoder->state.to_ip = decoder->last_ip; 1374 decoder->ip = decoder->last_ip; 1375 } 1376 } 1377 return 0; 1378 } 1379 1380 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1381 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1382 intel_pt_insn.rel; 1383 1384 if (decoder->pgd_ip && 1385 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1386 decoder->pgd_ip(to_ip, decoder->data)) { 1387 /* Conditional branch leaving filter region */ 1388 decoder->pge = false; 1389 decoder->continuous_period = false; 1390 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1391 decoder->ip = to_ip; 1392 decoder->state.from_ip = decoder->ip; 1393 decoder->state.to_ip = to_ip; 1394 decoder->state.type |= INTEL_PT_TRACE_END; 1395 return 0; 1396 } 1397 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1398 decoder->ip); 1399 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1400 return -ENOENT; 1401 } 1402 1403 return intel_pt_bug(decoder); 1404 } 1405 1406 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1407 { 1408 struct intel_pt_insn intel_pt_insn; 1409 int err; 1410 1411 while (1) { 1412 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1413 if (err == INTEL_PT_RETURN) 1414 return 0; 1415 if (err) 1416 return err; 1417 1418 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1419 if (!decoder->return_compression) { 1420 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1421 decoder->ip); 1422 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1423 return -ENOENT; 1424 } 1425 if (!decoder->ret_addr) { 1426 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1427 decoder->ip); 1428 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1429 return -ENOENT; 1430 } 1431 if (!(decoder->tnt.payload & BIT63)) { 1432 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1433 decoder->ip); 1434 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1435 return -ENOENT; 1436 } 1437 decoder->tnt.count -= 1; 1438 if (decoder->tnt.count) 1439 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1440 else 1441 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1442 decoder->tnt.payload <<= 1; 1443 decoder->state.from_ip = decoder->ip; 1444 decoder->ip = decoder->ret_addr; 1445 decoder->state.to_ip = decoder->ip; 1446 return 0; 1447 } 1448 1449 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1450 /* Handle deferred TIPs */ 1451 err = intel_pt_get_next_packet(decoder); 1452 if (err) 1453 return err; 1454 if (decoder->packet.type != INTEL_PT_TIP || 1455 decoder->packet.count == 0) { 1456 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1457 decoder->ip); 1458 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1459 decoder->pkt_step = 0; 1460 return -ENOENT; 1461 } 1462 intel_pt_set_last_ip(decoder); 1463 decoder->state.from_ip = decoder->ip; 1464 decoder->state.to_ip = decoder->last_ip; 1465 decoder->ip = decoder->last_ip; 1466 intel_pt_update_nr(decoder); 1467 return 0; 1468 } 1469 1470 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1471 decoder->tnt.count -= 1; 1472 if (decoder->tnt.count) 1473 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1474 else 1475 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1476 if (decoder->tnt.payload & BIT63) { 1477 decoder->tnt.payload <<= 1; 1478 decoder->state.from_ip = decoder->ip; 1479 decoder->ip += intel_pt_insn.length + 1480 intel_pt_insn.rel; 1481 decoder->state.to_ip = decoder->ip; 1482 return 0; 1483 } 1484 /* Instruction sample for a non-taken branch */ 1485 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1486 decoder->tnt.payload <<= 1; 1487 decoder->state.type = INTEL_PT_INSTRUCTION; 1488 decoder->state.from_ip = decoder->ip; 1489 decoder->state.to_ip = 0; 1490 decoder->ip += intel_pt_insn.length; 1491 return 0; 1492 } 1493 decoder->sample_cyc = false; 1494 decoder->ip += intel_pt_insn.length; 1495 if (!decoder->tnt.count) { 1496 intel_pt_update_sample_time(decoder); 1497 return -EAGAIN; 1498 } 1499 decoder->tnt.payload <<= 1; 1500 continue; 1501 } 1502 1503 return intel_pt_bug(decoder); 1504 } 1505 } 1506 1507 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1508 { 1509 unsigned int fup_tx_flags; 1510 int err; 1511 1512 fup_tx_flags = decoder->packet.payload & 1513 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1514 err = intel_pt_get_next_packet(decoder); 1515 if (err) 1516 return err; 1517 if (decoder->packet.type == INTEL_PT_FUP) { 1518 decoder->fup_tx_flags = fup_tx_flags; 1519 decoder->set_fup_tx_flags = true; 1520 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1521 *no_tip = true; 1522 } else { 1523 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1524 decoder->pos); 1525 intel_pt_update_in_tx(decoder); 1526 } 1527 return 0; 1528 } 1529 1530 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1531 { 1532 timestamp |= (ref_timestamp & (0xffULL << 56)); 1533 1534 if (timestamp < ref_timestamp) { 1535 if (ref_timestamp - timestamp > (1ULL << 55)) 1536 timestamp += (1ULL << 56); 1537 } else { 1538 if (timestamp - ref_timestamp > (1ULL << 55)) 1539 timestamp -= (1ULL << 56); 1540 } 1541 1542 return timestamp; 1543 } 1544 1545 /* For use only when decoder->vm_time_correlation is true */ 1546 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1547 uint64_t timestamp) 1548 { 1549 uint64_t max_timestamp = decoder->buf_timestamp; 1550 1551 if (!max_timestamp) { 1552 max_timestamp = decoder->last_reliable_timestamp + 1553 0x400000000ULL; 1554 } 1555 return timestamp >= decoder->last_reliable_timestamp && 1556 timestamp < decoder->buf_timestamp; 1557 } 1558 1559 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1560 { 1561 uint64_t timestamp; 1562 bool bad = false; 1563 1564 decoder->have_tma = false; 1565 1566 if (decoder->ref_timestamp) { 1567 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1568 decoder->ref_timestamp); 1569 decoder->tsc_timestamp = timestamp; 1570 decoder->timestamp = timestamp; 1571 decoder->ref_timestamp = 0; 1572 decoder->timestamp_insn_cnt = 0; 1573 } else if (decoder->timestamp) { 1574 timestamp = decoder->packet.payload | 1575 (decoder->timestamp & (0xffULL << 56)); 1576 decoder->tsc_timestamp = timestamp; 1577 if (timestamp < decoder->timestamp && 1578 decoder->timestamp - timestamp < decoder->tsc_slip) { 1579 intel_pt_log_to("Suppressing backwards timestamp", 1580 timestamp); 1581 timestamp = decoder->timestamp; 1582 } 1583 if (timestamp < decoder->timestamp) { 1584 if (!decoder->buf_timestamp || 1585 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1586 intel_pt_log_to("Wraparound timestamp", timestamp); 1587 timestamp += (1ULL << 56); 1588 decoder->tsc_timestamp = timestamp; 1589 } else { 1590 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1591 timestamp = decoder->timestamp; 1592 bad = true; 1593 } 1594 } 1595 if (decoder->vm_time_correlation && 1596 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1597 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1598 p_log("Timestamp out of range"); 1599 decoder->timestamp = timestamp; 1600 decoder->timestamp_insn_cnt = 0; 1601 } 1602 1603 if (decoder->last_packet_type == INTEL_PT_CYC) { 1604 decoder->cyc_ref_timestamp = decoder->timestamp; 1605 decoder->cycle_cnt = 0; 1606 decoder->have_calc_cyc_to_tsc = false; 1607 intel_pt_calc_cyc_to_tsc(decoder, false); 1608 } 1609 1610 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1611 } 1612 1613 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1614 { 1615 intel_pt_log("ERROR: Buffer overflow\n"); 1616 intel_pt_clear_tx_flags(decoder); 1617 intel_pt_set_nr(decoder); 1618 decoder->timestamp_insn_cnt = 0; 1619 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1620 decoder->state.from_ip = decoder->ip; 1621 decoder->ip = 0; 1622 decoder->pge = false; 1623 decoder->set_fup_tx_flags = false; 1624 decoder->set_fup_ptw = false; 1625 decoder->set_fup_mwait = false; 1626 decoder->set_fup_pwre = false; 1627 decoder->set_fup_exstop = false; 1628 decoder->set_fup_bep = false; 1629 decoder->overflow = true; 1630 return -EOVERFLOW; 1631 } 1632 1633 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1634 { 1635 if (decoder->have_cyc) 1636 return; 1637 1638 decoder->cyc_cnt_timestamp = decoder->timestamp; 1639 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1640 } 1641 1642 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1643 { 1644 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1645 1646 if (decoder->pge) 1647 intel_pt_mtc_cyc_cnt_pge(decoder); 1648 } 1649 1650 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1651 { 1652 uint64_t tot_cyc_cnt, tsc_delta; 1653 1654 if (decoder->have_cyc) 1655 return; 1656 1657 decoder->sample_cyc = true; 1658 1659 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1660 return; 1661 1662 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1663 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1664 1665 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1666 decoder->tot_cyc_cnt = tot_cyc_cnt; 1667 } 1668 1669 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1670 { 1671 uint32_t ctc = decoder->packet.payload; 1672 uint32_t fc = decoder->packet.count; 1673 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1674 1675 if (!decoder->tsc_ctc_ratio_d) 1676 return; 1677 1678 if (decoder->pge && !decoder->in_psb) 1679 intel_pt_mtc_cyc_cnt_pge(decoder); 1680 else 1681 intel_pt_mtc_cyc_cnt_upd(decoder); 1682 1683 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1684 decoder->last_ctc = ctc - ctc_rem; 1685 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1686 if (decoder->tsc_ctc_mult) { 1687 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1688 } else { 1689 decoder->ctc_timestamp -= multdiv(ctc_rem, 1690 decoder->tsc_ctc_ratio_n, 1691 decoder->tsc_ctc_ratio_d); 1692 } 1693 decoder->ctc_delta = 0; 1694 decoder->have_tma = true; 1695 decoder->fixup_last_mtc = true; 1696 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1697 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1698 } 1699 1700 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1701 { 1702 uint64_t timestamp; 1703 uint32_t mtc, mtc_delta; 1704 1705 if (!decoder->have_tma) 1706 return; 1707 1708 mtc = decoder->packet.payload; 1709 1710 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1711 decoder->fixup_last_mtc = false; 1712 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1713 &decoder->last_mtc); 1714 } 1715 1716 if (mtc > decoder->last_mtc) 1717 mtc_delta = mtc - decoder->last_mtc; 1718 else 1719 mtc_delta = mtc + 256 - decoder->last_mtc; 1720 1721 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1722 1723 if (decoder->tsc_ctc_mult) { 1724 timestamp = decoder->ctc_timestamp + 1725 decoder->ctc_delta * decoder->tsc_ctc_mult; 1726 } else { 1727 timestamp = decoder->ctc_timestamp + 1728 multdiv(decoder->ctc_delta, 1729 decoder->tsc_ctc_ratio_n, 1730 decoder->tsc_ctc_ratio_d); 1731 } 1732 1733 if (timestamp < decoder->timestamp) 1734 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1735 timestamp, decoder->timestamp); 1736 else 1737 decoder->timestamp = timestamp; 1738 1739 intel_pt_mtc_cyc_cnt_upd(decoder); 1740 1741 decoder->timestamp_insn_cnt = 0; 1742 decoder->last_mtc = mtc; 1743 1744 if (decoder->last_packet_type == INTEL_PT_CYC) { 1745 decoder->cyc_ref_timestamp = decoder->timestamp; 1746 decoder->cycle_cnt = 0; 1747 decoder->have_calc_cyc_to_tsc = false; 1748 intel_pt_calc_cyc_to_tsc(decoder, true); 1749 } 1750 1751 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1752 } 1753 1754 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1755 { 1756 unsigned int cbr = decoder->packet.payload & 0xff; 1757 1758 decoder->cbr_payload = decoder->packet.payload; 1759 1760 if (decoder->cbr == cbr) 1761 return; 1762 1763 decoder->cbr = cbr; 1764 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1765 1766 intel_pt_mtc_cyc_cnt_cbr(decoder); 1767 } 1768 1769 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1770 { 1771 uint64_t timestamp = decoder->cyc_ref_timestamp; 1772 1773 decoder->have_cyc = true; 1774 1775 decoder->cycle_cnt += decoder->packet.payload; 1776 if (decoder->pge) 1777 decoder->tot_cyc_cnt += decoder->packet.payload; 1778 decoder->sample_cyc = true; 1779 1780 if (!decoder->cyc_ref_timestamp) 1781 return; 1782 1783 if (decoder->have_calc_cyc_to_tsc) 1784 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1785 else if (decoder->cbr) 1786 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1787 else 1788 return; 1789 1790 if (timestamp < decoder->timestamp) 1791 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1792 timestamp, decoder->timestamp); 1793 else 1794 decoder->timestamp = timestamp; 1795 1796 decoder->timestamp_insn_cnt = 0; 1797 1798 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1799 } 1800 1801 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1802 { 1803 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1804 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1805 decoder->state.items.is_32_bit = false; 1806 } 1807 decoder->blk_type = decoder->packet.payload; 1808 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1809 if (decoder->blk_type == INTEL_PT_GP_REGS) 1810 decoder->state.items.is_32_bit = decoder->packet.count; 1811 if (decoder->blk_type_pos < 0) { 1812 intel_pt_log("WARNING: Unknown block type %u\n", 1813 decoder->blk_type); 1814 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1815 intel_pt_log("WARNING: Duplicate block type %u\n", 1816 decoder->blk_type); 1817 } 1818 } 1819 1820 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1821 { 1822 uint32_t id = decoder->packet.count; 1823 uint32_t bit = 1 << id; 1824 int pos = decoder->blk_type_pos; 1825 1826 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1827 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1828 id, decoder->blk_type); 1829 return; 1830 } 1831 1832 if (decoder->state.items.mask[pos] & bit) { 1833 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1834 id, decoder->blk_type); 1835 } 1836 1837 decoder->state.items.mask[pos] |= bit; 1838 decoder->state.items.val[pos][id] = decoder->packet.payload; 1839 } 1840 1841 /* Walk PSB+ packets when already in sync. */ 1842 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1843 { 1844 int err; 1845 1846 decoder->in_psb = true; 1847 1848 while (1) { 1849 err = intel_pt_get_next_packet(decoder); 1850 if (err) 1851 goto out; 1852 1853 switch (decoder->packet.type) { 1854 case INTEL_PT_PSBEND: 1855 err = 0; 1856 goto out; 1857 1858 case INTEL_PT_TIP_PGD: 1859 case INTEL_PT_TIP_PGE: 1860 case INTEL_PT_TIP: 1861 case INTEL_PT_TNT: 1862 case INTEL_PT_TRACESTOP: 1863 case INTEL_PT_BAD: 1864 case INTEL_PT_PSB: 1865 case INTEL_PT_PTWRITE: 1866 case INTEL_PT_PTWRITE_IP: 1867 case INTEL_PT_EXSTOP: 1868 case INTEL_PT_EXSTOP_IP: 1869 case INTEL_PT_MWAIT: 1870 case INTEL_PT_PWRE: 1871 case INTEL_PT_PWRX: 1872 case INTEL_PT_BBP: 1873 case INTEL_PT_BIP: 1874 case INTEL_PT_BEP: 1875 case INTEL_PT_BEP_IP: 1876 decoder->have_tma = false; 1877 intel_pt_log("ERROR: Unexpected packet\n"); 1878 err = -EAGAIN; 1879 goto out; 1880 1881 case INTEL_PT_OVF: 1882 err = intel_pt_overflow(decoder); 1883 goto out; 1884 1885 case INTEL_PT_TSC: 1886 intel_pt_calc_tsc_timestamp(decoder); 1887 break; 1888 1889 case INTEL_PT_TMA: 1890 intel_pt_calc_tma(decoder); 1891 break; 1892 1893 case INTEL_PT_CBR: 1894 intel_pt_calc_cbr(decoder); 1895 break; 1896 1897 case INTEL_PT_MODE_EXEC: 1898 decoder->exec_mode = decoder->packet.payload; 1899 break; 1900 1901 case INTEL_PT_PIP: 1902 intel_pt_set_pip(decoder); 1903 break; 1904 1905 case INTEL_PT_FUP: 1906 decoder->pge = true; 1907 if (decoder->packet.count) { 1908 intel_pt_set_last_ip(decoder); 1909 decoder->psb_ip = decoder->last_ip; 1910 } 1911 break; 1912 1913 case INTEL_PT_MODE_TSX: 1914 intel_pt_update_in_tx(decoder); 1915 break; 1916 1917 case INTEL_PT_MTC: 1918 intel_pt_calc_mtc_timestamp(decoder); 1919 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1920 decoder->state.type |= INTEL_PT_INSTRUCTION; 1921 break; 1922 1923 case INTEL_PT_CYC: 1924 intel_pt_calc_cyc_timestamp(decoder); 1925 break; 1926 1927 case INTEL_PT_VMCS: 1928 case INTEL_PT_MNT: 1929 case INTEL_PT_PAD: 1930 default: 1931 break; 1932 } 1933 } 1934 out: 1935 decoder->in_psb = false; 1936 1937 return err; 1938 } 1939 1940 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1941 { 1942 int err; 1943 1944 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1945 decoder->tx_flags = 0; 1946 decoder->state.flags &= ~INTEL_PT_IN_TX; 1947 decoder->state.flags |= INTEL_PT_ABORT_TX; 1948 } else { 1949 decoder->state.flags |= INTEL_PT_ASYNC; 1950 } 1951 1952 while (1) { 1953 err = intel_pt_get_next_packet(decoder); 1954 if (err) 1955 return err; 1956 1957 switch (decoder->packet.type) { 1958 case INTEL_PT_TNT: 1959 case INTEL_PT_FUP: 1960 case INTEL_PT_TRACESTOP: 1961 case INTEL_PT_PSB: 1962 case INTEL_PT_TSC: 1963 case INTEL_PT_TMA: 1964 case INTEL_PT_MODE_TSX: 1965 case INTEL_PT_BAD: 1966 case INTEL_PT_PSBEND: 1967 case INTEL_PT_PTWRITE: 1968 case INTEL_PT_PTWRITE_IP: 1969 case INTEL_PT_EXSTOP: 1970 case INTEL_PT_EXSTOP_IP: 1971 case INTEL_PT_MWAIT: 1972 case INTEL_PT_PWRE: 1973 case INTEL_PT_PWRX: 1974 case INTEL_PT_BBP: 1975 case INTEL_PT_BIP: 1976 case INTEL_PT_BEP: 1977 case INTEL_PT_BEP_IP: 1978 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1979 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1980 decoder->pkt_step = 0; 1981 return -ENOENT; 1982 1983 case INTEL_PT_CBR: 1984 intel_pt_calc_cbr(decoder); 1985 break; 1986 1987 case INTEL_PT_OVF: 1988 return intel_pt_overflow(decoder); 1989 1990 case INTEL_PT_TIP_PGD: 1991 decoder->state.from_ip = decoder->ip; 1992 if (decoder->packet.count == 0) { 1993 decoder->state.to_ip = 0; 1994 } else { 1995 intel_pt_set_ip(decoder); 1996 decoder->state.to_ip = decoder->ip; 1997 } 1998 decoder->pge = false; 1999 decoder->continuous_period = false; 2000 decoder->state.type |= INTEL_PT_TRACE_END; 2001 intel_pt_update_nr(decoder); 2002 return 0; 2003 2004 case INTEL_PT_TIP_PGE: 2005 decoder->pge = true; 2006 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 2007 decoder->ip); 2008 decoder->state.from_ip = 0; 2009 if (decoder->packet.count == 0) { 2010 decoder->state.to_ip = 0; 2011 } else { 2012 intel_pt_set_ip(decoder); 2013 decoder->state.to_ip = decoder->ip; 2014 } 2015 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2016 intel_pt_mtc_cyc_cnt_pge(decoder); 2017 intel_pt_set_nr(decoder); 2018 return 0; 2019 2020 case INTEL_PT_TIP: 2021 decoder->state.from_ip = decoder->ip; 2022 if (decoder->packet.count == 0) { 2023 decoder->state.to_ip = 0; 2024 } else { 2025 intel_pt_set_ip(decoder); 2026 decoder->state.to_ip = decoder->ip; 2027 } 2028 intel_pt_update_nr(decoder); 2029 return 0; 2030 2031 case INTEL_PT_PIP: 2032 intel_pt_update_pip(decoder); 2033 break; 2034 2035 case INTEL_PT_MTC: 2036 intel_pt_calc_mtc_timestamp(decoder); 2037 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2038 decoder->state.type |= INTEL_PT_INSTRUCTION; 2039 break; 2040 2041 case INTEL_PT_CYC: 2042 intel_pt_calc_cyc_timestamp(decoder); 2043 break; 2044 2045 case INTEL_PT_MODE_EXEC: 2046 decoder->exec_mode = decoder->packet.payload; 2047 break; 2048 2049 case INTEL_PT_VMCS: 2050 case INTEL_PT_MNT: 2051 case INTEL_PT_PAD: 2052 break; 2053 2054 default: 2055 return intel_pt_bug(decoder); 2056 } 2057 } 2058 } 2059 2060 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2061 { 2062 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2063 decoder->state.type = INTEL_PT_INSTRUCTION; 2064 decoder->state.from_ip = decoder->ip; 2065 decoder->state.to_ip = 0; 2066 return 0; 2067 } 2068 2069 struct intel_pt_vm_tsc_info { 2070 struct intel_pt_pkt pip_packet; 2071 struct intel_pt_pkt vmcs_packet; 2072 struct intel_pt_pkt tma_packet; 2073 bool tsc, pip, vmcs, tma, psbend; 2074 uint64_t ctc_delta; 2075 uint64_t last_ctc; 2076 int max_lookahead; 2077 }; 2078 2079 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2080 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2081 { 2082 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2083 2084 switch (pkt_info->packet.type) { 2085 case INTEL_PT_PAD: 2086 case INTEL_PT_MNT: 2087 case INTEL_PT_MODE_EXEC: 2088 case INTEL_PT_MODE_TSX: 2089 case INTEL_PT_MTC: 2090 case INTEL_PT_FUP: 2091 case INTEL_PT_CYC: 2092 case INTEL_PT_CBR: 2093 break; 2094 2095 case INTEL_PT_TSC: 2096 data->tsc = true; 2097 break; 2098 2099 case INTEL_PT_TMA: 2100 data->tma_packet = pkt_info->packet; 2101 data->tma = true; 2102 break; 2103 2104 case INTEL_PT_PIP: 2105 data->pip_packet = pkt_info->packet; 2106 data->pip = true; 2107 break; 2108 2109 case INTEL_PT_VMCS: 2110 data->vmcs_packet = pkt_info->packet; 2111 data->vmcs = true; 2112 break; 2113 2114 case INTEL_PT_PSBEND: 2115 data->psbend = true; 2116 return 1; 2117 2118 case INTEL_PT_TIP_PGE: 2119 case INTEL_PT_PTWRITE: 2120 case INTEL_PT_PTWRITE_IP: 2121 case INTEL_PT_EXSTOP: 2122 case INTEL_PT_EXSTOP_IP: 2123 case INTEL_PT_MWAIT: 2124 case INTEL_PT_PWRE: 2125 case INTEL_PT_PWRX: 2126 case INTEL_PT_BBP: 2127 case INTEL_PT_BIP: 2128 case INTEL_PT_BEP: 2129 case INTEL_PT_BEP_IP: 2130 case INTEL_PT_OVF: 2131 case INTEL_PT_BAD: 2132 case INTEL_PT_TNT: 2133 case INTEL_PT_TIP_PGD: 2134 case INTEL_PT_TIP: 2135 case INTEL_PT_PSB: 2136 case INTEL_PT_TRACESTOP: 2137 default: 2138 return 1; 2139 } 2140 2141 return 0; 2142 } 2143 2144 struct intel_pt_ovf_fup_info { 2145 int max_lookahead; 2146 bool found; 2147 }; 2148 2149 /* Lookahead to detect a FUP packet after OVF */ 2150 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2151 { 2152 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2153 2154 if (pkt_info->packet.type == INTEL_PT_CYC || 2155 pkt_info->packet.type == INTEL_PT_MTC || 2156 pkt_info->packet.type == INTEL_PT_TSC) 2157 return !--(data->max_lookahead); 2158 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2159 return 1; 2160 } 2161 2162 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2163 { 2164 struct intel_pt_ovf_fup_info data = { 2165 .max_lookahead = 16, 2166 .found = false, 2167 }; 2168 2169 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2170 return data.found; 2171 } 2172 2173 /* Lookahead and get the TMA packet after TSC */ 2174 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2175 { 2176 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2177 2178 if (pkt_info->packet.type == INTEL_PT_CYC || 2179 pkt_info->packet.type == INTEL_PT_MTC) 2180 return !--(data->max_lookahead); 2181 2182 if (pkt_info->packet.type == INTEL_PT_TMA) { 2183 data->tma_packet = pkt_info->packet; 2184 data->tma = true; 2185 } 2186 return 1; 2187 } 2188 2189 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2190 { 2191 if (decoder->tsc_ctc_mult) 2192 return ctc * decoder->tsc_ctc_mult; 2193 else 2194 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2195 } 2196 2197 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2198 uint32_t ctc, 2199 uint32_t fc, 2200 uint64_t last_ctc_timestamp, 2201 uint64_t ctc_delta, 2202 uint32_t last_ctc) 2203 { 2204 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2205 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2206 /* 2207 * Number of CTC ticks from there until current TMA packet. We would 2208 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2209 * past an MTC, so a sign-extended value is used. 2210 */ 2211 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2212 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2213 uint64_t new_ctc_delta = ctc_delta + delta; 2214 uint64_t expected_tsc; 2215 2216 /* 2217 * Convert CTC ticks to TSC ticks, add the starting point 2218 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2219 */ 2220 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2221 2222 if (intel_pt_enable_logging) { 2223 intel_pt_log_x64(last_mtc_ctc); 2224 intel_pt_log_x32(last_ctc); 2225 intel_pt_log_x64(ctc_delta); 2226 intel_pt_log_x64(delta); 2227 intel_pt_log_x32(ctc); 2228 intel_pt_log_x64(new_ctc_delta); 2229 intel_pt_log_x64(last_ctc_timestamp); 2230 intel_pt_log_x32(fc); 2231 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2232 intel_pt_log_x64(expected_tsc); 2233 } 2234 2235 return expected_tsc; 2236 } 2237 2238 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2239 struct intel_pt_vm_tsc_info *data) 2240 { 2241 uint32_t ctc = data->tma_packet.payload; 2242 uint32_t fc = data->tma_packet.count; 2243 2244 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2245 decoder->ctc_timestamp, 2246 data->ctc_delta, data->last_ctc); 2247 } 2248 2249 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2250 struct intel_pt_vmcs_info *vmcs_info) 2251 { 2252 uint64_t payload = decoder->packet.payload; 2253 2254 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2255 decoder->packet.payload -= vmcs_info->tsc_offset; 2256 /* TSC packet has only 7 bytes */ 2257 decoder->packet.payload &= SEVEN_BYTES; 2258 2259 /* 2260 * The buffer is mmapped from the data file, so this also updates the 2261 * data file. 2262 */ 2263 if (!decoder->vm_tm_corr_dry_run) 2264 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2265 2266 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2267 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2268 payload, decoder->packet.payload, vmcs_info->vmcs, 2269 vmcs_info->tsc_offset); 2270 } 2271 2272 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2273 uint64_t tsc_offset) 2274 { 2275 struct intel_pt_vmcs_info vmcs_info = { 2276 .vmcs = NO_VMCS, 2277 .tsc_offset = tsc_offset 2278 }; 2279 2280 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2281 } 2282 2283 static inline bool in_vm(uint64_t pip_payload) 2284 { 2285 return pip_payload & 1; 2286 } 2287 2288 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2289 { 2290 return pip_packet->payload & 1; 2291 } 2292 2293 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2294 { 2295 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2296 vmcs_info->vmcs, vmcs_info->tsc_offset); 2297 } 2298 2299 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2300 struct intel_pt_vm_tsc_info *data) 2301 { 2302 memset(data, 0, sizeof(*data)); 2303 data->ctc_delta = decoder->ctc_delta; 2304 data->last_ctc = decoder->last_ctc; 2305 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2306 if (data->tsc && !data->psbend) 2307 p_log("ERROR: PSB without PSBEND"); 2308 decoder->in_psb = data->psbend; 2309 } 2310 2311 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2312 struct intel_pt_vm_tsc_info *data, 2313 struct intel_pt_vmcs_info *vmcs_info, 2314 uint64_t host_tsc) 2315 { 2316 if (!decoder->in_psb) { 2317 /* Can't happen */ 2318 p_log("ERROR: First TSC is not in PSB+"); 2319 } 2320 2321 if (data->pip) { 2322 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2323 if (vmcs_info && vmcs_info->tsc_offset) { 2324 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2325 decoder->vm_tm_corr_reliable = true; 2326 } else { 2327 p_log("ERROR: First TSC, unknown TSC Offset"); 2328 } 2329 } else { /* Host */ 2330 decoder->vm_tm_corr_reliable = true; 2331 } 2332 } else { /* Host or Guest */ 2333 decoder->vm_tm_corr_reliable = false; 2334 if (intel_pt_time_in_range(decoder, host_tsc)) { 2335 /* Assume Host */ 2336 } else { 2337 /* Assume Guest */ 2338 if (vmcs_info && vmcs_info->tsc_offset) 2339 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2340 else 2341 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2342 } 2343 } 2344 } 2345 2346 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2347 struct intel_pt_vm_tsc_info *data) 2348 { 2349 struct intel_pt_vmcs_info *vmcs_info; 2350 uint64_t tsc_offset = 0; 2351 uint64_t vmcs; 2352 bool reliable = true; 2353 uint64_t expected_tsc; 2354 uint64_t host_tsc; 2355 uint64_t ref_timestamp; 2356 2357 bool assign = false; 2358 bool assign_reliable = false; 2359 2360 /* Already have 'data' for the in_psb case */ 2361 if (!decoder->in_psb) { 2362 memset(data, 0, sizeof(*data)); 2363 data->ctc_delta = decoder->ctc_delta; 2364 data->last_ctc = decoder->last_ctc; 2365 data->max_lookahead = 16; 2366 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2367 if (decoder->pge) { 2368 data->pip = true; 2369 data->pip_packet.payload = decoder->pip_payload; 2370 } 2371 } 2372 2373 /* Calculations depend on having TMA packets */ 2374 if (!data->tma) { 2375 p_log("ERROR: TSC without TMA"); 2376 return; 2377 } 2378 2379 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2380 if (vmcs == NO_VMCS) 2381 vmcs = 0; 2382 2383 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2384 2385 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2386 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2387 2388 if (!decoder->ctc_timestamp) { 2389 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2390 return; 2391 } 2392 2393 expected_tsc = intel_pt_expected_tsc(decoder, data); 2394 2395 tsc_offset = host_tsc - expected_tsc; 2396 2397 /* Determine if TSC is from Host or Guest */ 2398 if (data->pip) { 2399 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2400 if (!vmcs_info) { 2401 /* PIP NR=1 without VMCS cannot happen */ 2402 p_log("ERROR: Missing VMCS"); 2403 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2404 decoder->vm_tm_corr_reliable = false; 2405 return; 2406 } 2407 } else { /* Host */ 2408 decoder->last_reliable_timestamp = host_tsc; 2409 decoder->vm_tm_corr_reliable = true; 2410 return; 2411 } 2412 } else { /* Host or Guest */ 2413 reliable = false; /* Host/Guest is a guess, so not reliable */ 2414 if (decoder->in_psb) { 2415 if (!tsc_offset) 2416 return; /* Zero TSC Offset, assume Host */ 2417 /* 2418 * TSC packet has only 7 bytes of TSC. We have no 2419 * information about the Guest's 8th byte, but it 2420 * doesn't matter because we only need 7 bytes. 2421 * Here, since the 8th byte is unreliable and 2422 * irrelevant, compare only 7 byes. 2423 */ 2424 if (vmcs_info && 2425 (tsc_offset & SEVEN_BYTES) == 2426 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2427 /* Same TSC Offset as last VMCS, assume Guest */ 2428 goto guest; 2429 } 2430 } 2431 /* 2432 * Check if the host_tsc is within the expected range. 2433 * Note, we could narrow the range more by looking ahead for 2434 * the next host TSC in the same buffer, but we don't bother to 2435 * do that because this is probably good enough. 2436 */ 2437 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2438 /* Within expected range for Host TSC, assume Host */ 2439 decoder->vm_tm_corr_reliable = false; 2440 return; 2441 } 2442 } 2443 2444 guest: /* Assuming Guest */ 2445 2446 /* Determine whether to assign TSC Offset */ 2447 if (vmcs_info && vmcs_info->vmcs) { 2448 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2449 assign = false; 2450 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2451 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2452 /* Continuous tracing, TSC in a PSB is not a time loss */ 2453 assign = true; 2454 assign_reliable = true; 2455 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2456 /* 2457 * Unlikely to be a time loss TSC in a PSB which is not 2458 * at the start of a buffer. 2459 */ 2460 assign = true; 2461 assign_reliable = false; 2462 } 2463 } 2464 2465 /* Record VMCS TSC Offset */ 2466 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2467 vmcs_info->reliable != assign_reliable)) { 2468 bool print = vmcs_info->tsc_offset != tsc_offset; 2469 2470 vmcs_info->tsc_offset = tsc_offset; 2471 vmcs_info->reliable = assign_reliable; 2472 if (print) 2473 intel_pt_print_vmcs_info(vmcs_info); 2474 } 2475 2476 /* Determine what TSC Offset to use */ 2477 if (vmcs_info && vmcs_info->tsc_offset) { 2478 if (!vmcs_info->reliable) 2479 reliable = false; 2480 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2481 } else { 2482 reliable = false; 2483 if (vmcs_info) { 2484 if (!vmcs_info->error_printed) { 2485 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2486 vmcs_info->vmcs); 2487 vmcs_info->error_printed = true; 2488 } 2489 } else { 2490 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2491 p_log("ERROR: Unknown VMCS"); 2492 } 2493 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2494 } 2495 2496 decoder->vm_tm_corr_reliable = reliable; 2497 } 2498 2499 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2500 { 2501 uint64_t host_tsc = decoder->packet.payload; 2502 uint64_t guest_tsc = decoder->packet.payload; 2503 struct intel_pt_vmcs_info *vmcs_info; 2504 uint64_t vmcs; 2505 2506 vmcs = decoder->vmcs; 2507 if (vmcs == NO_VMCS) 2508 vmcs = 0; 2509 2510 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2511 2512 if (decoder->pge) { 2513 if (in_vm(decoder->pip_payload)) { /* Guest */ 2514 if (!vmcs_info) { 2515 /* PIP NR=1 without VMCS cannot happen */ 2516 p_log("ERROR: Missing VMCS"); 2517 } 2518 } else { /* Host */ 2519 return; 2520 } 2521 } else { /* Host or Guest */ 2522 if (intel_pt_time_in_range(decoder, host_tsc)) { 2523 /* Within expected range for Host TSC, assume Host */ 2524 return; 2525 } 2526 } 2527 2528 if (vmcs_info) { 2529 /* Translate Guest TSC to Host TSC */ 2530 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2531 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2532 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2533 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2534 guest_tsc, host_tsc, vmcs_info->vmcs, 2535 vmcs_info->tsc_offset); 2536 if (!intel_pt_time_in_range(decoder, host_tsc) && 2537 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2538 p_log("Timestamp out of range"); 2539 } else { 2540 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2541 p_log("ERROR: Unknown VMCS"); 2542 host_tsc = decoder->timestamp; 2543 } 2544 2545 decoder->packet.payload = host_tsc; 2546 2547 if (!decoder->vm_tm_corr_dry_run) 2548 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2549 } 2550 2551 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2552 { 2553 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2554 bool pge; 2555 int err; 2556 2557 if (decoder->in_psb) 2558 intel_pt_vm_tm_corr_psb(decoder, &data); 2559 2560 while (1) { 2561 err = intel_pt_get_next_packet(decoder); 2562 if (err == -ENOLINK) 2563 continue; 2564 if (err) 2565 break; 2566 2567 switch (decoder->packet.type) { 2568 case INTEL_PT_TIP_PGD: 2569 decoder->pge = false; 2570 decoder->vm_tm_corr_continuous = false; 2571 break; 2572 2573 case INTEL_PT_TNT: 2574 case INTEL_PT_TIP: 2575 case INTEL_PT_TIP_PGE: 2576 decoder->pge = true; 2577 break; 2578 2579 case INTEL_PT_OVF: 2580 decoder->in_psb = false; 2581 pge = decoder->pge; 2582 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2583 if (pge != decoder->pge) 2584 intel_pt_log("Surprising PGE change in OVF!"); 2585 if (!decoder->pge) 2586 decoder->vm_tm_corr_continuous = false; 2587 break; 2588 2589 case INTEL_PT_FUP: 2590 if (decoder->in_psb) 2591 decoder->pge = true; 2592 break; 2593 2594 case INTEL_PT_TRACESTOP: 2595 decoder->pge = false; 2596 decoder->vm_tm_corr_continuous = false; 2597 decoder->have_tma = false; 2598 break; 2599 2600 case INTEL_PT_PSB: 2601 intel_pt_vm_tm_corr_psb(decoder, &data); 2602 break; 2603 2604 case INTEL_PT_PIP: 2605 decoder->pip_payload = decoder->packet.payload; 2606 break; 2607 2608 case INTEL_PT_MTC: 2609 intel_pt_calc_mtc_timestamp(decoder); 2610 break; 2611 2612 case INTEL_PT_TSC: 2613 intel_pt_vm_tm_corr_tsc(decoder, &data); 2614 intel_pt_calc_tsc_timestamp(decoder); 2615 decoder->vm_tm_corr_same_buf = true; 2616 decoder->vm_tm_corr_continuous = decoder->pge; 2617 break; 2618 2619 case INTEL_PT_TMA: 2620 intel_pt_calc_tma(decoder); 2621 break; 2622 2623 case INTEL_PT_CYC: 2624 intel_pt_calc_cyc_timestamp(decoder); 2625 break; 2626 2627 case INTEL_PT_CBR: 2628 intel_pt_calc_cbr(decoder); 2629 break; 2630 2631 case INTEL_PT_PSBEND: 2632 decoder->in_psb = false; 2633 data.psbend = false; 2634 break; 2635 2636 case INTEL_PT_VMCS: 2637 if (decoder->packet.payload != NO_VMCS) 2638 decoder->vmcs = decoder->packet.payload; 2639 break; 2640 2641 case INTEL_PT_BBP: 2642 decoder->blk_type = decoder->packet.payload; 2643 break; 2644 2645 case INTEL_PT_BIP: 2646 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2647 decoder->packet.count == 2) 2648 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2649 break; 2650 2651 case INTEL_PT_BEP: 2652 case INTEL_PT_BEP_IP: 2653 decoder->blk_type = 0; 2654 break; 2655 2656 case INTEL_PT_MODE_EXEC: 2657 case INTEL_PT_MODE_TSX: 2658 case INTEL_PT_MNT: 2659 case INTEL_PT_PAD: 2660 case INTEL_PT_PTWRITE_IP: 2661 case INTEL_PT_PTWRITE: 2662 case INTEL_PT_MWAIT: 2663 case INTEL_PT_PWRE: 2664 case INTEL_PT_EXSTOP_IP: 2665 case INTEL_PT_EXSTOP: 2666 case INTEL_PT_PWRX: 2667 case INTEL_PT_BAD: /* Does not happen */ 2668 default: 2669 break; 2670 } 2671 } 2672 2673 return err; 2674 } 2675 2676 #define HOP_PROCESS 0 2677 #define HOP_IGNORE 1 2678 #define HOP_RETURN 2 2679 #define HOP_AGAIN 3 2680 2681 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2682 2683 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2684 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2685 { 2686 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2687 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2688 *err = intel_pt_scan_for_psb(decoder); 2689 if (*err) 2690 return HOP_RETURN; 2691 } 2692 2693 switch (decoder->packet.type) { 2694 case INTEL_PT_TNT: 2695 return HOP_IGNORE; 2696 2697 case INTEL_PT_TIP_PGD: 2698 decoder->pge = false; 2699 if (!decoder->packet.count) { 2700 intel_pt_set_nr(decoder); 2701 return HOP_IGNORE; 2702 } 2703 intel_pt_set_ip(decoder); 2704 decoder->state.type |= INTEL_PT_TRACE_END; 2705 decoder->state.from_ip = 0; 2706 decoder->state.to_ip = decoder->ip; 2707 intel_pt_update_nr(decoder); 2708 return HOP_RETURN; 2709 2710 case INTEL_PT_TIP: 2711 if (!decoder->packet.count) { 2712 intel_pt_set_nr(decoder); 2713 return HOP_IGNORE; 2714 } 2715 intel_pt_set_ip(decoder); 2716 decoder->state.type = INTEL_PT_INSTRUCTION; 2717 decoder->state.from_ip = decoder->ip; 2718 decoder->state.to_ip = 0; 2719 intel_pt_update_nr(decoder); 2720 return HOP_RETURN; 2721 2722 case INTEL_PT_FUP: 2723 if (!decoder->packet.count) 2724 return HOP_IGNORE; 2725 intel_pt_set_ip(decoder); 2726 if (intel_pt_fup_event(decoder)) 2727 return HOP_RETURN; 2728 if (!decoder->branch_enable || !decoder->pge) 2729 *no_tip = true; 2730 if (*no_tip) { 2731 decoder->state.type = INTEL_PT_INSTRUCTION; 2732 decoder->state.from_ip = decoder->ip; 2733 decoder->state.to_ip = 0; 2734 return HOP_RETURN; 2735 } 2736 *err = intel_pt_walk_fup_tip(decoder); 2737 if (!*err) 2738 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2739 return HOP_RETURN; 2740 2741 case INTEL_PT_PSB: 2742 decoder->state.psb_offset = decoder->pos; 2743 decoder->psb_ip = 0; 2744 decoder->last_ip = 0; 2745 decoder->have_last_ip = true; 2746 *err = intel_pt_walk_psbend(decoder); 2747 if (*err == -EAGAIN) 2748 return HOP_AGAIN; 2749 if (*err) 2750 return HOP_RETURN; 2751 decoder->state.type = INTEL_PT_PSB_EVT; 2752 if (decoder->psb_ip) { 2753 decoder->state.type |= INTEL_PT_INSTRUCTION; 2754 decoder->ip = decoder->psb_ip; 2755 } 2756 decoder->state.from_ip = decoder->psb_ip; 2757 decoder->state.to_ip = 0; 2758 return HOP_RETURN; 2759 2760 case INTEL_PT_BAD: 2761 case INTEL_PT_PAD: 2762 case INTEL_PT_TIP_PGE: 2763 case INTEL_PT_TSC: 2764 case INTEL_PT_TMA: 2765 case INTEL_PT_MODE_EXEC: 2766 case INTEL_PT_MODE_TSX: 2767 case INTEL_PT_MTC: 2768 case INTEL_PT_CYC: 2769 case INTEL_PT_VMCS: 2770 case INTEL_PT_PSBEND: 2771 case INTEL_PT_CBR: 2772 case INTEL_PT_TRACESTOP: 2773 case INTEL_PT_PIP: 2774 case INTEL_PT_OVF: 2775 case INTEL_PT_MNT: 2776 case INTEL_PT_PTWRITE: 2777 case INTEL_PT_PTWRITE_IP: 2778 case INTEL_PT_EXSTOP: 2779 case INTEL_PT_EXSTOP_IP: 2780 case INTEL_PT_MWAIT: 2781 case INTEL_PT_PWRE: 2782 case INTEL_PT_PWRX: 2783 case INTEL_PT_BBP: 2784 case INTEL_PT_BIP: 2785 case INTEL_PT_BEP: 2786 case INTEL_PT_BEP_IP: 2787 default: 2788 return HOP_PROCESS; 2789 } 2790 } 2791 2792 struct intel_pt_psb_info { 2793 struct intel_pt_pkt fup_packet; 2794 bool fup; 2795 int after_psbend; 2796 }; 2797 2798 /* Lookahead and get the FUP packet from PSB+ */ 2799 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2800 { 2801 struct intel_pt_psb_info *data = pkt_info->data; 2802 2803 switch (pkt_info->packet.type) { 2804 case INTEL_PT_PAD: 2805 case INTEL_PT_MNT: 2806 case INTEL_PT_TSC: 2807 case INTEL_PT_TMA: 2808 case INTEL_PT_MODE_EXEC: 2809 case INTEL_PT_MODE_TSX: 2810 case INTEL_PT_MTC: 2811 case INTEL_PT_CYC: 2812 case INTEL_PT_VMCS: 2813 case INTEL_PT_CBR: 2814 case INTEL_PT_PIP: 2815 if (data->after_psbend) { 2816 data->after_psbend -= 1; 2817 if (!data->after_psbend) 2818 return 1; 2819 } 2820 break; 2821 2822 case INTEL_PT_FUP: 2823 if (data->after_psbend) 2824 return 1; 2825 if (data->fup || pkt_info->packet.count == 0) 2826 return 1; 2827 data->fup_packet = pkt_info->packet; 2828 data->fup = true; 2829 break; 2830 2831 case INTEL_PT_PSBEND: 2832 if (!data->fup) 2833 return 1; 2834 /* Keep going to check for a TIP.PGE */ 2835 data->after_psbend = 6; 2836 break; 2837 2838 case INTEL_PT_TIP_PGE: 2839 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2840 if (data->after_psbend) 2841 data->fup = false; 2842 return 1; 2843 2844 case INTEL_PT_PTWRITE: 2845 case INTEL_PT_PTWRITE_IP: 2846 case INTEL_PT_EXSTOP: 2847 case INTEL_PT_EXSTOP_IP: 2848 case INTEL_PT_MWAIT: 2849 case INTEL_PT_PWRE: 2850 case INTEL_PT_PWRX: 2851 case INTEL_PT_BBP: 2852 case INTEL_PT_BIP: 2853 case INTEL_PT_BEP: 2854 case INTEL_PT_BEP_IP: 2855 if (data->after_psbend) { 2856 data->after_psbend -= 1; 2857 if (!data->after_psbend) 2858 return 1; 2859 break; 2860 } 2861 return 1; 2862 2863 case INTEL_PT_OVF: 2864 case INTEL_PT_BAD: 2865 case INTEL_PT_TNT: 2866 case INTEL_PT_TIP_PGD: 2867 case INTEL_PT_TIP: 2868 case INTEL_PT_PSB: 2869 case INTEL_PT_TRACESTOP: 2870 default: 2871 return 1; 2872 } 2873 2874 return 0; 2875 } 2876 2877 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2878 { 2879 int err; 2880 2881 decoder->last_ip = 0; 2882 decoder->psb_ip = 0; 2883 decoder->have_last_ip = true; 2884 intel_pt_clear_stack(&decoder->stack); 2885 err = intel_pt_walk_psbend(decoder); 2886 if (err) 2887 return err; 2888 decoder->state.type = INTEL_PT_PSB_EVT; 2889 decoder->state.from_ip = decoder->psb_ip; 2890 decoder->state.to_ip = 0; 2891 return 0; 2892 } 2893 2894 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2895 { 2896 int err; 2897 2898 if (decoder->ip != decoder->last_ip) { 2899 err = intel_pt_walk_fup(decoder); 2900 if (!err || err != -EAGAIN) 2901 return err; 2902 } 2903 2904 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2905 err = intel_pt_psb(decoder); 2906 if (err) { 2907 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2908 return -ENOENT; 2909 } 2910 2911 return 0; 2912 } 2913 2914 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2915 { 2916 struct intel_pt_psb_info data = { .fup = false }; 2917 2918 if (!decoder->branch_enable) 2919 return false; 2920 2921 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2922 if (!data.fup) 2923 return false; 2924 2925 decoder->packet = data.fup_packet; 2926 intel_pt_set_last_ip(decoder); 2927 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2928 2929 *err = intel_pt_fup_in_psb(decoder); 2930 2931 return true; 2932 } 2933 2934 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2935 { 2936 int last_packet_type = INTEL_PT_PAD; 2937 bool no_tip = false; 2938 int err; 2939 2940 while (1) { 2941 err = intel_pt_get_next_packet(decoder); 2942 if (err) 2943 return err; 2944 next: 2945 if (decoder->cyc_threshold) { 2946 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2947 decoder->sample_cyc = false; 2948 last_packet_type = decoder->packet.type; 2949 } 2950 2951 if (decoder->hop) { 2952 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2953 case HOP_IGNORE: 2954 continue; 2955 case HOP_RETURN: 2956 return err; 2957 case HOP_AGAIN: 2958 goto next; 2959 default: 2960 break; 2961 } 2962 } 2963 2964 switch (decoder->packet.type) { 2965 case INTEL_PT_TNT: 2966 if (!decoder->packet.count) 2967 break; 2968 decoder->tnt = decoder->packet; 2969 decoder->pkt_state = INTEL_PT_STATE_TNT; 2970 err = intel_pt_walk_tnt(decoder); 2971 if (err == -EAGAIN) 2972 break; 2973 return err; 2974 2975 case INTEL_PT_TIP_PGD: 2976 if (decoder->packet.count != 0) 2977 intel_pt_set_last_ip(decoder); 2978 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2979 return intel_pt_walk_tip(decoder); 2980 2981 case INTEL_PT_TIP_PGE: { 2982 decoder->pge = true; 2983 decoder->overflow = false; 2984 intel_pt_mtc_cyc_cnt_pge(decoder); 2985 intel_pt_set_nr(decoder); 2986 if (decoder->packet.count == 0) { 2987 intel_pt_log_at("Skipping zero TIP.PGE", 2988 decoder->pos); 2989 break; 2990 } 2991 intel_pt_set_ip(decoder); 2992 decoder->state.from_ip = 0; 2993 decoder->state.to_ip = decoder->ip; 2994 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2995 /* 2996 * In hop mode, resample to get the to_ip as an 2997 * "instruction" sample. 2998 */ 2999 if (decoder->hop) 3000 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3001 return 0; 3002 } 3003 3004 case INTEL_PT_OVF: 3005 return intel_pt_overflow(decoder); 3006 3007 case INTEL_PT_TIP: 3008 if (decoder->packet.count != 0) 3009 intel_pt_set_last_ip(decoder); 3010 decoder->pkt_state = INTEL_PT_STATE_TIP; 3011 return intel_pt_walk_tip(decoder); 3012 3013 case INTEL_PT_FUP: 3014 if (decoder->packet.count == 0) { 3015 intel_pt_log_at("Skipping zero FUP", 3016 decoder->pos); 3017 no_tip = false; 3018 break; 3019 } 3020 intel_pt_set_last_ip(decoder); 3021 if (!decoder->branch_enable || !decoder->pge) { 3022 decoder->ip = decoder->last_ip; 3023 if (intel_pt_fup_event(decoder)) 3024 return 0; 3025 no_tip = false; 3026 break; 3027 } 3028 if (decoder->set_fup_mwait) 3029 no_tip = true; 3030 if (no_tip) 3031 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3032 else 3033 decoder->pkt_state = INTEL_PT_STATE_FUP; 3034 err = intel_pt_walk_fup(decoder); 3035 if (err != -EAGAIN) 3036 return err; 3037 if (no_tip) { 3038 no_tip = false; 3039 break; 3040 } 3041 return intel_pt_walk_fup_tip(decoder); 3042 3043 case INTEL_PT_TRACESTOP: 3044 decoder->pge = false; 3045 decoder->continuous_period = false; 3046 intel_pt_clear_tx_flags(decoder); 3047 decoder->have_tma = false; 3048 break; 3049 3050 case INTEL_PT_PSB: 3051 decoder->state.psb_offset = decoder->pos; 3052 decoder->psb_ip = 0; 3053 if (intel_pt_psb_with_fup(decoder, &err)) 3054 return err; 3055 err = intel_pt_psb(decoder); 3056 if (err == -EAGAIN) 3057 goto next; 3058 return err; 3059 3060 case INTEL_PT_PIP: 3061 intel_pt_update_pip(decoder); 3062 break; 3063 3064 case INTEL_PT_MTC: 3065 intel_pt_calc_mtc_timestamp(decoder); 3066 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3067 break; 3068 /* 3069 * Ensure that there has been an instruction since the 3070 * last MTC. 3071 */ 3072 if (!decoder->mtc_insn) 3073 break; 3074 decoder->mtc_insn = false; 3075 /* Ensure that there is a timestamp */ 3076 if (!decoder->timestamp) 3077 break; 3078 decoder->state.type = INTEL_PT_INSTRUCTION; 3079 decoder->state.from_ip = decoder->ip; 3080 decoder->state.to_ip = 0; 3081 decoder->mtc_insn = false; 3082 return 0; 3083 3084 case INTEL_PT_TSC: 3085 intel_pt_calc_tsc_timestamp(decoder); 3086 break; 3087 3088 case INTEL_PT_TMA: 3089 intel_pt_calc_tma(decoder); 3090 break; 3091 3092 case INTEL_PT_CYC: 3093 intel_pt_calc_cyc_timestamp(decoder); 3094 break; 3095 3096 case INTEL_PT_CBR: 3097 intel_pt_calc_cbr(decoder); 3098 if (decoder->cbr != decoder->cbr_seen) { 3099 decoder->state.type = 0; 3100 return 0; 3101 } 3102 break; 3103 3104 case INTEL_PT_MODE_EXEC: 3105 decoder->exec_mode = decoder->packet.payload; 3106 break; 3107 3108 case INTEL_PT_MODE_TSX: 3109 /* MODE_TSX need not be followed by FUP */ 3110 if (!decoder->pge || decoder->in_psb) { 3111 intel_pt_update_in_tx(decoder); 3112 break; 3113 } 3114 err = intel_pt_mode_tsx(decoder, &no_tip); 3115 if (err) 3116 return err; 3117 goto next; 3118 3119 case INTEL_PT_BAD: /* Does not happen */ 3120 return intel_pt_bug(decoder); 3121 3122 case INTEL_PT_PSBEND: 3123 case INTEL_PT_VMCS: 3124 case INTEL_PT_MNT: 3125 case INTEL_PT_PAD: 3126 break; 3127 3128 case INTEL_PT_PTWRITE_IP: 3129 decoder->fup_ptw_payload = decoder->packet.payload; 3130 err = intel_pt_get_next_packet(decoder); 3131 if (err) 3132 return err; 3133 if (decoder->packet.type == INTEL_PT_FUP) { 3134 decoder->set_fup_ptw = true; 3135 no_tip = true; 3136 } else { 3137 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3138 decoder->pos); 3139 } 3140 goto next; 3141 3142 case INTEL_PT_PTWRITE: 3143 decoder->state.type = INTEL_PT_PTW; 3144 decoder->state.from_ip = decoder->ip; 3145 decoder->state.to_ip = 0; 3146 decoder->state.ptw_payload = decoder->packet.payload; 3147 return 0; 3148 3149 case INTEL_PT_MWAIT: 3150 decoder->fup_mwait_payload = decoder->packet.payload; 3151 decoder->set_fup_mwait = true; 3152 break; 3153 3154 case INTEL_PT_PWRE: 3155 if (decoder->set_fup_mwait) { 3156 decoder->fup_pwre_payload = 3157 decoder->packet.payload; 3158 decoder->set_fup_pwre = true; 3159 break; 3160 } 3161 decoder->state.type = INTEL_PT_PWR_ENTRY; 3162 decoder->state.from_ip = decoder->ip; 3163 decoder->state.to_ip = 0; 3164 decoder->state.pwrx_payload = decoder->packet.payload; 3165 return 0; 3166 3167 case INTEL_PT_EXSTOP_IP: 3168 err = intel_pt_get_next_packet(decoder); 3169 if (err) 3170 return err; 3171 if (decoder->packet.type == INTEL_PT_FUP) { 3172 decoder->set_fup_exstop = true; 3173 no_tip = true; 3174 } else { 3175 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3176 decoder->pos); 3177 } 3178 goto next; 3179 3180 case INTEL_PT_EXSTOP: 3181 decoder->state.type = INTEL_PT_EX_STOP; 3182 decoder->state.from_ip = decoder->ip; 3183 decoder->state.to_ip = 0; 3184 return 0; 3185 3186 case INTEL_PT_PWRX: 3187 decoder->state.type = INTEL_PT_PWR_EXIT; 3188 decoder->state.from_ip = decoder->ip; 3189 decoder->state.to_ip = 0; 3190 decoder->state.pwrx_payload = decoder->packet.payload; 3191 return 0; 3192 3193 case INTEL_PT_BBP: 3194 intel_pt_bbp(decoder); 3195 break; 3196 3197 case INTEL_PT_BIP: 3198 intel_pt_bip(decoder); 3199 break; 3200 3201 case INTEL_PT_BEP: 3202 decoder->state.type = INTEL_PT_BLK_ITEMS; 3203 decoder->state.from_ip = decoder->ip; 3204 decoder->state.to_ip = 0; 3205 return 0; 3206 3207 case INTEL_PT_BEP_IP: 3208 err = intel_pt_get_next_packet(decoder); 3209 if (err) 3210 return err; 3211 if (decoder->packet.type == INTEL_PT_FUP) { 3212 decoder->set_fup_bep = true; 3213 no_tip = true; 3214 } else { 3215 intel_pt_log_at("ERROR: Missing FUP after BEP", 3216 decoder->pos); 3217 } 3218 goto next; 3219 3220 default: 3221 return intel_pt_bug(decoder); 3222 } 3223 } 3224 } 3225 3226 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3227 { 3228 return decoder->packet.count && 3229 (decoder->have_last_ip || decoder->packet.count == 3 || 3230 decoder->packet.count == 6); 3231 } 3232 3233 /* Walk PSB+ packets to get in sync. */ 3234 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3235 { 3236 int err; 3237 3238 decoder->in_psb = true; 3239 3240 while (1) { 3241 err = intel_pt_get_next_packet(decoder); 3242 if (err) 3243 goto out; 3244 3245 switch (decoder->packet.type) { 3246 case INTEL_PT_TIP_PGD: 3247 decoder->continuous_period = false; 3248 __fallthrough; 3249 case INTEL_PT_TIP_PGE: 3250 case INTEL_PT_TIP: 3251 case INTEL_PT_PTWRITE: 3252 case INTEL_PT_PTWRITE_IP: 3253 case INTEL_PT_EXSTOP: 3254 case INTEL_PT_EXSTOP_IP: 3255 case INTEL_PT_MWAIT: 3256 case INTEL_PT_PWRE: 3257 case INTEL_PT_PWRX: 3258 case INTEL_PT_BBP: 3259 case INTEL_PT_BIP: 3260 case INTEL_PT_BEP: 3261 case INTEL_PT_BEP_IP: 3262 intel_pt_log("ERROR: Unexpected packet\n"); 3263 err = -ENOENT; 3264 goto out; 3265 3266 case INTEL_PT_FUP: 3267 decoder->pge = true; 3268 if (intel_pt_have_ip(decoder)) { 3269 uint64_t current_ip = decoder->ip; 3270 3271 intel_pt_set_ip(decoder); 3272 decoder->psb_ip = decoder->ip; 3273 if (current_ip) 3274 intel_pt_log_to("Setting IP", 3275 decoder->ip); 3276 } 3277 break; 3278 3279 case INTEL_PT_MTC: 3280 intel_pt_calc_mtc_timestamp(decoder); 3281 break; 3282 3283 case INTEL_PT_TSC: 3284 intel_pt_calc_tsc_timestamp(decoder); 3285 break; 3286 3287 case INTEL_PT_TMA: 3288 intel_pt_calc_tma(decoder); 3289 break; 3290 3291 case INTEL_PT_CYC: 3292 intel_pt_calc_cyc_timestamp(decoder); 3293 break; 3294 3295 case INTEL_PT_CBR: 3296 intel_pt_calc_cbr(decoder); 3297 break; 3298 3299 case INTEL_PT_PIP: 3300 intel_pt_set_pip(decoder); 3301 break; 3302 3303 case INTEL_PT_MODE_EXEC: 3304 decoder->exec_mode = decoder->packet.payload; 3305 break; 3306 3307 case INTEL_PT_MODE_TSX: 3308 intel_pt_update_in_tx(decoder); 3309 break; 3310 3311 case INTEL_PT_TRACESTOP: 3312 decoder->pge = false; 3313 decoder->continuous_period = false; 3314 intel_pt_clear_tx_flags(decoder); 3315 __fallthrough; 3316 3317 case INTEL_PT_TNT: 3318 decoder->have_tma = false; 3319 intel_pt_log("ERROR: Unexpected packet\n"); 3320 if (decoder->ip) 3321 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3322 else 3323 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3324 err = -ENOENT; 3325 goto out; 3326 3327 case INTEL_PT_BAD: /* Does not happen */ 3328 err = intel_pt_bug(decoder); 3329 goto out; 3330 3331 case INTEL_PT_OVF: 3332 err = intel_pt_overflow(decoder); 3333 goto out; 3334 3335 case INTEL_PT_PSBEND: 3336 err = 0; 3337 goto out; 3338 3339 case INTEL_PT_PSB: 3340 case INTEL_PT_VMCS: 3341 case INTEL_PT_MNT: 3342 case INTEL_PT_PAD: 3343 default: 3344 break; 3345 } 3346 } 3347 out: 3348 decoder->in_psb = false; 3349 3350 return err; 3351 } 3352 3353 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3354 { 3355 int err; 3356 3357 while (1) { 3358 err = intel_pt_get_next_packet(decoder); 3359 if (err) 3360 return err; 3361 3362 switch (decoder->packet.type) { 3363 case INTEL_PT_TIP_PGD: 3364 decoder->continuous_period = false; 3365 decoder->pge = false; 3366 if (intel_pt_have_ip(decoder)) 3367 intel_pt_set_ip(decoder); 3368 if (!decoder->ip) 3369 break; 3370 decoder->state.type |= INTEL_PT_TRACE_END; 3371 return 0; 3372 3373 case INTEL_PT_TIP_PGE: 3374 decoder->pge = true; 3375 intel_pt_mtc_cyc_cnt_pge(decoder); 3376 if (intel_pt_have_ip(decoder)) 3377 intel_pt_set_ip(decoder); 3378 if (!decoder->ip) 3379 break; 3380 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3381 return 0; 3382 3383 case INTEL_PT_TIP: 3384 decoder->pge = true; 3385 if (intel_pt_have_ip(decoder)) 3386 intel_pt_set_ip(decoder); 3387 if (!decoder->ip) 3388 break; 3389 return 0; 3390 3391 case INTEL_PT_FUP: 3392 if (intel_pt_have_ip(decoder)) 3393 intel_pt_set_ip(decoder); 3394 if (decoder->ip) 3395 return 0; 3396 break; 3397 3398 case INTEL_PT_MTC: 3399 intel_pt_calc_mtc_timestamp(decoder); 3400 break; 3401 3402 case INTEL_PT_TSC: 3403 intel_pt_calc_tsc_timestamp(decoder); 3404 break; 3405 3406 case INTEL_PT_TMA: 3407 intel_pt_calc_tma(decoder); 3408 break; 3409 3410 case INTEL_PT_CYC: 3411 intel_pt_calc_cyc_timestamp(decoder); 3412 break; 3413 3414 case INTEL_PT_CBR: 3415 intel_pt_calc_cbr(decoder); 3416 break; 3417 3418 case INTEL_PT_PIP: 3419 intel_pt_set_pip(decoder); 3420 break; 3421 3422 case INTEL_PT_MODE_EXEC: 3423 decoder->exec_mode = decoder->packet.payload; 3424 break; 3425 3426 case INTEL_PT_MODE_TSX: 3427 intel_pt_update_in_tx(decoder); 3428 break; 3429 3430 case INTEL_PT_OVF: 3431 return intel_pt_overflow(decoder); 3432 3433 case INTEL_PT_BAD: /* Does not happen */ 3434 return intel_pt_bug(decoder); 3435 3436 case INTEL_PT_TRACESTOP: 3437 decoder->pge = false; 3438 decoder->continuous_period = false; 3439 intel_pt_clear_tx_flags(decoder); 3440 decoder->have_tma = false; 3441 break; 3442 3443 case INTEL_PT_PSB: 3444 decoder->state.psb_offset = decoder->pos; 3445 decoder->psb_ip = 0; 3446 decoder->last_ip = 0; 3447 decoder->have_last_ip = true; 3448 intel_pt_clear_stack(&decoder->stack); 3449 err = intel_pt_walk_psb(decoder); 3450 if (err) 3451 return err; 3452 decoder->state.type = INTEL_PT_PSB_EVT; 3453 decoder->state.from_ip = decoder->psb_ip; 3454 decoder->state.to_ip = 0; 3455 return 0; 3456 3457 case INTEL_PT_TNT: 3458 case INTEL_PT_PSBEND: 3459 case INTEL_PT_VMCS: 3460 case INTEL_PT_MNT: 3461 case INTEL_PT_PAD: 3462 case INTEL_PT_PTWRITE: 3463 case INTEL_PT_PTWRITE_IP: 3464 case INTEL_PT_EXSTOP: 3465 case INTEL_PT_EXSTOP_IP: 3466 case INTEL_PT_MWAIT: 3467 case INTEL_PT_PWRE: 3468 case INTEL_PT_PWRX: 3469 case INTEL_PT_BBP: 3470 case INTEL_PT_BIP: 3471 case INTEL_PT_BEP: 3472 case INTEL_PT_BEP_IP: 3473 default: 3474 break; 3475 } 3476 } 3477 } 3478 3479 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3480 { 3481 int err; 3482 3483 decoder->set_fup_tx_flags = false; 3484 decoder->set_fup_ptw = false; 3485 decoder->set_fup_mwait = false; 3486 decoder->set_fup_pwre = false; 3487 decoder->set_fup_exstop = false; 3488 decoder->set_fup_bep = false; 3489 decoder->overflow = false; 3490 3491 if (!decoder->branch_enable) { 3492 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3493 decoder->state.type = 0; /* Do not have a sample */ 3494 return 0; 3495 } 3496 3497 intel_pt_log("Scanning for full IP\n"); 3498 err = intel_pt_walk_to_ip(decoder); 3499 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3500 return err; 3501 3502 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3503 if (decoder->hop) 3504 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3505 else 3506 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3507 3508 decoder->state.from_ip = 0; 3509 decoder->state.to_ip = decoder->ip; 3510 intel_pt_log_to("Setting IP", decoder->ip); 3511 3512 return 0; 3513 } 3514 3515 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3516 { 3517 const unsigned char *end = decoder->buf + decoder->len; 3518 size_t i; 3519 3520 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3521 if (i > decoder->len) 3522 continue; 3523 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3524 return i; 3525 } 3526 return 0; 3527 } 3528 3529 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3530 { 3531 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3532 const char *psb = INTEL_PT_PSB_STR; 3533 3534 if (rest_psb > decoder->len || 3535 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3536 return 0; 3537 3538 return rest_psb; 3539 } 3540 3541 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3542 int part_psb) 3543 { 3544 int rest_psb, ret; 3545 3546 decoder->pos += decoder->len; 3547 decoder->len = 0; 3548 3549 ret = intel_pt_get_next_data(decoder, false); 3550 if (ret) 3551 return ret; 3552 3553 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3554 if (!rest_psb) 3555 return 0; 3556 3557 decoder->pos -= part_psb; 3558 decoder->next_buf = decoder->buf + rest_psb; 3559 decoder->next_len = decoder->len - rest_psb; 3560 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3561 decoder->buf = decoder->temp_buf; 3562 decoder->len = INTEL_PT_PSB_LEN; 3563 3564 return 0; 3565 } 3566 3567 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3568 { 3569 unsigned char *next; 3570 int ret; 3571 3572 intel_pt_log("Scanning for PSB\n"); 3573 while (1) { 3574 if (!decoder->len) { 3575 ret = intel_pt_get_next_data(decoder, false); 3576 if (ret) 3577 return ret; 3578 } 3579 3580 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3581 INTEL_PT_PSB_LEN); 3582 if (!next) { 3583 int part_psb; 3584 3585 part_psb = intel_pt_part_psb(decoder); 3586 if (part_psb) { 3587 ret = intel_pt_get_split_psb(decoder, part_psb); 3588 if (ret) 3589 return ret; 3590 } else { 3591 decoder->pos += decoder->len; 3592 decoder->len = 0; 3593 } 3594 continue; 3595 } 3596 3597 decoder->pkt_step = next - decoder->buf; 3598 return intel_pt_get_next_packet(decoder); 3599 } 3600 } 3601 3602 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3603 { 3604 int err; 3605 3606 decoder->pge = false; 3607 decoder->continuous_period = false; 3608 decoder->have_last_ip = false; 3609 decoder->last_ip = 0; 3610 decoder->psb_ip = 0; 3611 decoder->ip = 0; 3612 intel_pt_clear_stack(&decoder->stack); 3613 3614 err = intel_pt_scan_for_psb(decoder); 3615 if (err) 3616 return err; 3617 3618 if (decoder->vm_time_correlation) { 3619 decoder->in_psb = true; 3620 if (!decoder->timestamp) 3621 decoder->timestamp = 1; 3622 decoder->state.type = 0; 3623 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3624 return 0; 3625 } 3626 3627 decoder->have_last_ip = true; 3628 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3629 3630 err = intel_pt_walk_psb(decoder); 3631 if (err) 3632 return err; 3633 3634 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3635 decoder->state.from_ip = decoder->psb_ip; 3636 decoder->state.to_ip = 0; 3637 3638 if (decoder->ip) { 3639 /* 3640 * In hop mode, resample to get the PSB FUP ip as an 3641 * "instruction" sample. 3642 */ 3643 if (decoder->hop) 3644 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3645 else 3646 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3647 } 3648 3649 return 0; 3650 } 3651 3652 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3653 { 3654 uint64_t est = decoder->sample_insn_cnt << 1; 3655 3656 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3657 goto out; 3658 3659 est *= decoder->max_non_turbo_ratio; 3660 est /= decoder->cbr; 3661 out: 3662 return decoder->sample_timestamp + est; 3663 } 3664 3665 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3666 { 3667 int err; 3668 3669 do { 3670 decoder->state.type = INTEL_PT_BRANCH; 3671 decoder->state.flags = 0; 3672 3673 switch (decoder->pkt_state) { 3674 case INTEL_PT_STATE_NO_PSB: 3675 err = intel_pt_sync(decoder); 3676 break; 3677 case INTEL_PT_STATE_NO_IP: 3678 decoder->have_last_ip = false; 3679 decoder->last_ip = 0; 3680 decoder->ip = 0; 3681 __fallthrough; 3682 case INTEL_PT_STATE_ERR_RESYNC: 3683 err = intel_pt_sync_ip(decoder); 3684 break; 3685 case INTEL_PT_STATE_IN_SYNC: 3686 err = intel_pt_walk_trace(decoder); 3687 break; 3688 case INTEL_PT_STATE_TNT: 3689 case INTEL_PT_STATE_TNT_CONT: 3690 err = intel_pt_walk_tnt(decoder); 3691 if (err == -EAGAIN) 3692 err = intel_pt_walk_trace(decoder); 3693 break; 3694 case INTEL_PT_STATE_TIP: 3695 case INTEL_PT_STATE_TIP_PGD: 3696 err = intel_pt_walk_tip(decoder); 3697 break; 3698 case INTEL_PT_STATE_FUP: 3699 err = intel_pt_walk_fup(decoder); 3700 if (err == -EAGAIN) 3701 err = intel_pt_walk_fup_tip(decoder); 3702 break; 3703 case INTEL_PT_STATE_FUP_NO_TIP: 3704 err = intel_pt_walk_fup(decoder); 3705 if (err == -EAGAIN) 3706 err = intel_pt_walk_trace(decoder); 3707 break; 3708 case INTEL_PT_STATE_FUP_IN_PSB: 3709 err = intel_pt_fup_in_psb(decoder); 3710 break; 3711 case INTEL_PT_STATE_RESAMPLE: 3712 err = intel_pt_resample(decoder); 3713 break; 3714 case INTEL_PT_STATE_VM_TIME_CORRELATION: 3715 err = intel_pt_vm_time_correlation(decoder); 3716 break; 3717 default: 3718 err = intel_pt_bug(decoder); 3719 break; 3720 } 3721 } while (err == -ENOLINK); 3722 3723 if (err) { 3724 decoder->state.err = intel_pt_ext_err(err); 3725 if (err != -EOVERFLOW) 3726 decoder->state.from_ip = decoder->ip; 3727 intel_pt_update_sample_time(decoder); 3728 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3729 intel_pt_set_nr(decoder); 3730 } else { 3731 decoder->state.err = 0; 3732 if (decoder->cbr != decoder->cbr_seen) { 3733 decoder->cbr_seen = decoder->cbr; 3734 if (!decoder->state.type) { 3735 decoder->state.from_ip = decoder->ip; 3736 decoder->state.to_ip = 0; 3737 } 3738 decoder->state.type |= INTEL_PT_CBR_CHG; 3739 decoder->state.cbr_payload = decoder->cbr_payload; 3740 decoder->state.cbr = decoder->cbr; 3741 } 3742 if (intel_pt_sample_time(decoder->pkt_state)) { 3743 intel_pt_update_sample_time(decoder); 3744 if (decoder->sample_cyc) { 3745 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3746 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3747 decoder->sample_cyc = false; 3748 } 3749 } 3750 /* 3751 * When using only TSC/MTC to compute cycles, IPC can be 3752 * sampled as soon as the cycle count changes. 3753 */ 3754 if (!decoder->have_cyc) 3755 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3756 } 3757 3758 /* Let PSB event always have TSC timestamp */ 3759 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3760 decoder->sample_timestamp = decoder->tsc_timestamp; 3761 3762 decoder->state.from_nr = decoder->nr; 3763 decoder->state.to_nr = decoder->next_nr; 3764 decoder->nr = decoder->next_nr; 3765 3766 decoder->state.timestamp = decoder->sample_timestamp; 3767 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3768 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3769 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3770 3771 return &decoder->state; 3772 } 3773 3774 /** 3775 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3776 * @buf: pointer to buffer pointer 3777 * @len: size of buffer 3778 * 3779 * Updates the buffer pointer to point to the start of the next PSB packet if 3780 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3781 * @len is adjusted accordingly. 3782 * 3783 * Return: %true if a PSB packet is found, %false otherwise. 3784 */ 3785 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3786 { 3787 unsigned char *next; 3788 3789 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3790 if (next) { 3791 *len -= next - *buf; 3792 *buf = next; 3793 return true; 3794 } 3795 return false; 3796 } 3797 3798 /** 3799 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3800 * packet. 3801 * @buf: pointer to buffer pointer 3802 * @len: size of buffer 3803 * 3804 * Updates the buffer pointer to point to the start of the following PSB packet 3805 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3806 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3807 * 3808 * Return: %true if a PSB packet is found, %false otherwise. 3809 */ 3810 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3811 { 3812 unsigned char *next; 3813 3814 if (!*len) 3815 return false; 3816 3817 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3818 if (next) { 3819 *len -= next - *buf; 3820 *buf = next; 3821 return true; 3822 } 3823 return false; 3824 } 3825 3826 /** 3827 * intel_pt_last_psb - find the last PSB packet in a buffer. 3828 * @buf: buffer 3829 * @len: size of buffer 3830 * 3831 * This function finds the last PSB in a buffer. 3832 * 3833 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3834 */ 3835 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3836 { 3837 const char *n = INTEL_PT_PSB_STR; 3838 unsigned char *p; 3839 size_t k; 3840 3841 if (len < INTEL_PT_PSB_LEN) 3842 return NULL; 3843 3844 k = len - INTEL_PT_PSB_LEN + 1; 3845 while (1) { 3846 p = memrchr(buf, n[0], k); 3847 if (!p) 3848 return NULL; 3849 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3850 return p; 3851 k = p - buf; 3852 if (!k) 3853 return NULL; 3854 } 3855 } 3856 3857 /** 3858 * intel_pt_next_tsc - find and return next TSC. 3859 * @buf: buffer 3860 * @len: size of buffer 3861 * @tsc: TSC value returned 3862 * @rem: returns remaining size when TSC is found 3863 * 3864 * Find a TSC packet in @buf and return the TSC value. This function assumes 3865 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3866 * PSBEND packet is found. 3867 * 3868 * Return: %true if TSC is found, false otherwise. 3869 */ 3870 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3871 size_t *rem) 3872 { 3873 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3874 struct intel_pt_pkt packet; 3875 int ret; 3876 3877 while (len) { 3878 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3879 if (ret <= 0) 3880 return false; 3881 if (packet.type == INTEL_PT_TSC) { 3882 *tsc = packet.payload; 3883 *rem = len; 3884 return true; 3885 } 3886 if (packet.type == INTEL_PT_PSBEND) 3887 return false; 3888 buf += ret; 3889 len -= ret; 3890 } 3891 return false; 3892 } 3893 3894 /** 3895 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3896 * @tsc1: first TSC to compare 3897 * @tsc2: second TSC to compare 3898 * 3899 * This function compares 7-byte TSC values allowing for the possibility that 3900 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3901 * around so for that purpose this function assumes the absolute difference is 3902 * less than half the maximum difference. 3903 * 3904 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3905 * after @tsc2. 3906 */ 3907 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3908 { 3909 const uint64_t halfway = (1ULL << 55); 3910 3911 if (tsc1 == tsc2) 3912 return 0; 3913 3914 if (tsc1 < tsc2) { 3915 if (tsc2 - tsc1 < halfway) 3916 return -1; 3917 else 3918 return 1; 3919 } else { 3920 if (tsc1 - tsc2 < halfway) 3921 return 1; 3922 else 3923 return -1; 3924 } 3925 } 3926 3927 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3928 3929 /** 3930 * adj_for_padding - adjust overlap to account for padding. 3931 * @buf_b: second buffer 3932 * @buf_a: first buffer 3933 * @len_a: size of first buffer 3934 * 3935 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3936 * accordingly. 3937 * 3938 * Return: A pointer into @buf_b from where non-overlapped data starts 3939 */ 3940 static unsigned char *adj_for_padding(unsigned char *buf_b, 3941 unsigned char *buf_a, size_t len_a) 3942 { 3943 unsigned char *p = buf_b - MAX_PADDING; 3944 unsigned char *q = buf_a + len_a - MAX_PADDING; 3945 int i; 3946 3947 for (i = MAX_PADDING; i; i--, p++, q++) { 3948 if (*p != *q) 3949 break; 3950 } 3951 3952 return p; 3953 } 3954 3955 /** 3956 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3957 * using TSC. 3958 * @buf_a: first buffer 3959 * @len_a: size of first buffer 3960 * @buf_b: second buffer 3961 * @len_b: size of second buffer 3962 * @consecutive: returns true if there is data in buf_b that is consecutive 3963 * to buf_a 3964 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 3965 * 3966 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3967 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3968 * walk forward in @buf_b until a later TSC is found. A precondition is that 3969 * @buf_a and @buf_b are positioned at a PSB. 3970 * 3971 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3972 * @buf_b + @len_b if there is no non-overlapped data. 3973 */ 3974 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3975 size_t len_a, 3976 unsigned char *buf_b, 3977 size_t len_b, bool *consecutive, 3978 bool ooo_tsc) 3979 { 3980 uint64_t tsc_a, tsc_b; 3981 unsigned char *p; 3982 size_t len, rem_a, rem_b; 3983 3984 p = intel_pt_last_psb(buf_a, len_a); 3985 if (!p) 3986 return buf_b; /* No PSB in buf_a => no overlap */ 3987 3988 len = len_a - (p - buf_a); 3989 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3990 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3991 len_a -= len; 3992 p = intel_pt_last_psb(buf_a, len_a); 3993 if (!p) 3994 return buf_b; /* No full PSB+ => assume no overlap */ 3995 len = len_a - (p - buf_a); 3996 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3997 return buf_b; /* No TSC in buf_a => assume no overlap */ 3998 } 3999 4000 while (1) { 4001 /* Ignore PSB+ with no TSC */ 4002 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 4003 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 4004 4005 /* Same TSC, so buffers are consecutive */ 4006 if (!cmp && rem_b >= rem_a) { 4007 unsigned char *start; 4008 4009 *consecutive = true; 4010 start = buf_b + len_b - (rem_b - rem_a); 4011 return adj_for_padding(start, buf_a, len_a); 4012 } 4013 if (cmp < 0 && !ooo_tsc) 4014 return buf_b; /* tsc_a < tsc_b => no overlap */ 4015 } 4016 4017 if (!intel_pt_step_psb(&buf_b, &len_b)) 4018 return buf_b + len_b; /* No PSB in buf_b => no data */ 4019 } 4020 } 4021 4022 /** 4023 * intel_pt_find_overlap - determine start of non-overlapped trace data. 4024 * @buf_a: first buffer 4025 * @len_a: size of first buffer 4026 * @buf_b: second buffer 4027 * @len_b: size of second buffer 4028 * @have_tsc: can use TSC packets to detect overlap 4029 * @consecutive: returns true if there is data in buf_b that is consecutive 4030 * to buf_a 4031 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4032 * 4033 * When trace samples or snapshots are recorded there is the possibility that 4034 * the data overlaps. Note that, for the purposes of decoding, data is only 4035 * useful if it begins with a PSB packet. 4036 * 4037 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4038 * @buf_b + @len_b if there is no non-overlapped data. 4039 */ 4040 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4041 unsigned char *buf_b, size_t len_b, 4042 bool have_tsc, bool *consecutive, 4043 bool ooo_tsc) 4044 { 4045 unsigned char *found; 4046 4047 /* Buffer 'b' must start at PSB so throw away everything before that */ 4048 if (!intel_pt_next_psb(&buf_b, &len_b)) 4049 return buf_b + len_b; /* No PSB */ 4050 4051 if (!intel_pt_next_psb(&buf_a, &len_a)) 4052 return buf_b; /* No overlap */ 4053 4054 if (have_tsc) { 4055 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4056 consecutive, ooo_tsc); 4057 if (found) 4058 return found; 4059 } 4060 4061 /* 4062 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4063 * we can ignore the first part of buffer 'a'. 4064 */ 4065 while (len_b < len_a) { 4066 if (!intel_pt_step_psb(&buf_a, &len_a)) 4067 return buf_b; /* No overlap */ 4068 } 4069 4070 /* Now len_b >= len_a */ 4071 while (1) { 4072 /* Potential overlap so check the bytes */ 4073 found = memmem(buf_a, len_a, buf_b, len_a); 4074 if (found) { 4075 *consecutive = true; 4076 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4077 } 4078 4079 /* Try again at next PSB in buffer 'a' */ 4080 if (!intel_pt_step_psb(&buf_a, &len_a)) 4081 return buf_b; /* No overlap */ 4082 } 4083 } 4084 4085 /** 4086 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4087 * @timestamp: timestamp to fast forward towards 4088 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4089 * the fast forward timestamp. 4090 */ 4091 struct fast_forward_data { 4092 uint64_t timestamp; 4093 uint64_t buf_timestamp; 4094 }; 4095 4096 /** 4097 * intel_pt_ff_cb - fast forward lookahead callback. 4098 * @buffer: Intel PT trace buffer 4099 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4100 * 4101 * Determine if @buffer trace is past the fast forward timestamp. 4102 * 4103 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4104 * timestamp, and 0 otherwise. 4105 */ 4106 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4107 { 4108 struct fast_forward_data *d = data; 4109 unsigned char *buf; 4110 uint64_t tsc; 4111 size_t rem; 4112 size_t len; 4113 4114 buf = (unsigned char *)buffer->buf; 4115 len = buffer->len; 4116 4117 if (!intel_pt_next_psb(&buf, &len) || 4118 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4119 return 0; 4120 4121 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4122 4123 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4124 tsc, buffer->ref_timestamp); 4125 4126 /* 4127 * If the buffer contains a timestamp earlier that the fast forward 4128 * timestamp, then record it, else stop. 4129 */ 4130 if (tsc < d->timestamp) 4131 d->buf_timestamp = buffer->ref_timestamp; 4132 else 4133 return 1; 4134 4135 return 0; 4136 } 4137 4138 /** 4139 * intel_pt_fast_forward - reposition decoder forwards. 4140 * @decoder: Intel PT decoder 4141 * @timestamp: timestamp to fast forward towards 4142 * 4143 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4144 * 4145 * Return: 0 on success or negative error code on failure. 4146 */ 4147 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4148 { 4149 struct fast_forward_data d = { .timestamp = timestamp }; 4150 unsigned char *buf; 4151 size_t len; 4152 int err; 4153 4154 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4155 4156 /* Find buffer timestamp of buffer to fast forward to */ 4157 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4158 if (err < 0) 4159 return err; 4160 4161 /* Walk to buffer with same buffer timestamp */ 4162 if (d.buf_timestamp) { 4163 do { 4164 decoder->pos += decoder->len; 4165 decoder->len = 0; 4166 err = intel_pt_get_next_data(decoder, true); 4167 /* -ENOLINK means non-consecutive trace */ 4168 if (err && err != -ENOLINK) 4169 return err; 4170 } while (decoder->buf_timestamp != d.buf_timestamp); 4171 } 4172 4173 if (!decoder->buf) 4174 return 0; 4175 4176 buf = (unsigned char *)decoder->buf; 4177 len = decoder->len; 4178 4179 if (!intel_pt_next_psb(&buf, &len)) 4180 return 0; 4181 4182 /* 4183 * Walk PSBs while the PSB timestamp is less than the fast forward 4184 * timestamp. 4185 */ 4186 do { 4187 uint64_t tsc; 4188 size_t rem; 4189 4190 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4191 break; 4192 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4193 /* 4194 * A TSC packet can slip past MTC packets but, after fast 4195 * forward, decoding starts at the TSC timestamp. That means 4196 * the timestamps may not be exactly the same as the timestamps 4197 * that would have been decoded without fast forward. 4198 */ 4199 if (tsc < timestamp) { 4200 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4201 decoder->pos += decoder->len - len; 4202 decoder->buf = buf; 4203 decoder->len = len; 4204 intel_pt_reposition(decoder); 4205 } else { 4206 break; 4207 } 4208 } while (intel_pt_step_psb(&buf, &len)); 4209 4210 return 0; 4211 } 4212