1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * 4 * Function graph tracer. 5 * Copyright (c) 2008-2009 Frederic Weisbecker <[email protected]> 6 * Mostly borrowed from function tracer which 7 * is Copyright (c) Steven Rostedt <[email protected]> 8 * 9 */ 10 #include <linux/uaccess.h> 11 #include <linux/ftrace.h> 12 #include <linux/interrupt.h> 13 #include <linux/slab.h> 14 #include <linux/fs.h> 15 16 #include "trace.h" 17 #include "trace_output.h" 18 19 /* When set, irq functions will be ignored */ 20 static int ftrace_graph_skip_irqs; 21 22 struct fgraph_cpu_data { 23 pid_t last_pid; 24 int depth; 25 int depth_irq; 26 int ignore; 27 unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; 28 }; 29 30 struct fgraph_data { 31 struct fgraph_cpu_data __percpu *cpu_data; 32 33 /* Place to preserve last processed entry. */ 34 union { 35 struct ftrace_graph_ent_entry ent; 36 struct fgraph_retaddr_ent_entry rent; 37 } ent; 38 struct ftrace_graph_ret_entry ret; 39 int failed; 40 int cpu; 41 }; 42 43 #define TRACE_GRAPH_INDENT 2 44 45 unsigned int fgraph_max_depth; 46 47 static struct tracer_opt trace_opts[] = { 48 /* Display overruns? (for self-debug purpose) */ 49 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 50 /* Display CPU ? */ 51 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 52 /* Display Overhead ? */ 53 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 54 /* Display proc name/pid */ 55 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 56 /* Display duration of execution */ 57 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, 58 /* Display absolute time of an entry */ 59 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, 60 /* Display interrupts */ 61 { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, 62 /* Display function name after trailing } */ 63 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, 64 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 65 /* Display function return value ? */ 66 { TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) }, 67 /* Display function return value in hexadecimal format ? */ 68 { TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) }, 69 #endif 70 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 71 /* Display function return address ? */ 72 { TRACER_OPT(funcgraph-retaddr, TRACE_GRAPH_PRINT_RETADDR) }, 73 #endif 74 /* Include sleep time (scheduled out) between entry and return */ 75 { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, 76 77 #ifdef CONFIG_FUNCTION_PROFILER 78 /* Include time within nested functions */ 79 { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, 80 #endif 81 82 { } /* Empty entry */ 83 }; 84 85 static struct tracer_flags tracer_flags = { 86 /* Don't display overruns, proc, or tail by default */ 87 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | 88 TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS | 89 TRACE_GRAPH_SLEEP_TIME | TRACE_GRAPH_GRAPH_TIME, 90 .opts = trace_opts 91 }; 92 93 static bool tracer_flags_is_set(u32 flags) 94 { 95 return (tracer_flags.val & flags) == flags; 96 } 97 98 /* 99 * DURATION column is being also used to display IRQ signs, 100 * following values are used by print_graph_irq and others 101 * to fill in space into DURATION column. 102 */ 103 enum { 104 FLAGS_FILL_FULL = 1 << TRACE_GRAPH_PRINT_FILL_SHIFT, 105 FLAGS_FILL_START = 2 << TRACE_GRAPH_PRINT_FILL_SHIFT, 106 FLAGS_FILL_END = 3 << TRACE_GRAPH_PRINT_FILL_SHIFT, 107 }; 108 109 static void 110 print_graph_duration(struct trace_array *tr, unsigned long long duration, 111 struct trace_seq *s, u32 flags); 112 113 int __trace_graph_entry(struct trace_array *tr, 114 struct ftrace_graph_ent *trace, 115 unsigned int trace_ctx) 116 { 117 struct ring_buffer_event *event; 118 struct trace_buffer *buffer = tr->array_buffer.buffer; 119 struct ftrace_graph_ent_entry *entry; 120 121 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, 122 sizeof(*entry), trace_ctx); 123 if (!event) 124 return 0; 125 entry = ring_buffer_event_data(event); 126 entry->graph_ent = *trace; 127 trace_buffer_unlock_commit_nostack(buffer, event); 128 129 return 1; 130 } 131 132 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 133 int __trace_graph_retaddr_entry(struct trace_array *tr, 134 struct ftrace_graph_ent *trace, 135 unsigned int trace_ctx, 136 unsigned long retaddr) 137 { 138 struct ring_buffer_event *event; 139 struct trace_buffer *buffer = tr->array_buffer.buffer; 140 struct fgraph_retaddr_ent_entry *entry; 141 142 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RETADDR_ENT, 143 sizeof(*entry), trace_ctx); 144 if (!event) 145 return 0; 146 entry = ring_buffer_event_data(event); 147 entry->graph_ent.func = trace->func; 148 entry->graph_ent.depth = trace->depth; 149 entry->graph_ent.retaddr = retaddr; 150 trace_buffer_unlock_commit_nostack(buffer, event); 151 152 return 1; 153 } 154 #else 155 int __trace_graph_retaddr_entry(struct trace_array *tr, 156 struct ftrace_graph_ent *trace, 157 unsigned int trace_ctx, 158 unsigned long retaddr) 159 { 160 return 1; 161 } 162 #endif 163 164 static inline int ftrace_graph_ignore_irqs(void) 165 { 166 if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT)) 167 return 0; 168 169 return in_hardirq(); 170 } 171 172 struct fgraph_times { 173 unsigned long long calltime; 174 unsigned long long sleeptime; /* may be optional! */ 175 }; 176 177 int trace_graph_entry(struct ftrace_graph_ent *trace, 178 struct fgraph_ops *gops) 179 { 180 unsigned long *task_var = fgraph_get_task_var(gops); 181 struct trace_array *tr = gops->private; 182 struct trace_array_cpu *data; 183 struct fgraph_times *ftimes; 184 unsigned int trace_ctx; 185 long disabled; 186 int ret = 0; 187 int cpu; 188 189 if (*task_var & TRACE_GRAPH_NOTRACE) 190 return 0; 191 192 /* 193 * Do not trace a function if it's filtered by set_graph_notrace. 194 * Make the index of ret stack negative to indicate that it should 195 * ignore further functions. But it needs its own ret stack entry 196 * to recover the original index in order to continue tracing after 197 * returning from the function. 198 */ 199 if (ftrace_graph_notrace_addr(trace->func)) { 200 *task_var |= TRACE_GRAPH_NOTRACE_BIT; 201 /* 202 * Need to return 1 to have the return called 203 * that will clear the NOTRACE bit. 204 */ 205 return 1; 206 } 207 208 if (!ftrace_trace_task(tr)) 209 return 0; 210 211 if (ftrace_graph_ignore_func(gops, trace)) 212 return 0; 213 214 if (ftrace_graph_ignore_irqs()) 215 return 0; 216 217 if (fgraph_sleep_time) { 218 /* Only need to record the calltime */ 219 ftimes = fgraph_reserve_data(gops->idx, sizeof(ftimes->calltime)); 220 } else { 221 ftimes = fgraph_reserve_data(gops->idx, sizeof(*ftimes)); 222 if (ftimes) 223 ftimes->sleeptime = current->ftrace_sleeptime; 224 } 225 if (!ftimes) 226 return 0; 227 228 ftimes->calltime = trace_clock_local(); 229 230 /* 231 * Stop here if tracing_threshold is set. We only write function return 232 * events to the ring buffer. 233 */ 234 if (tracing_thresh) 235 return 1; 236 237 preempt_disable_notrace(); 238 cpu = raw_smp_processor_id(); 239 data = per_cpu_ptr(tr->array_buffer.data, cpu); 240 disabled = atomic_read(&data->disabled); 241 if (likely(!disabled)) { 242 trace_ctx = tracing_gen_ctx(); 243 if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && 244 tracer_flags_is_set(TRACE_GRAPH_PRINT_RETADDR)) { 245 unsigned long retaddr = ftrace_graph_top_ret_addr(current); 246 ret = __trace_graph_retaddr_entry(tr, trace, trace_ctx, retaddr); 247 } else { 248 ret = __trace_graph_entry(tr, trace, trace_ctx); 249 } 250 } 251 preempt_enable_notrace(); 252 253 return ret; 254 } 255 256 static void 257 __trace_graph_function(struct trace_array *tr, 258 unsigned long ip, unsigned int trace_ctx) 259 { 260 u64 time = trace_clock_local(); 261 struct ftrace_graph_ent ent = { 262 .func = ip, 263 .depth = 0, 264 }; 265 struct ftrace_graph_ret ret = { 266 .func = ip, 267 .depth = 0, 268 .calltime = time, 269 .rettime = time, 270 }; 271 272 __trace_graph_entry(tr, &ent, trace_ctx); 273 __trace_graph_return(tr, &ret, trace_ctx); 274 } 275 276 void 277 trace_graph_function(struct trace_array *tr, 278 unsigned long ip, unsigned long parent_ip, 279 unsigned int trace_ctx) 280 { 281 __trace_graph_function(tr, ip, trace_ctx); 282 } 283 284 void __trace_graph_return(struct trace_array *tr, 285 struct ftrace_graph_ret *trace, 286 unsigned int trace_ctx) 287 { 288 struct ring_buffer_event *event; 289 struct trace_buffer *buffer = tr->array_buffer.buffer; 290 struct ftrace_graph_ret_entry *entry; 291 292 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, 293 sizeof(*entry), trace_ctx); 294 if (!event) 295 return; 296 entry = ring_buffer_event_data(event); 297 entry->ret = *trace; 298 trace_buffer_unlock_commit_nostack(buffer, event); 299 } 300 301 static void handle_nosleeptime(struct ftrace_graph_ret *trace, 302 struct fgraph_times *ftimes, 303 int size) 304 { 305 if (fgraph_sleep_time || size < sizeof(*ftimes)) 306 return; 307 308 ftimes->calltime += current->ftrace_sleeptime - ftimes->sleeptime; 309 } 310 311 void trace_graph_return(struct ftrace_graph_ret *trace, 312 struct fgraph_ops *gops) 313 { 314 unsigned long *task_var = fgraph_get_task_var(gops); 315 struct trace_array *tr = gops->private; 316 struct trace_array_cpu *data; 317 struct fgraph_times *ftimes; 318 unsigned int trace_ctx; 319 long disabled; 320 int size; 321 int cpu; 322 323 ftrace_graph_addr_finish(gops, trace); 324 325 if (*task_var & TRACE_GRAPH_NOTRACE) { 326 *task_var &= ~TRACE_GRAPH_NOTRACE; 327 return; 328 } 329 330 ftimes = fgraph_retrieve_data(gops->idx, &size); 331 if (!ftimes) 332 return; 333 334 handle_nosleeptime(trace, ftimes, size); 335 336 trace->calltime = ftimes->calltime; 337 338 preempt_disable_notrace(); 339 cpu = raw_smp_processor_id(); 340 data = per_cpu_ptr(tr->array_buffer.data, cpu); 341 disabled = atomic_read(&data->disabled); 342 if (likely(!disabled)) { 343 trace_ctx = tracing_gen_ctx(); 344 __trace_graph_return(tr, trace, trace_ctx); 345 } 346 preempt_enable_notrace(); 347 } 348 349 static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, 350 struct fgraph_ops *gops) 351 { 352 struct fgraph_times *ftimes; 353 int size; 354 355 ftrace_graph_addr_finish(gops, trace); 356 357 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { 358 trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); 359 return; 360 } 361 362 ftimes = fgraph_retrieve_data(gops->idx, &size); 363 if (!ftimes) 364 return; 365 366 handle_nosleeptime(trace, ftimes, size); 367 368 trace->calltime = ftimes->calltime; 369 370 if (tracing_thresh && 371 (trace->rettime - ftimes->calltime < tracing_thresh)) 372 return; 373 else 374 trace_graph_return(trace, gops); 375 } 376 377 static struct fgraph_ops funcgraph_ops = { 378 .entryfunc = &trace_graph_entry, 379 .retfunc = &trace_graph_return, 380 }; 381 382 int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 383 { 384 struct fgraph_ops *gops; 385 386 gops = kzalloc(sizeof(*gops), GFP_KERNEL); 387 if (!gops) 388 return -ENOMEM; 389 390 gops->entryfunc = &trace_graph_entry; 391 gops->retfunc = &trace_graph_return; 392 393 tr->gops = gops; 394 gops->private = tr; 395 396 fgraph_init_ops(&gops->ops, ops); 397 398 return 0; 399 } 400 401 void free_fgraph_ops(struct trace_array *tr) 402 { 403 kfree(tr->gops); 404 } 405 406 __init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 407 { 408 tr->gops = &funcgraph_ops; 409 funcgraph_ops.private = tr; 410 fgraph_init_ops(&tr->gops->ops, ops); 411 } 412 413 static int graph_trace_init(struct trace_array *tr) 414 { 415 int ret; 416 417 tr->gops->entryfunc = trace_graph_entry; 418 419 if (tracing_thresh) 420 tr->gops->retfunc = trace_graph_thresh_return; 421 else 422 tr->gops->retfunc = trace_graph_return; 423 424 /* Make gops functions are visible before we start tracing */ 425 smp_mb(); 426 427 ret = register_ftrace_graph(tr->gops); 428 if (ret) 429 return ret; 430 tracing_start_cmdline_record(); 431 432 return 0; 433 } 434 435 static void graph_trace_reset(struct trace_array *tr) 436 { 437 tracing_stop_cmdline_record(); 438 unregister_ftrace_graph(tr->gops); 439 } 440 441 static int graph_trace_update_thresh(struct trace_array *tr) 442 { 443 graph_trace_reset(tr); 444 return graph_trace_init(tr); 445 } 446 447 static int max_bytes_for_cpu; 448 449 static void print_graph_cpu(struct trace_seq *s, int cpu) 450 { 451 /* 452 * Start with a space character - to make it stand out 453 * to the right a bit when trace output is pasted into 454 * email: 455 */ 456 trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu); 457 } 458 459 #define TRACE_GRAPH_PROCINFO_LENGTH 14 460 461 static void print_graph_proc(struct trace_seq *s, pid_t pid) 462 { 463 char comm[TASK_COMM_LEN]; 464 /* sign + log10(MAX_INT) + '\0' */ 465 char pid_str[11]; 466 int spaces = 0; 467 int len; 468 int i; 469 470 trace_find_cmdline(pid, comm); 471 comm[7] = '\0'; 472 sprintf(pid_str, "%d", pid); 473 474 /* 1 stands for the "-" character */ 475 len = strlen(comm) + strlen(pid_str) + 1; 476 477 if (len < TRACE_GRAPH_PROCINFO_LENGTH) 478 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len; 479 480 /* First spaces to align center */ 481 for (i = 0; i < spaces / 2; i++) 482 trace_seq_putc(s, ' '); 483 484 trace_seq_printf(s, "%s-%s", comm, pid_str); 485 486 /* Last spaces to align center */ 487 for (i = 0; i < spaces - (spaces / 2); i++) 488 trace_seq_putc(s, ' '); 489 } 490 491 492 static void print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry) 493 { 494 trace_seq_putc(s, ' '); 495 trace_print_lat_fmt(s, entry); 496 trace_seq_puts(s, " | "); 497 } 498 499 /* If the pid changed since the last trace, output this event */ 500 static void 501 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data) 502 { 503 pid_t prev_pid; 504 pid_t *last_pid; 505 506 if (!data) 507 return; 508 509 last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 510 511 if (*last_pid == pid) 512 return; 513 514 prev_pid = *last_pid; 515 *last_pid = pid; 516 517 if (prev_pid == -1) 518 return; 519 /* 520 * Context-switch trace line: 521 522 ------------------------------------------ 523 | 1) migration/0--1 => sshd-1755 524 ------------------------------------------ 525 526 */ 527 trace_seq_puts(s, " ------------------------------------------\n"); 528 print_graph_cpu(s, cpu); 529 print_graph_proc(s, prev_pid); 530 trace_seq_puts(s, " => "); 531 print_graph_proc(s, pid); 532 trace_seq_puts(s, "\n ------------------------------------------\n\n"); 533 } 534 535 static struct ftrace_graph_ret_entry * 536 get_return_for_leaf(struct trace_iterator *iter, 537 struct ftrace_graph_ent_entry *curr) 538 { 539 struct fgraph_data *data = iter->private; 540 struct ring_buffer_iter *ring_iter = NULL; 541 struct ring_buffer_event *event; 542 struct ftrace_graph_ret_entry *next; 543 544 /* 545 * If the previous output failed to write to the seq buffer, 546 * then we just reuse the data from before. 547 */ 548 if (data && data->failed) { 549 curr = &data->ent.ent; 550 next = &data->ret; 551 } else { 552 553 ring_iter = trace_buffer_iter(iter, iter->cpu); 554 555 /* First peek to compare current entry and the next one */ 556 if (ring_iter) 557 event = ring_buffer_iter_peek(ring_iter, NULL); 558 else { 559 /* 560 * We need to consume the current entry to see 561 * the next one. 562 */ 563 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, 564 NULL, NULL); 565 event = ring_buffer_peek(iter->array_buffer->buffer, iter->cpu, 566 NULL, NULL); 567 } 568 569 if (!event) 570 return NULL; 571 572 next = ring_buffer_event_data(event); 573 574 if (data) { 575 /* 576 * Save current and next entries for later reference 577 * if the output fails. 578 */ 579 if (unlikely(curr->ent.type == TRACE_GRAPH_RETADDR_ENT)) 580 data->ent.rent = *(struct fgraph_retaddr_ent_entry *)curr; 581 else 582 data->ent.ent = *curr; 583 /* 584 * If the next event is not a return type, then 585 * we only care about what type it is. Otherwise we can 586 * safely copy the entire event. 587 */ 588 if (next->ent.type == TRACE_GRAPH_RET) 589 data->ret = *next; 590 else 591 data->ret.ent.type = next->ent.type; 592 } 593 } 594 595 if (next->ent.type != TRACE_GRAPH_RET) 596 return NULL; 597 598 if (curr->ent.pid != next->ent.pid || 599 curr->graph_ent.func != next->ret.func) 600 return NULL; 601 602 /* this is a leaf, now advance the iterator */ 603 if (ring_iter) 604 ring_buffer_iter_advance(ring_iter); 605 606 return next; 607 } 608 609 static void print_graph_abs_time(u64 t, struct trace_seq *s) 610 { 611 unsigned long usecs_rem; 612 613 usecs_rem = do_div(t, NSEC_PER_SEC); 614 usecs_rem /= 1000; 615 616 trace_seq_printf(s, "%5lu.%06lu | ", 617 (unsigned long)t, usecs_rem); 618 } 619 620 static void 621 print_graph_rel_time(struct trace_iterator *iter, struct trace_seq *s) 622 { 623 unsigned long long usecs; 624 625 usecs = iter->ts - iter->array_buffer->time_start; 626 do_div(usecs, NSEC_PER_USEC); 627 628 trace_seq_printf(s, "%9llu us | ", usecs); 629 } 630 631 static void 632 print_graph_irq(struct trace_iterator *iter, unsigned long addr, 633 enum trace_type type, int cpu, pid_t pid, u32 flags) 634 { 635 struct trace_array *tr = iter->tr; 636 struct trace_seq *s = &iter->seq; 637 struct trace_entry *ent = iter->ent; 638 639 addr += iter->tr->text_delta; 640 641 if (addr < (unsigned long)__irqentry_text_start || 642 addr >= (unsigned long)__irqentry_text_end) 643 return; 644 645 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 646 /* Absolute time */ 647 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 648 print_graph_abs_time(iter->ts, s); 649 650 /* Relative time */ 651 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 652 print_graph_rel_time(iter, s); 653 654 /* Cpu */ 655 if (flags & TRACE_GRAPH_PRINT_CPU) 656 print_graph_cpu(s, cpu); 657 658 /* Proc */ 659 if (flags & TRACE_GRAPH_PRINT_PROC) { 660 print_graph_proc(s, pid); 661 trace_seq_puts(s, " | "); 662 } 663 664 /* Latency format */ 665 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 666 print_graph_lat_fmt(s, ent); 667 } 668 669 /* No overhead */ 670 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_START); 671 672 if (type == TRACE_GRAPH_ENT) 673 trace_seq_puts(s, "==========>"); 674 else 675 trace_seq_puts(s, "<=========="); 676 677 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_END); 678 trace_seq_putc(s, '\n'); 679 } 680 681 void 682 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s) 683 { 684 unsigned long nsecs_rem = do_div(duration, 1000); 685 /* log10(ULONG_MAX) + '\0' */ 686 char usecs_str[21]; 687 char nsecs_str[5]; 688 int len; 689 int i; 690 691 sprintf(usecs_str, "%lu", (unsigned long) duration); 692 693 /* Print msecs */ 694 trace_seq_printf(s, "%s", usecs_str); 695 696 len = strlen(usecs_str); 697 698 /* Print nsecs (we don't want to exceed 7 numbers) */ 699 if (len < 7) { 700 size_t slen = min_t(size_t, sizeof(nsecs_str), 8UL - len); 701 702 snprintf(nsecs_str, slen, "%03lu", nsecs_rem); 703 trace_seq_printf(s, ".%s", nsecs_str); 704 len += strlen(nsecs_str) + 1; 705 } 706 707 trace_seq_puts(s, " us "); 708 709 /* Print remaining spaces to fit the row's width */ 710 for (i = len; i < 8; i++) 711 trace_seq_putc(s, ' '); 712 } 713 714 static void 715 print_graph_duration(struct trace_array *tr, unsigned long long duration, 716 struct trace_seq *s, u32 flags) 717 { 718 if (!(flags & TRACE_GRAPH_PRINT_DURATION) || 719 !(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 720 return; 721 722 /* No real adata, just filling the column with spaces */ 723 switch (flags & TRACE_GRAPH_PRINT_FILL_MASK) { 724 case FLAGS_FILL_FULL: 725 trace_seq_puts(s, " | "); 726 return; 727 case FLAGS_FILL_START: 728 trace_seq_puts(s, " "); 729 return; 730 case FLAGS_FILL_END: 731 trace_seq_puts(s, " |"); 732 return; 733 } 734 735 /* Signal a overhead of time execution to the output */ 736 if (flags & TRACE_GRAPH_PRINT_OVERHEAD) 737 trace_seq_printf(s, "%c ", trace_find_mark(duration)); 738 else 739 trace_seq_puts(s, " "); 740 741 trace_print_graph_duration(duration, s); 742 trace_seq_puts(s, "| "); 743 } 744 745 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 746 #define __TRACE_GRAPH_PRINT_RETVAL TRACE_GRAPH_PRINT_RETVAL 747 #else 748 #define __TRACE_GRAPH_PRINT_RETVAL 0 749 #endif 750 751 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 752 #define __TRACE_GRAPH_PRINT_RETADDR TRACE_GRAPH_PRINT_RETADDR 753 static void print_graph_retaddr(struct trace_seq *s, struct fgraph_retaddr_ent_entry *entry, 754 u32 trace_flags, bool comment) 755 { 756 if (comment) 757 trace_seq_puts(s, " /*"); 758 759 trace_seq_puts(s, " <-"); 760 seq_print_ip_sym(s, entry->graph_ent.retaddr, trace_flags | TRACE_ITER_SYM_OFFSET); 761 762 if (comment) 763 trace_seq_puts(s, " */"); 764 } 765 #else 766 #define __TRACE_GRAPH_PRINT_RETADDR 0 767 #define print_graph_retaddr(_seq, _entry, _tflags, _comment) do { } while (0) 768 #endif 769 770 #if defined(CONFIG_FUNCTION_GRAPH_RETVAL) || defined(CONFIG_FUNCTION_GRAPH_RETADDR) 771 772 static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entry *entry, 773 struct ftrace_graph_ret *graph_ret, void *func, 774 u32 opt_flags, u32 trace_flags) 775 { 776 unsigned long err_code = 0; 777 unsigned long retval = 0; 778 bool print_retaddr = false; 779 bool print_retval = false; 780 bool hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); 781 782 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 783 retval = graph_ret->retval; 784 print_retval = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL); 785 #endif 786 787 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 788 print_retaddr = !!(opt_flags & TRACE_GRAPH_PRINT_RETADDR); 789 #endif 790 791 if (print_retval && retval && !hex_format) { 792 /* Check if the return value matches the negative format */ 793 if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && 794 (((u64)retval) >> 32) == 0) { 795 err_code = sign_extend64(retval, 31); 796 } else { 797 err_code = retval; 798 } 799 800 if (!IS_ERR_VALUE(err_code)) 801 err_code = 0; 802 } 803 804 if (entry) { 805 if (entry->ent.type != TRACE_GRAPH_RETADDR_ENT) 806 print_retaddr = false; 807 808 trace_seq_printf(s, "%ps();", func); 809 if (print_retval || print_retaddr) 810 trace_seq_puts(s, " /*"); 811 else 812 trace_seq_putc(s, '\n'); 813 } else { 814 print_retaddr = false; 815 trace_seq_printf(s, "} /* %ps", func); 816 } 817 818 if (print_retaddr) 819 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 820 trace_flags, false); 821 822 if (print_retval) { 823 if (hex_format || (err_code == 0)) 824 trace_seq_printf(s, " ret=0x%lx", retval); 825 else 826 trace_seq_printf(s, " ret=%ld", err_code); 827 } 828 829 if (!entry || print_retval || print_retaddr) 830 trace_seq_puts(s, " */\n"); 831 } 832 833 #else 834 835 #define print_graph_retval(_seq, _ent, _ret, _func, _opt_flags, _trace_flags) do {} while (0) 836 837 #endif 838 839 /* Case of a leaf function on its call entry */ 840 static enum print_line_t 841 print_graph_entry_leaf(struct trace_iterator *iter, 842 struct ftrace_graph_ent_entry *entry, 843 struct ftrace_graph_ret_entry *ret_entry, 844 struct trace_seq *s, u32 flags) 845 { 846 struct fgraph_data *data = iter->private; 847 struct trace_array *tr = iter->tr; 848 struct ftrace_graph_ret *graph_ret; 849 struct ftrace_graph_ent *call; 850 unsigned long long duration; 851 unsigned long func; 852 int cpu = iter->cpu; 853 int i; 854 855 graph_ret = &ret_entry->ret; 856 call = &entry->graph_ent; 857 duration = graph_ret->rettime - graph_ret->calltime; 858 859 func = call->func + iter->tr->text_delta; 860 861 if (data) { 862 struct fgraph_cpu_data *cpu_data; 863 864 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 865 866 /* 867 * Comments display at + 1 to depth. Since 868 * this is a leaf function, keep the comments 869 * equal to this depth. 870 */ 871 cpu_data->depth = call->depth - 1; 872 873 /* No need to keep this function around for this depth */ 874 if (call->depth < FTRACE_RETFUNC_DEPTH && 875 !WARN_ON_ONCE(call->depth < 0)) 876 cpu_data->enter_funcs[call->depth] = 0; 877 } 878 879 /* Overhead and duration */ 880 print_graph_duration(tr, duration, s, flags); 881 882 /* Function */ 883 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 884 trace_seq_putc(s, ' '); 885 886 /* 887 * Write out the function return value or return address 888 */ 889 if (flags & (__TRACE_GRAPH_PRINT_RETVAL | __TRACE_GRAPH_PRINT_RETADDR)) { 890 print_graph_retval(s, entry, graph_ret, 891 (void *)graph_ret->func + iter->tr->text_delta, 892 flags, tr->trace_flags); 893 } else { 894 trace_seq_printf(s, "%ps();\n", (void *)func); 895 } 896 897 print_graph_irq(iter, graph_ret->func, TRACE_GRAPH_RET, 898 cpu, iter->ent->pid, flags); 899 900 return trace_handle_return(s); 901 } 902 903 static enum print_line_t 904 print_graph_entry_nested(struct trace_iterator *iter, 905 struct ftrace_graph_ent_entry *entry, 906 struct trace_seq *s, int cpu, u32 flags) 907 { 908 struct ftrace_graph_ent *call = &entry->graph_ent; 909 struct fgraph_data *data = iter->private; 910 struct trace_array *tr = iter->tr; 911 unsigned long func; 912 int i; 913 914 if (data) { 915 struct fgraph_cpu_data *cpu_data; 916 int cpu = iter->cpu; 917 918 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 919 cpu_data->depth = call->depth; 920 921 /* Save this function pointer to see if the exit matches */ 922 if (call->depth < FTRACE_RETFUNC_DEPTH && 923 !WARN_ON_ONCE(call->depth < 0)) 924 cpu_data->enter_funcs[call->depth] = call->func; 925 } 926 927 /* No time */ 928 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 929 930 /* Function */ 931 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 932 trace_seq_putc(s, ' '); 933 934 func = call->func + iter->tr->text_delta; 935 936 trace_seq_printf(s, "%ps() {", (void *)func); 937 if (flags & __TRACE_GRAPH_PRINT_RETADDR && 938 entry->ent.type == TRACE_GRAPH_RETADDR_ENT) 939 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 940 tr->trace_flags, true); 941 trace_seq_putc(s, '\n'); 942 943 if (trace_seq_has_overflowed(s)) 944 return TRACE_TYPE_PARTIAL_LINE; 945 946 /* 947 * we already consumed the current entry to check the next one 948 * and see if this is a leaf. 949 */ 950 return TRACE_TYPE_NO_CONSUME; 951 } 952 953 static void 954 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s, 955 int type, unsigned long addr, u32 flags) 956 { 957 struct fgraph_data *data = iter->private; 958 struct trace_entry *ent = iter->ent; 959 struct trace_array *tr = iter->tr; 960 int cpu = iter->cpu; 961 962 /* Pid */ 963 verif_pid(s, ent->pid, cpu, data); 964 965 if (type) 966 /* Interrupt */ 967 print_graph_irq(iter, addr, type, cpu, ent->pid, flags); 968 969 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 970 return; 971 972 /* Absolute time */ 973 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 974 print_graph_abs_time(iter->ts, s); 975 976 /* Relative time */ 977 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 978 print_graph_rel_time(iter, s); 979 980 /* Cpu */ 981 if (flags & TRACE_GRAPH_PRINT_CPU) 982 print_graph_cpu(s, cpu); 983 984 /* Proc */ 985 if (flags & TRACE_GRAPH_PRINT_PROC) { 986 print_graph_proc(s, ent->pid); 987 trace_seq_puts(s, " | "); 988 } 989 990 /* Latency format */ 991 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 992 print_graph_lat_fmt(s, ent); 993 994 return; 995 } 996 997 /* 998 * Entry check for irq code 999 * 1000 * returns 1 if 1001 * - we are inside irq code 1002 * - we just entered irq code 1003 * 1004 * returns 0 if 1005 * - funcgraph-interrupts option is set 1006 * - we are not inside irq code 1007 */ 1008 static int 1009 check_irq_entry(struct trace_iterator *iter, u32 flags, 1010 unsigned long addr, int depth) 1011 { 1012 int cpu = iter->cpu; 1013 int *depth_irq; 1014 struct fgraph_data *data = iter->private; 1015 1016 addr += iter->tr->text_delta; 1017 1018 /* 1019 * If we are either displaying irqs, or we got called as 1020 * a graph event and private data does not exist, 1021 * then we bypass the irq check. 1022 */ 1023 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1024 (!data)) 1025 return 0; 1026 1027 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1028 1029 /* 1030 * We are inside the irq code 1031 */ 1032 if (*depth_irq >= 0) 1033 return 1; 1034 1035 if ((addr < (unsigned long)__irqentry_text_start) || 1036 (addr >= (unsigned long)__irqentry_text_end)) 1037 return 0; 1038 1039 /* 1040 * We are entering irq code. 1041 */ 1042 *depth_irq = depth; 1043 return 1; 1044 } 1045 1046 /* 1047 * Return check for irq code 1048 * 1049 * returns 1 if 1050 * - we are inside irq code 1051 * - we just left irq code 1052 * 1053 * returns 0 if 1054 * - funcgraph-interrupts option is set 1055 * - we are not inside irq code 1056 */ 1057 static int 1058 check_irq_return(struct trace_iterator *iter, u32 flags, int depth) 1059 { 1060 int cpu = iter->cpu; 1061 int *depth_irq; 1062 struct fgraph_data *data = iter->private; 1063 1064 /* 1065 * If we are either displaying irqs, or we got called as 1066 * a graph event and private data does not exist, 1067 * then we bypass the irq check. 1068 */ 1069 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1070 (!data)) 1071 return 0; 1072 1073 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1074 1075 /* 1076 * We are not inside the irq code. 1077 */ 1078 if (*depth_irq == -1) 1079 return 0; 1080 1081 /* 1082 * We are inside the irq code, and this is returning entry. 1083 * Let's not trace it and clear the entry depth, since 1084 * we are out of irq code. 1085 * 1086 * This condition ensures that we 'leave the irq code' once 1087 * we are out of the entry depth. Thus protecting us from 1088 * the RETURN entry loss. 1089 */ 1090 if (*depth_irq >= depth) { 1091 *depth_irq = -1; 1092 return 1; 1093 } 1094 1095 /* 1096 * We are inside the irq code, and this is not the entry. 1097 */ 1098 return 1; 1099 } 1100 1101 static enum print_line_t 1102 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 1103 struct trace_iterator *iter, u32 flags) 1104 { 1105 struct fgraph_data *data = iter->private; 1106 struct ftrace_graph_ent *call = &field->graph_ent; 1107 struct ftrace_graph_ret_entry *leaf_ret; 1108 static enum print_line_t ret; 1109 int cpu = iter->cpu; 1110 1111 if (check_irq_entry(iter, flags, call->func, call->depth)) 1112 return TRACE_TYPE_HANDLED; 1113 1114 print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags); 1115 1116 leaf_ret = get_return_for_leaf(iter, field); 1117 if (leaf_ret) 1118 ret = print_graph_entry_leaf(iter, field, leaf_ret, s, flags); 1119 else 1120 ret = print_graph_entry_nested(iter, field, s, cpu, flags); 1121 1122 if (data) { 1123 /* 1124 * If we failed to write our output, then we need to make 1125 * note of it. Because we already consumed our entry. 1126 */ 1127 if (s->full) { 1128 data->failed = 1; 1129 data->cpu = cpu; 1130 } else 1131 data->failed = 0; 1132 } 1133 1134 return ret; 1135 } 1136 1137 static enum print_line_t 1138 print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, 1139 struct trace_entry *ent, struct trace_iterator *iter, 1140 u32 flags) 1141 { 1142 unsigned long long duration = trace->rettime - trace->calltime; 1143 struct fgraph_data *data = iter->private; 1144 struct trace_array *tr = iter->tr; 1145 unsigned long func; 1146 pid_t pid = ent->pid; 1147 int cpu = iter->cpu; 1148 int func_match = 1; 1149 int i; 1150 1151 func = trace->func + iter->tr->text_delta; 1152 1153 if (check_irq_return(iter, flags, trace->depth)) 1154 return TRACE_TYPE_HANDLED; 1155 1156 if (data) { 1157 struct fgraph_cpu_data *cpu_data; 1158 int cpu = iter->cpu; 1159 1160 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 1161 1162 /* 1163 * Comments display at + 1 to depth. This is the 1164 * return from a function, we now want the comments 1165 * to display at the same level of the bracket. 1166 */ 1167 cpu_data->depth = trace->depth - 1; 1168 1169 if (trace->depth < FTRACE_RETFUNC_DEPTH && 1170 !WARN_ON_ONCE(trace->depth < 0)) { 1171 if (cpu_data->enter_funcs[trace->depth] != trace->func) 1172 func_match = 0; 1173 cpu_data->enter_funcs[trace->depth] = 0; 1174 } 1175 } 1176 1177 print_graph_prologue(iter, s, 0, 0, flags); 1178 1179 /* Overhead and duration */ 1180 print_graph_duration(tr, duration, s, flags); 1181 1182 /* Closing brace */ 1183 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) 1184 trace_seq_putc(s, ' '); 1185 1186 /* 1187 * Always write out the function name and its return value if the 1188 * funcgraph-retval option is enabled. 1189 */ 1190 if (flags & __TRACE_GRAPH_PRINT_RETVAL) { 1191 print_graph_retval(s, NULL, trace, (void *)func, flags, tr->trace_flags); 1192 } else { 1193 /* 1194 * If the return function does not have a matching entry, 1195 * then the entry was lost. Instead of just printing 1196 * the '}' and letting the user guess what function this 1197 * belongs to, write out the function name. Always do 1198 * that if the funcgraph-tail option is enabled. 1199 */ 1200 if (func_match && !(flags & TRACE_GRAPH_PRINT_TAIL)) 1201 trace_seq_puts(s, "}\n"); 1202 else 1203 trace_seq_printf(s, "} /* %ps */\n", (void *)func); 1204 } 1205 1206 /* Overrun */ 1207 if (flags & TRACE_GRAPH_PRINT_OVERRUN) 1208 trace_seq_printf(s, " (Overruns: %u)\n", 1209 trace->overrun); 1210 1211 print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, 1212 cpu, pid, flags); 1213 1214 return trace_handle_return(s); 1215 } 1216 1217 static enum print_line_t 1218 print_graph_comment(struct trace_seq *s, struct trace_entry *ent, 1219 struct trace_iterator *iter, u32 flags) 1220 { 1221 struct trace_array *tr = iter->tr; 1222 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 1223 struct fgraph_data *data = iter->private; 1224 struct trace_event *event; 1225 int depth = 0; 1226 int ret; 1227 int i; 1228 1229 if (data) 1230 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth; 1231 1232 print_graph_prologue(iter, s, 0, 0, flags); 1233 1234 /* No time */ 1235 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 1236 1237 /* Indentation */ 1238 if (depth > 0) 1239 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) 1240 trace_seq_putc(s, ' '); 1241 1242 /* The comment */ 1243 trace_seq_puts(s, "/* "); 1244 1245 switch (iter->ent->type) { 1246 case TRACE_BPUTS: 1247 ret = trace_print_bputs_msg_only(iter); 1248 if (ret != TRACE_TYPE_HANDLED) 1249 return ret; 1250 break; 1251 case TRACE_BPRINT: 1252 ret = trace_print_bprintk_msg_only(iter); 1253 if (ret != TRACE_TYPE_HANDLED) 1254 return ret; 1255 break; 1256 case TRACE_PRINT: 1257 ret = trace_print_printk_msg_only(iter); 1258 if (ret != TRACE_TYPE_HANDLED) 1259 return ret; 1260 break; 1261 default: 1262 event = ftrace_find_event(ent->type); 1263 if (!event) 1264 return TRACE_TYPE_UNHANDLED; 1265 1266 ret = event->funcs->trace(iter, sym_flags, event); 1267 if (ret != TRACE_TYPE_HANDLED) 1268 return ret; 1269 } 1270 1271 if (trace_seq_has_overflowed(s)) 1272 goto out; 1273 1274 /* Strip ending newline */ 1275 if (s->buffer[s->seq.len - 1] == '\n') { 1276 s->buffer[s->seq.len - 1] = '\0'; 1277 s->seq.len--; 1278 } 1279 1280 trace_seq_puts(s, " */\n"); 1281 out: 1282 return trace_handle_return(s); 1283 } 1284 1285 1286 enum print_line_t 1287 print_graph_function_flags(struct trace_iterator *iter, u32 flags) 1288 { 1289 struct ftrace_graph_ent_entry *field; 1290 struct fgraph_data *data = iter->private; 1291 struct trace_entry *entry = iter->ent; 1292 struct trace_seq *s = &iter->seq; 1293 int cpu = iter->cpu; 1294 int ret; 1295 1296 if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) { 1297 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0; 1298 return TRACE_TYPE_HANDLED; 1299 } 1300 1301 /* 1302 * If the last output failed, there's a possibility we need 1303 * to print out the missing entry which would never go out. 1304 */ 1305 if (data && data->failed) { 1306 field = &data->ent.ent; 1307 iter->cpu = data->cpu; 1308 ret = print_graph_entry(field, s, iter, flags); 1309 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { 1310 per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; 1311 ret = TRACE_TYPE_NO_CONSUME; 1312 } 1313 iter->cpu = cpu; 1314 return ret; 1315 } 1316 1317 switch (entry->type) { 1318 case TRACE_GRAPH_ENT: { 1319 /* 1320 * print_graph_entry() may consume the current event, 1321 * thus @field may become invalid, so we need to save it. 1322 * sizeof(struct ftrace_graph_ent_entry) is very small, 1323 * it can be safely saved at the stack. 1324 */ 1325 struct ftrace_graph_ent_entry saved; 1326 trace_assign_type(field, entry); 1327 saved = *field; 1328 return print_graph_entry(&saved, s, iter, flags); 1329 } 1330 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1331 case TRACE_GRAPH_RETADDR_ENT: { 1332 struct fgraph_retaddr_ent_entry saved; 1333 struct fgraph_retaddr_ent_entry *rfield; 1334 1335 trace_assign_type(rfield, entry); 1336 saved = *rfield; 1337 return print_graph_entry((struct ftrace_graph_ent_entry *)&saved, s, iter, flags); 1338 } 1339 #endif 1340 case TRACE_GRAPH_RET: { 1341 struct ftrace_graph_ret_entry *field; 1342 trace_assign_type(field, entry); 1343 return print_graph_return(&field->ret, s, entry, iter, flags); 1344 } 1345 case TRACE_STACK: 1346 case TRACE_FN: 1347 /* dont trace stack and functions as comments */ 1348 return TRACE_TYPE_UNHANDLED; 1349 1350 default: 1351 return print_graph_comment(s, entry, iter, flags); 1352 } 1353 1354 return TRACE_TYPE_HANDLED; 1355 } 1356 1357 static enum print_line_t 1358 print_graph_function(struct trace_iterator *iter) 1359 { 1360 return print_graph_function_flags(iter, tracer_flags.val); 1361 } 1362 1363 static enum print_line_t 1364 print_graph_function_event(struct trace_iterator *iter, int flags, 1365 struct trace_event *event) 1366 { 1367 return print_graph_function(iter); 1368 } 1369 1370 static void print_lat_header(struct seq_file *s, u32 flags) 1371 { 1372 static const char spaces[] = " " /* 16 spaces */ 1373 " " /* 4 spaces */ 1374 " "; /* 17 spaces */ 1375 int size = 0; 1376 1377 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1378 size += 16; 1379 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1380 size += 16; 1381 if (flags & TRACE_GRAPH_PRINT_CPU) 1382 size += 4; 1383 if (flags & TRACE_GRAPH_PRINT_PROC) 1384 size += 17; 1385 1386 seq_printf(s, "#%.*s _-----=> irqs-off \n", size, spaces); 1387 seq_printf(s, "#%.*s / _----=> need-resched \n", size, spaces); 1388 seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces); 1389 seq_printf(s, "#%.*s|| / _--=> preempt-depth \n", size, spaces); 1390 seq_printf(s, "#%.*s||| / \n", size, spaces); 1391 } 1392 1393 static void __print_graph_headers_flags(struct trace_array *tr, 1394 struct seq_file *s, u32 flags) 1395 { 1396 int lat = tr->trace_flags & TRACE_ITER_LATENCY_FMT; 1397 1398 if (lat) 1399 print_lat_header(s, flags); 1400 1401 /* 1st line */ 1402 seq_putc(s, '#'); 1403 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1404 seq_puts(s, " TIME "); 1405 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1406 seq_puts(s, " REL TIME "); 1407 if (flags & TRACE_GRAPH_PRINT_CPU) 1408 seq_puts(s, " CPU"); 1409 if (flags & TRACE_GRAPH_PRINT_PROC) 1410 seq_puts(s, " TASK/PID "); 1411 if (lat) 1412 seq_puts(s, "|||| "); 1413 if (flags & TRACE_GRAPH_PRINT_DURATION) 1414 seq_puts(s, " DURATION "); 1415 seq_puts(s, " FUNCTION CALLS\n"); 1416 1417 /* 2nd line */ 1418 seq_putc(s, '#'); 1419 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1420 seq_puts(s, " | "); 1421 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1422 seq_puts(s, " | "); 1423 if (flags & TRACE_GRAPH_PRINT_CPU) 1424 seq_puts(s, " | "); 1425 if (flags & TRACE_GRAPH_PRINT_PROC) 1426 seq_puts(s, " | | "); 1427 if (lat) 1428 seq_puts(s, "|||| "); 1429 if (flags & TRACE_GRAPH_PRINT_DURATION) 1430 seq_puts(s, " | | "); 1431 seq_puts(s, " | | | |\n"); 1432 } 1433 1434 static void print_graph_headers(struct seq_file *s) 1435 { 1436 print_graph_headers_flags(s, tracer_flags.val); 1437 } 1438 1439 void print_graph_headers_flags(struct seq_file *s, u32 flags) 1440 { 1441 struct trace_iterator *iter = s->private; 1442 struct trace_array *tr = iter->tr; 1443 1444 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 1445 return; 1446 1447 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) { 1448 /* print nothing if the buffers are empty */ 1449 if (trace_empty(iter)) 1450 return; 1451 1452 print_trace_header(s, iter); 1453 } 1454 1455 __print_graph_headers_flags(tr, s, flags); 1456 } 1457 1458 void graph_trace_open(struct trace_iterator *iter) 1459 { 1460 /* pid and depth on the last trace processed */ 1461 struct fgraph_data *data; 1462 gfp_t gfpflags; 1463 int cpu; 1464 1465 iter->private = NULL; 1466 1467 /* We can be called in atomic context via ftrace_dump() */ 1468 gfpflags = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL; 1469 1470 data = kzalloc(sizeof(*data), gfpflags); 1471 if (!data) 1472 goto out_err; 1473 1474 data->cpu_data = alloc_percpu_gfp(struct fgraph_cpu_data, gfpflags); 1475 if (!data->cpu_data) 1476 goto out_err_free; 1477 1478 for_each_possible_cpu(cpu) { 1479 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 1480 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); 1481 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore); 1482 int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1483 1484 *pid = -1; 1485 *depth = 0; 1486 *ignore = 0; 1487 *depth_irq = -1; 1488 } 1489 1490 iter->private = data; 1491 1492 return; 1493 1494 out_err_free: 1495 kfree(data); 1496 out_err: 1497 pr_warn("function graph tracer: not enough memory\n"); 1498 } 1499 1500 void graph_trace_close(struct trace_iterator *iter) 1501 { 1502 struct fgraph_data *data = iter->private; 1503 1504 if (data) { 1505 free_percpu(data->cpu_data); 1506 kfree(data); 1507 } 1508 } 1509 1510 static int 1511 func_graph_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1512 { 1513 if (bit == TRACE_GRAPH_PRINT_IRQS) 1514 ftrace_graph_skip_irqs = !set; 1515 1516 if (bit == TRACE_GRAPH_SLEEP_TIME) 1517 ftrace_graph_sleep_time_control(set); 1518 1519 if (bit == TRACE_GRAPH_GRAPH_TIME) 1520 ftrace_graph_graph_time_control(set); 1521 1522 return 0; 1523 } 1524 1525 static struct trace_event_functions graph_functions = { 1526 .trace = print_graph_function_event, 1527 }; 1528 1529 static struct trace_event graph_trace_entry_event = { 1530 .type = TRACE_GRAPH_ENT, 1531 .funcs = &graph_functions, 1532 }; 1533 1534 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1535 static struct trace_event graph_trace_retaddr_entry_event = { 1536 .type = TRACE_GRAPH_RETADDR_ENT, 1537 .funcs = &graph_functions, 1538 }; 1539 #endif 1540 1541 static struct trace_event graph_trace_ret_event = { 1542 .type = TRACE_GRAPH_RET, 1543 .funcs = &graph_functions 1544 }; 1545 1546 static struct tracer graph_trace __tracer_data = { 1547 .name = "function_graph", 1548 .update_thresh = graph_trace_update_thresh, 1549 .open = graph_trace_open, 1550 .pipe_open = graph_trace_open, 1551 .close = graph_trace_close, 1552 .pipe_close = graph_trace_close, 1553 .init = graph_trace_init, 1554 .reset = graph_trace_reset, 1555 .print_line = print_graph_function, 1556 .print_header = print_graph_headers, 1557 .flags = &tracer_flags, 1558 .set_flag = func_graph_set_flag, 1559 .allow_instances = true, 1560 #ifdef CONFIG_FTRACE_SELFTEST 1561 .selftest = trace_selftest_startup_function_graph, 1562 #endif 1563 }; 1564 1565 1566 static ssize_t 1567 graph_depth_write(struct file *filp, const char __user *ubuf, size_t cnt, 1568 loff_t *ppos) 1569 { 1570 unsigned long val; 1571 int ret; 1572 1573 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 1574 if (ret) 1575 return ret; 1576 1577 fgraph_max_depth = val; 1578 1579 *ppos += cnt; 1580 1581 return cnt; 1582 } 1583 1584 static ssize_t 1585 graph_depth_read(struct file *filp, char __user *ubuf, size_t cnt, 1586 loff_t *ppos) 1587 { 1588 char buf[15]; /* More than enough to hold UINT_MAX + "\n"*/ 1589 int n; 1590 1591 n = sprintf(buf, "%d\n", fgraph_max_depth); 1592 1593 return simple_read_from_buffer(ubuf, cnt, ppos, buf, n); 1594 } 1595 1596 static const struct file_operations graph_depth_fops = { 1597 .open = tracing_open_generic, 1598 .write = graph_depth_write, 1599 .read = graph_depth_read, 1600 .llseek = generic_file_llseek, 1601 }; 1602 1603 static __init int init_graph_tracefs(void) 1604 { 1605 int ret; 1606 1607 ret = tracing_init_dentry(); 1608 if (ret) 1609 return 0; 1610 1611 trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL, 1612 NULL, &graph_depth_fops); 1613 1614 return 0; 1615 } 1616 fs_initcall(init_graph_tracefs); 1617 1618 static __init int init_graph_trace(void) 1619 { 1620 max_bytes_for_cpu = snprintf(NULL, 0, "%u", nr_cpu_ids - 1); 1621 1622 if (!register_trace_event(&graph_trace_entry_event)) { 1623 pr_warn("Warning: could not register graph trace events\n"); 1624 return 1; 1625 } 1626 1627 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1628 if (!register_trace_event(&graph_trace_retaddr_entry_event)) { 1629 pr_warn("Warning: could not register graph trace retaddr events\n"); 1630 return 1; 1631 } 1632 #endif 1633 1634 if (!register_trace_event(&graph_trace_ret_event)) { 1635 pr_warn("Warning: could not register graph trace events\n"); 1636 return 1; 1637 } 1638 1639 return register_tracer(&graph_trace); 1640 } 1641 1642 core_initcall(init_graph_trace); 1643