1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * 4 * Function graph tracer. 5 * Copyright (c) 2008-2009 Frederic Weisbecker <[email protected]> 6 * Mostly borrowed from function tracer which 7 * is Copyright (c) Steven Rostedt <[email protected]> 8 * 9 */ 10 #include <linux/uaccess.h> 11 #include <linux/ftrace.h> 12 #include <linux/interrupt.h> 13 #include <linux/slab.h> 14 #include <linux/fs.h> 15 16 #include "trace.h" 17 #include "trace_output.h" 18 19 /* When set, irq functions will be ignored */ 20 static int ftrace_graph_skip_irqs; 21 22 struct fgraph_cpu_data { 23 pid_t last_pid; 24 int depth; 25 int depth_irq; 26 int ignore; 27 unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; 28 }; 29 30 struct fgraph_data { 31 struct fgraph_cpu_data __percpu *cpu_data; 32 33 /* Place to preserve last processed entry. */ 34 union { 35 struct ftrace_graph_ent_entry ent; 36 struct fgraph_retaddr_ent_entry rent; 37 } ent; 38 struct ftrace_graph_ret_entry ret; 39 int failed; 40 int cpu; 41 }; 42 43 #define TRACE_GRAPH_INDENT 2 44 45 unsigned int fgraph_max_depth; 46 47 static struct tracer_opt trace_opts[] = { 48 /* Display overruns? (for self-debug purpose) */ 49 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 50 /* Display CPU ? */ 51 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 52 /* Display Overhead ? */ 53 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 54 /* Display proc name/pid */ 55 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 56 /* Display duration of execution */ 57 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, 58 /* Display absolute time of an entry */ 59 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, 60 /* Display interrupts */ 61 { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, 62 /* Display function name after trailing } */ 63 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, 64 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 65 /* Display function return value ? */ 66 { TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) }, 67 /* Display function return value in hexadecimal format ? */ 68 { TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) }, 69 #endif 70 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 71 /* Display function return address ? */ 72 { TRACER_OPT(funcgraph-retaddr, TRACE_GRAPH_PRINT_RETADDR) }, 73 #endif 74 /* Include sleep time (scheduled out) between entry and return */ 75 { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, 76 77 #ifdef CONFIG_FUNCTION_PROFILER 78 /* Include time within nested functions */ 79 { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, 80 #endif 81 82 { } /* Empty entry */ 83 }; 84 85 static struct tracer_flags tracer_flags = { 86 /* Don't display overruns, proc, or tail by default */ 87 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | 88 TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS | 89 TRACE_GRAPH_SLEEP_TIME | TRACE_GRAPH_GRAPH_TIME, 90 .opts = trace_opts 91 }; 92 93 static bool tracer_flags_is_set(u32 flags) 94 { 95 return (tracer_flags.val & flags) == flags; 96 } 97 98 /* 99 * DURATION column is being also used to display IRQ signs, 100 * following values are used by print_graph_irq and others 101 * to fill in space into DURATION column. 102 */ 103 enum { 104 FLAGS_FILL_FULL = 1 << TRACE_GRAPH_PRINT_FILL_SHIFT, 105 FLAGS_FILL_START = 2 << TRACE_GRAPH_PRINT_FILL_SHIFT, 106 FLAGS_FILL_END = 3 << TRACE_GRAPH_PRINT_FILL_SHIFT, 107 }; 108 109 static void 110 print_graph_duration(struct trace_array *tr, unsigned long long duration, 111 struct trace_seq *s, u32 flags); 112 113 int __trace_graph_entry(struct trace_array *tr, 114 struct ftrace_graph_ent *trace, 115 unsigned int trace_ctx) 116 { 117 struct ring_buffer_event *event; 118 struct trace_buffer *buffer = tr->array_buffer.buffer; 119 struct ftrace_graph_ent_entry *entry; 120 121 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, 122 sizeof(*entry), trace_ctx); 123 if (!event) 124 return 0; 125 entry = ring_buffer_event_data(event); 126 entry->graph_ent = *trace; 127 trace_buffer_unlock_commit_nostack(buffer, event); 128 129 return 1; 130 } 131 132 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 133 int __trace_graph_retaddr_entry(struct trace_array *tr, 134 struct ftrace_graph_ent *trace, 135 unsigned int trace_ctx, 136 unsigned long retaddr) 137 { 138 struct ring_buffer_event *event; 139 struct trace_buffer *buffer = tr->array_buffer.buffer; 140 struct fgraph_retaddr_ent_entry *entry; 141 142 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RETADDR_ENT, 143 sizeof(*entry), trace_ctx); 144 if (!event) 145 return 0; 146 entry = ring_buffer_event_data(event); 147 entry->graph_ent.func = trace->func; 148 entry->graph_ent.depth = trace->depth; 149 entry->graph_ent.retaddr = retaddr; 150 trace_buffer_unlock_commit_nostack(buffer, event); 151 152 return 1; 153 } 154 #else 155 int __trace_graph_retaddr_entry(struct trace_array *tr, 156 struct ftrace_graph_ent *trace, 157 unsigned int trace_ctx, 158 unsigned long retaddr) 159 { 160 return 1; 161 } 162 #endif 163 164 static inline int ftrace_graph_ignore_irqs(void) 165 { 166 if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT)) 167 return 0; 168 169 return in_hardirq(); 170 } 171 172 struct fgraph_times { 173 unsigned long long calltime; 174 unsigned long long sleeptime; /* may be optional! */ 175 }; 176 177 int trace_graph_entry(struct ftrace_graph_ent *trace, 178 struct fgraph_ops *gops, 179 struct ftrace_regs *fregs) 180 { 181 unsigned long *task_var = fgraph_get_task_var(gops); 182 struct trace_array *tr = gops->private; 183 struct trace_array_cpu *data; 184 struct fgraph_times *ftimes; 185 unsigned int trace_ctx; 186 long disabled; 187 int ret = 0; 188 int cpu; 189 190 if (*task_var & TRACE_GRAPH_NOTRACE) 191 return 0; 192 193 /* 194 * Do not trace a function if it's filtered by set_graph_notrace. 195 * Make the index of ret stack negative to indicate that it should 196 * ignore further functions. But it needs its own ret stack entry 197 * to recover the original index in order to continue tracing after 198 * returning from the function. 199 */ 200 if (ftrace_graph_notrace_addr(trace->func)) { 201 *task_var |= TRACE_GRAPH_NOTRACE_BIT; 202 /* 203 * Need to return 1 to have the return called 204 * that will clear the NOTRACE bit. 205 */ 206 return 1; 207 } 208 209 if (!ftrace_trace_task(tr)) 210 return 0; 211 212 if (ftrace_graph_ignore_func(gops, trace)) 213 return 0; 214 215 if (ftrace_graph_ignore_irqs()) 216 return 0; 217 218 if (fgraph_sleep_time) { 219 /* Only need to record the calltime */ 220 ftimes = fgraph_reserve_data(gops->idx, sizeof(ftimes->calltime)); 221 } else { 222 ftimes = fgraph_reserve_data(gops->idx, sizeof(*ftimes)); 223 if (ftimes) 224 ftimes->sleeptime = current->ftrace_sleeptime; 225 } 226 if (!ftimes) 227 return 0; 228 229 ftimes->calltime = trace_clock_local(); 230 231 /* 232 * Stop here if tracing_threshold is set. We only write function return 233 * events to the ring buffer. 234 */ 235 if (tracing_thresh) 236 return 1; 237 238 preempt_disable_notrace(); 239 cpu = raw_smp_processor_id(); 240 data = per_cpu_ptr(tr->array_buffer.data, cpu); 241 disabled = atomic_read(&data->disabled); 242 if (likely(!disabled)) { 243 trace_ctx = tracing_gen_ctx(); 244 if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && 245 tracer_flags_is_set(TRACE_GRAPH_PRINT_RETADDR)) { 246 unsigned long retaddr = ftrace_graph_top_ret_addr(current); 247 ret = __trace_graph_retaddr_entry(tr, trace, trace_ctx, retaddr); 248 } else { 249 ret = __trace_graph_entry(tr, trace, trace_ctx); 250 } 251 } 252 preempt_enable_notrace(); 253 254 return ret; 255 } 256 257 static void 258 __trace_graph_function(struct trace_array *tr, 259 unsigned long ip, unsigned int trace_ctx) 260 { 261 u64 time = trace_clock_local(); 262 struct ftrace_graph_ent ent = { 263 .func = ip, 264 .depth = 0, 265 }; 266 struct ftrace_graph_ret ret = { 267 .func = ip, 268 .depth = 0, 269 .calltime = time, 270 .rettime = time, 271 }; 272 273 __trace_graph_entry(tr, &ent, trace_ctx); 274 __trace_graph_return(tr, &ret, trace_ctx); 275 } 276 277 void 278 trace_graph_function(struct trace_array *tr, 279 unsigned long ip, unsigned long parent_ip, 280 unsigned int trace_ctx) 281 { 282 __trace_graph_function(tr, ip, trace_ctx); 283 } 284 285 void __trace_graph_return(struct trace_array *tr, 286 struct ftrace_graph_ret *trace, 287 unsigned int trace_ctx) 288 { 289 struct ring_buffer_event *event; 290 struct trace_buffer *buffer = tr->array_buffer.buffer; 291 struct ftrace_graph_ret_entry *entry; 292 293 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, 294 sizeof(*entry), trace_ctx); 295 if (!event) 296 return; 297 entry = ring_buffer_event_data(event); 298 entry->ret = *trace; 299 trace_buffer_unlock_commit_nostack(buffer, event); 300 } 301 302 static void handle_nosleeptime(struct ftrace_graph_ret *trace, 303 struct fgraph_times *ftimes, 304 int size) 305 { 306 if (fgraph_sleep_time || size < sizeof(*ftimes)) 307 return; 308 309 ftimes->calltime += current->ftrace_sleeptime - ftimes->sleeptime; 310 } 311 312 void trace_graph_return(struct ftrace_graph_ret *trace, 313 struct fgraph_ops *gops) 314 { 315 unsigned long *task_var = fgraph_get_task_var(gops); 316 struct trace_array *tr = gops->private; 317 struct trace_array_cpu *data; 318 struct fgraph_times *ftimes; 319 unsigned int trace_ctx; 320 long disabled; 321 int size; 322 int cpu; 323 324 ftrace_graph_addr_finish(gops, trace); 325 326 if (*task_var & TRACE_GRAPH_NOTRACE) { 327 *task_var &= ~TRACE_GRAPH_NOTRACE; 328 return; 329 } 330 331 ftimes = fgraph_retrieve_data(gops->idx, &size); 332 if (!ftimes) 333 return; 334 335 handle_nosleeptime(trace, ftimes, size); 336 337 trace->calltime = ftimes->calltime; 338 339 preempt_disable_notrace(); 340 cpu = raw_smp_processor_id(); 341 data = per_cpu_ptr(tr->array_buffer.data, cpu); 342 disabled = atomic_read(&data->disabled); 343 if (likely(!disabled)) { 344 trace_ctx = tracing_gen_ctx(); 345 __trace_graph_return(tr, trace, trace_ctx); 346 } 347 preempt_enable_notrace(); 348 } 349 350 static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, 351 struct fgraph_ops *gops) 352 { 353 struct fgraph_times *ftimes; 354 int size; 355 356 ftrace_graph_addr_finish(gops, trace); 357 358 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { 359 trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); 360 return; 361 } 362 363 ftimes = fgraph_retrieve_data(gops->idx, &size); 364 if (!ftimes) 365 return; 366 367 handle_nosleeptime(trace, ftimes, size); 368 369 trace->calltime = ftimes->calltime; 370 371 if (tracing_thresh && 372 (trace->rettime - ftimes->calltime < tracing_thresh)) 373 return; 374 else 375 trace_graph_return(trace, gops); 376 } 377 378 static struct fgraph_ops funcgraph_ops = { 379 .entryfunc = &trace_graph_entry, 380 .retfunc = &trace_graph_return, 381 }; 382 383 int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 384 { 385 struct fgraph_ops *gops; 386 387 gops = kzalloc(sizeof(*gops), GFP_KERNEL); 388 if (!gops) 389 return -ENOMEM; 390 391 gops->entryfunc = &trace_graph_entry; 392 gops->retfunc = &trace_graph_return; 393 394 tr->gops = gops; 395 gops->private = tr; 396 397 fgraph_init_ops(&gops->ops, ops); 398 399 return 0; 400 } 401 402 void free_fgraph_ops(struct trace_array *tr) 403 { 404 kfree(tr->gops); 405 } 406 407 __init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 408 { 409 tr->gops = &funcgraph_ops; 410 funcgraph_ops.private = tr; 411 fgraph_init_ops(&tr->gops->ops, ops); 412 } 413 414 static int graph_trace_init(struct trace_array *tr) 415 { 416 int ret; 417 418 tr->gops->entryfunc = trace_graph_entry; 419 420 if (tracing_thresh) 421 tr->gops->retfunc = trace_graph_thresh_return; 422 else 423 tr->gops->retfunc = trace_graph_return; 424 425 /* Make gops functions are visible before we start tracing */ 426 smp_mb(); 427 428 ret = register_ftrace_graph(tr->gops); 429 if (ret) 430 return ret; 431 tracing_start_cmdline_record(); 432 433 return 0; 434 } 435 436 static void graph_trace_reset(struct trace_array *tr) 437 { 438 tracing_stop_cmdline_record(); 439 unregister_ftrace_graph(tr->gops); 440 } 441 442 static int graph_trace_update_thresh(struct trace_array *tr) 443 { 444 graph_trace_reset(tr); 445 return graph_trace_init(tr); 446 } 447 448 static int max_bytes_for_cpu; 449 450 static void print_graph_cpu(struct trace_seq *s, int cpu) 451 { 452 /* 453 * Start with a space character - to make it stand out 454 * to the right a bit when trace output is pasted into 455 * email: 456 */ 457 trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu); 458 } 459 460 #define TRACE_GRAPH_PROCINFO_LENGTH 14 461 462 static void print_graph_proc(struct trace_seq *s, pid_t pid) 463 { 464 char comm[TASK_COMM_LEN]; 465 /* sign + log10(MAX_INT) + '\0' */ 466 char pid_str[11]; 467 int spaces = 0; 468 int len; 469 int i; 470 471 trace_find_cmdline(pid, comm); 472 comm[7] = '\0'; 473 sprintf(pid_str, "%d", pid); 474 475 /* 1 stands for the "-" character */ 476 len = strlen(comm) + strlen(pid_str) + 1; 477 478 if (len < TRACE_GRAPH_PROCINFO_LENGTH) 479 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len; 480 481 /* First spaces to align center */ 482 for (i = 0; i < spaces / 2; i++) 483 trace_seq_putc(s, ' '); 484 485 trace_seq_printf(s, "%s-%s", comm, pid_str); 486 487 /* Last spaces to align center */ 488 for (i = 0; i < spaces - (spaces / 2); i++) 489 trace_seq_putc(s, ' '); 490 } 491 492 493 static void print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry) 494 { 495 trace_seq_putc(s, ' '); 496 trace_print_lat_fmt(s, entry); 497 trace_seq_puts(s, " | "); 498 } 499 500 /* If the pid changed since the last trace, output this event */ 501 static void 502 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data) 503 { 504 pid_t prev_pid; 505 pid_t *last_pid; 506 507 if (!data) 508 return; 509 510 last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 511 512 if (*last_pid == pid) 513 return; 514 515 prev_pid = *last_pid; 516 *last_pid = pid; 517 518 if (prev_pid == -1) 519 return; 520 /* 521 * Context-switch trace line: 522 523 ------------------------------------------ 524 | 1) migration/0--1 => sshd-1755 525 ------------------------------------------ 526 527 */ 528 trace_seq_puts(s, " ------------------------------------------\n"); 529 print_graph_cpu(s, cpu); 530 print_graph_proc(s, prev_pid); 531 trace_seq_puts(s, " => "); 532 print_graph_proc(s, pid); 533 trace_seq_puts(s, "\n ------------------------------------------\n\n"); 534 } 535 536 static struct ftrace_graph_ret_entry * 537 get_return_for_leaf(struct trace_iterator *iter, 538 struct ftrace_graph_ent_entry *curr) 539 { 540 struct fgraph_data *data = iter->private; 541 struct ring_buffer_iter *ring_iter = NULL; 542 struct ring_buffer_event *event; 543 struct ftrace_graph_ret_entry *next; 544 545 /* 546 * If the previous output failed to write to the seq buffer, 547 * then we just reuse the data from before. 548 */ 549 if (data && data->failed) { 550 curr = &data->ent.ent; 551 next = &data->ret; 552 } else { 553 554 ring_iter = trace_buffer_iter(iter, iter->cpu); 555 556 /* First peek to compare current entry and the next one */ 557 if (ring_iter) 558 event = ring_buffer_iter_peek(ring_iter, NULL); 559 else { 560 /* 561 * We need to consume the current entry to see 562 * the next one. 563 */ 564 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, 565 NULL, NULL); 566 event = ring_buffer_peek(iter->array_buffer->buffer, iter->cpu, 567 NULL, NULL); 568 } 569 570 if (!event) 571 return NULL; 572 573 next = ring_buffer_event_data(event); 574 575 if (data) { 576 /* 577 * Save current and next entries for later reference 578 * if the output fails. 579 */ 580 if (unlikely(curr->ent.type == TRACE_GRAPH_RETADDR_ENT)) 581 data->ent.rent = *(struct fgraph_retaddr_ent_entry *)curr; 582 else 583 data->ent.ent = *curr; 584 /* 585 * If the next event is not a return type, then 586 * we only care about what type it is. Otherwise we can 587 * safely copy the entire event. 588 */ 589 if (next->ent.type == TRACE_GRAPH_RET) 590 data->ret = *next; 591 else 592 data->ret.ent.type = next->ent.type; 593 } 594 } 595 596 if (next->ent.type != TRACE_GRAPH_RET) 597 return NULL; 598 599 if (curr->ent.pid != next->ent.pid || 600 curr->graph_ent.func != next->ret.func) 601 return NULL; 602 603 /* this is a leaf, now advance the iterator */ 604 if (ring_iter) 605 ring_buffer_iter_advance(ring_iter); 606 607 return next; 608 } 609 610 static void print_graph_abs_time(u64 t, struct trace_seq *s) 611 { 612 unsigned long usecs_rem; 613 614 usecs_rem = do_div(t, NSEC_PER_SEC); 615 usecs_rem /= 1000; 616 617 trace_seq_printf(s, "%5lu.%06lu | ", 618 (unsigned long)t, usecs_rem); 619 } 620 621 static void 622 print_graph_rel_time(struct trace_iterator *iter, struct trace_seq *s) 623 { 624 unsigned long long usecs; 625 626 usecs = iter->ts - iter->array_buffer->time_start; 627 do_div(usecs, NSEC_PER_USEC); 628 629 trace_seq_printf(s, "%9llu us | ", usecs); 630 } 631 632 static void 633 print_graph_irq(struct trace_iterator *iter, unsigned long addr, 634 enum trace_type type, int cpu, pid_t pid, u32 flags) 635 { 636 struct trace_array *tr = iter->tr; 637 struct trace_seq *s = &iter->seq; 638 struct trace_entry *ent = iter->ent; 639 640 addr += iter->tr->text_delta; 641 642 if (addr < (unsigned long)__irqentry_text_start || 643 addr >= (unsigned long)__irqentry_text_end) 644 return; 645 646 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 647 /* Absolute time */ 648 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 649 print_graph_abs_time(iter->ts, s); 650 651 /* Relative time */ 652 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 653 print_graph_rel_time(iter, s); 654 655 /* Cpu */ 656 if (flags & TRACE_GRAPH_PRINT_CPU) 657 print_graph_cpu(s, cpu); 658 659 /* Proc */ 660 if (flags & TRACE_GRAPH_PRINT_PROC) { 661 print_graph_proc(s, pid); 662 trace_seq_puts(s, " | "); 663 } 664 665 /* Latency format */ 666 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 667 print_graph_lat_fmt(s, ent); 668 } 669 670 /* No overhead */ 671 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_START); 672 673 if (type == TRACE_GRAPH_ENT) 674 trace_seq_puts(s, "==========>"); 675 else 676 trace_seq_puts(s, "<=========="); 677 678 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_END); 679 trace_seq_putc(s, '\n'); 680 } 681 682 void 683 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s) 684 { 685 unsigned long nsecs_rem = do_div(duration, 1000); 686 /* log10(ULONG_MAX) + '\0' */ 687 char usecs_str[21]; 688 char nsecs_str[5]; 689 int len; 690 int i; 691 692 sprintf(usecs_str, "%lu", (unsigned long) duration); 693 694 /* Print msecs */ 695 trace_seq_printf(s, "%s", usecs_str); 696 697 len = strlen(usecs_str); 698 699 /* Print nsecs (we don't want to exceed 7 numbers) */ 700 if (len < 7) { 701 size_t slen = min_t(size_t, sizeof(nsecs_str), 8UL - len); 702 703 snprintf(nsecs_str, slen, "%03lu", nsecs_rem); 704 trace_seq_printf(s, ".%s", nsecs_str); 705 len += strlen(nsecs_str) + 1; 706 } 707 708 trace_seq_puts(s, " us "); 709 710 /* Print remaining spaces to fit the row's width */ 711 for (i = len; i < 8; i++) 712 trace_seq_putc(s, ' '); 713 } 714 715 static void 716 print_graph_duration(struct trace_array *tr, unsigned long long duration, 717 struct trace_seq *s, u32 flags) 718 { 719 if (!(flags & TRACE_GRAPH_PRINT_DURATION) || 720 !(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 721 return; 722 723 /* No real adata, just filling the column with spaces */ 724 switch (flags & TRACE_GRAPH_PRINT_FILL_MASK) { 725 case FLAGS_FILL_FULL: 726 trace_seq_puts(s, " | "); 727 return; 728 case FLAGS_FILL_START: 729 trace_seq_puts(s, " "); 730 return; 731 case FLAGS_FILL_END: 732 trace_seq_puts(s, " |"); 733 return; 734 } 735 736 /* Signal a overhead of time execution to the output */ 737 if (flags & TRACE_GRAPH_PRINT_OVERHEAD) 738 trace_seq_printf(s, "%c ", trace_find_mark(duration)); 739 else 740 trace_seq_puts(s, " "); 741 742 trace_print_graph_duration(duration, s); 743 trace_seq_puts(s, "| "); 744 } 745 746 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 747 #define __TRACE_GRAPH_PRINT_RETVAL TRACE_GRAPH_PRINT_RETVAL 748 #else 749 #define __TRACE_GRAPH_PRINT_RETVAL 0 750 #endif 751 752 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 753 #define __TRACE_GRAPH_PRINT_RETADDR TRACE_GRAPH_PRINT_RETADDR 754 static void print_graph_retaddr(struct trace_seq *s, struct fgraph_retaddr_ent_entry *entry, 755 u32 trace_flags, bool comment) 756 { 757 if (comment) 758 trace_seq_puts(s, " /*"); 759 760 trace_seq_puts(s, " <-"); 761 seq_print_ip_sym(s, entry->graph_ent.retaddr, trace_flags | TRACE_ITER_SYM_OFFSET); 762 763 if (comment) 764 trace_seq_puts(s, " */"); 765 } 766 #else 767 #define __TRACE_GRAPH_PRINT_RETADDR 0 768 #define print_graph_retaddr(_seq, _entry, _tflags, _comment) do { } while (0) 769 #endif 770 771 #if defined(CONFIG_FUNCTION_GRAPH_RETVAL) || defined(CONFIG_FUNCTION_GRAPH_RETADDR) 772 773 static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entry *entry, 774 struct ftrace_graph_ret *graph_ret, void *func, 775 u32 opt_flags, u32 trace_flags) 776 { 777 unsigned long err_code = 0; 778 unsigned long retval = 0; 779 bool print_retaddr = false; 780 bool print_retval = false; 781 bool hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); 782 783 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 784 retval = graph_ret->retval; 785 print_retval = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL); 786 #endif 787 788 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 789 print_retaddr = !!(opt_flags & TRACE_GRAPH_PRINT_RETADDR); 790 #endif 791 792 if (print_retval && retval && !hex_format) { 793 /* Check if the return value matches the negative format */ 794 if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && 795 (((u64)retval) >> 32) == 0) { 796 err_code = sign_extend64(retval, 31); 797 } else { 798 err_code = retval; 799 } 800 801 if (!IS_ERR_VALUE(err_code)) 802 err_code = 0; 803 } 804 805 if (entry) { 806 if (entry->ent.type != TRACE_GRAPH_RETADDR_ENT) 807 print_retaddr = false; 808 809 trace_seq_printf(s, "%ps();", func); 810 if (print_retval || print_retaddr) 811 trace_seq_puts(s, " /*"); 812 else 813 trace_seq_putc(s, '\n'); 814 } else { 815 print_retaddr = false; 816 trace_seq_printf(s, "} /* %ps", func); 817 } 818 819 if (print_retaddr) 820 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 821 trace_flags, false); 822 823 if (print_retval) { 824 if (hex_format || (err_code == 0)) 825 trace_seq_printf(s, " ret=0x%lx", retval); 826 else 827 trace_seq_printf(s, " ret=%ld", err_code); 828 } 829 830 if (!entry || print_retval || print_retaddr) 831 trace_seq_puts(s, " */\n"); 832 } 833 834 #else 835 836 #define print_graph_retval(_seq, _ent, _ret, _func, _opt_flags, _trace_flags) do {} while (0) 837 838 #endif 839 840 /* Case of a leaf function on its call entry */ 841 static enum print_line_t 842 print_graph_entry_leaf(struct trace_iterator *iter, 843 struct ftrace_graph_ent_entry *entry, 844 struct ftrace_graph_ret_entry *ret_entry, 845 struct trace_seq *s, u32 flags) 846 { 847 struct fgraph_data *data = iter->private; 848 struct trace_array *tr = iter->tr; 849 struct ftrace_graph_ret *graph_ret; 850 struct ftrace_graph_ent *call; 851 unsigned long long duration; 852 unsigned long func; 853 int cpu = iter->cpu; 854 int i; 855 856 graph_ret = &ret_entry->ret; 857 call = &entry->graph_ent; 858 duration = graph_ret->rettime - graph_ret->calltime; 859 860 func = call->func + iter->tr->text_delta; 861 862 if (data) { 863 struct fgraph_cpu_data *cpu_data; 864 865 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 866 867 /* 868 * Comments display at + 1 to depth. Since 869 * this is a leaf function, keep the comments 870 * equal to this depth. 871 */ 872 cpu_data->depth = call->depth - 1; 873 874 /* No need to keep this function around for this depth */ 875 if (call->depth < FTRACE_RETFUNC_DEPTH && 876 !WARN_ON_ONCE(call->depth < 0)) 877 cpu_data->enter_funcs[call->depth] = 0; 878 } 879 880 /* Overhead and duration */ 881 print_graph_duration(tr, duration, s, flags); 882 883 /* Function */ 884 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 885 trace_seq_putc(s, ' '); 886 887 /* 888 * Write out the function return value or return address 889 */ 890 if (flags & (__TRACE_GRAPH_PRINT_RETVAL | __TRACE_GRAPH_PRINT_RETADDR)) { 891 print_graph_retval(s, entry, graph_ret, 892 (void *)graph_ret->func + iter->tr->text_delta, 893 flags, tr->trace_flags); 894 } else { 895 trace_seq_printf(s, "%ps();\n", (void *)func); 896 } 897 898 print_graph_irq(iter, graph_ret->func, TRACE_GRAPH_RET, 899 cpu, iter->ent->pid, flags); 900 901 return trace_handle_return(s); 902 } 903 904 static enum print_line_t 905 print_graph_entry_nested(struct trace_iterator *iter, 906 struct ftrace_graph_ent_entry *entry, 907 struct trace_seq *s, int cpu, u32 flags) 908 { 909 struct ftrace_graph_ent *call = &entry->graph_ent; 910 struct fgraph_data *data = iter->private; 911 struct trace_array *tr = iter->tr; 912 unsigned long func; 913 int i; 914 915 if (data) { 916 struct fgraph_cpu_data *cpu_data; 917 int cpu = iter->cpu; 918 919 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 920 cpu_data->depth = call->depth; 921 922 /* Save this function pointer to see if the exit matches */ 923 if (call->depth < FTRACE_RETFUNC_DEPTH && 924 !WARN_ON_ONCE(call->depth < 0)) 925 cpu_data->enter_funcs[call->depth] = call->func; 926 } 927 928 /* No time */ 929 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 930 931 /* Function */ 932 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 933 trace_seq_putc(s, ' '); 934 935 func = call->func + iter->tr->text_delta; 936 937 trace_seq_printf(s, "%ps() {", (void *)func); 938 if (flags & __TRACE_GRAPH_PRINT_RETADDR && 939 entry->ent.type == TRACE_GRAPH_RETADDR_ENT) 940 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 941 tr->trace_flags, true); 942 trace_seq_putc(s, '\n'); 943 944 if (trace_seq_has_overflowed(s)) 945 return TRACE_TYPE_PARTIAL_LINE; 946 947 /* 948 * we already consumed the current entry to check the next one 949 * and see if this is a leaf. 950 */ 951 return TRACE_TYPE_NO_CONSUME; 952 } 953 954 static void 955 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s, 956 int type, unsigned long addr, u32 flags) 957 { 958 struct fgraph_data *data = iter->private; 959 struct trace_entry *ent = iter->ent; 960 struct trace_array *tr = iter->tr; 961 int cpu = iter->cpu; 962 963 /* Pid */ 964 verif_pid(s, ent->pid, cpu, data); 965 966 if (type) 967 /* Interrupt */ 968 print_graph_irq(iter, addr, type, cpu, ent->pid, flags); 969 970 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 971 return; 972 973 /* Absolute time */ 974 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 975 print_graph_abs_time(iter->ts, s); 976 977 /* Relative time */ 978 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 979 print_graph_rel_time(iter, s); 980 981 /* Cpu */ 982 if (flags & TRACE_GRAPH_PRINT_CPU) 983 print_graph_cpu(s, cpu); 984 985 /* Proc */ 986 if (flags & TRACE_GRAPH_PRINT_PROC) { 987 print_graph_proc(s, ent->pid); 988 trace_seq_puts(s, " | "); 989 } 990 991 /* Latency format */ 992 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 993 print_graph_lat_fmt(s, ent); 994 995 return; 996 } 997 998 /* 999 * Entry check for irq code 1000 * 1001 * returns 1 if 1002 * - we are inside irq code 1003 * - we just entered irq code 1004 * 1005 * returns 0 if 1006 * - funcgraph-interrupts option is set 1007 * - we are not inside irq code 1008 */ 1009 static int 1010 check_irq_entry(struct trace_iterator *iter, u32 flags, 1011 unsigned long addr, int depth) 1012 { 1013 int cpu = iter->cpu; 1014 int *depth_irq; 1015 struct fgraph_data *data = iter->private; 1016 1017 addr += iter->tr->text_delta; 1018 1019 /* 1020 * If we are either displaying irqs, or we got called as 1021 * a graph event and private data does not exist, 1022 * then we bypass the irq check. 1023 */ 1024 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1025 (!data)) 1026 return 0; 1027 1028 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1029 1030 /* 1031 * We are inside the irq code 1032 */ 1033 if (*depth_irq >= 0) 1034 return 1; 1035 1036 if ((addr < (unsigned long)__irqentry_text_start) || 1037 (addr >= (unsigned long)__irqentry_text_end)) 1038 return 0; 1039 1040 /* 1041 * We are entering irq code. 1042 */ 1043 *depth_irq = depth; 1044 return 1; 1045 } 1046 1047 /* 1048 * Return check for irq code 1049 * 1050 * returns 1 if 1051 * - we are inside irq code 1052 * - we just left irq code 1053 * 1054 * returns 0 if 1055 * - funcgraph-interrupts option is set 1056 * - we are not inside irq code 1057 */ 1058 static int 1059 check_irq_return(struct trace_iterator *iter, u32 flags, int depth) 1060 { 1061 int cpu = iter->cpu; 1062 int *depth_irq; 1063 struct fgraph_data *data = iter->private; 1064 1065 /* 1066 * If we are either displaying irqs, or we got called as 1067 * a graph event and private data does not exist, 1068 * then we bypass the irq check. 1069 */ 1070 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1071 (!data)) 1072 return 0; 1073 1074 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1075 1076 /* 1077 * We are not inside the irq code. 1078 */ 1079 if (*depth_irq == -1) 1080 return 0; 1081 1082 /* 1083 * We are inside the irq code, and this is returning entry. 1084 * Let's not trace it and clear the entry depth, since 1085 * we are out of irq code. 1086 * 1087 * This condition ensures that we 'leave the irq code' once 1088 * we are out of the entry depth. Thus protecting us from 1089 * the RETURN entry loss. 1090 */ 1091 if (*depth_irq >= depth) { 1092 *depth_irq = -1; 1093 return 1; 1094 } 1095 1096 /* 1097 * We are inside the irq code, and this is not the entry. 1098 */ 1099 return 1; 1100 } 1101 1102 static enum print_line_t 1103 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 1104 struct trace_iterator *iter, u32 flags) 1105 { 1106 struct fgraph_data *data = iter->private; 1107 struct ftrace_graph_ent *call = &field->graph_ent; 1108 struct ftrace_graph_ret_entry *leaf_ret; 1109 static enum print_line_t ret; 1110 int cpu = iter->cpu; 1111 1112 if (check_irq_entry(iter, flags, call->func, call->depth)) 1113 return TRACE_TYPE_HANDLED; 1114 1115 print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags); 1116 1117 leaf_ret = get_return_for_leaf(iter, field); 1118 if (leaf_ret) 1119 ret = print_graph_entry_leaf(iter, field, leaf_ret, s, flags); 1120 else 1121 ret = print_graph_entry_nested(iter, field, s, cpu, flags); 1122 1123 if (data) { 1124 /* 1125 * If we failed to write our output, then we need to make 1126 * note of it. Because we already consumed our entry. 1127 */ 1128 if (s->full) { 1129 data->failed = 1; 1130 data->cpu = cpu; 1131 } else 1132 data->failed = 0; 1133 } 1134 1135 return ret; 1136 } 1137 1138 static enum print_line_t 1139 print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, 1140 struct trace_entry *ent, struct trace_iterator *iter, 1141 u32 flags) 1142 { 1143 unsigned long long duration = trace->rettime - trace->calltime; 1144 struct fgraph_data *data = iter->private; 1145 struct trace_array *tr = iter->tr; 1146 unsigned long func; 1147 pid_t pid = ent->pid; 1148 int cpu = iter->cpu; 1149 int func_match = 1; 1150 int i; 1151 1152 func = trace->func + iter->tr->text_delta; 1153 1154 if (check_irq_return(iter, flags, trace->depth)) 1155 return TRACE_TYPE_HANDLED; 1156 1157 if (data) { 1158 struct fgraph_cpu_data *cpu_data; 1159 int cpu = iter->cpu; 1160 1161 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 1162 1163 /* 1164 * Comments display at + 1 to depth. This is the 1165 * return from a function, we now want the comments 1166 * to display at the same level of the bracket. 1167 */ 1168 cpu_data->depth = trace->depth - 1; 1169 1170 if (trace->depth < FTRACE_RETFUNC_DEPTH && 1171 !WARN_ON_ONCE(trace->depth < 0)) { 1172 if (cpu_data->enter_funcs[trace->depth] != trace->func) 1173 func_match = 0; 1174 cpu_data->enter_funcs[trace->depth] = 0; 1175 } 1176 } 1177 1178 print_graph_prologue(iter, s, 0, 0, flags); 1179 1180 /* Overhead and duration */ 1181 print_graph_duration(tr, duration, s, flags); 1182 1183 /* Closing brace */ 1184 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) 1185 trace_seq_putc(s, ' '); 1186 1187 /* 1188 * Always write out the function name and its return value if the 1189 * funcgraph-retval option is enabled. 1190 */ 1191 if (flags & __TRACE_GRAPH_PRINT_RETVAL) { 1192 print_graph_retval(s, NULL, trace, (void *)func, flags, tr->trace_flags); 1193 } else { 1194 /* 1195 * If the return function does not have a matching entry, 1196 * then the entry was lost. Instead of just printing 1197 * the '}' and letting the user guess what function this 1198 * belongs to, write out the function name. Always do 1199 * that if the funcgraph-tail option is enabled. 1200 */ 1201 if (func_match && !(flags & TRACE_GRAPH_PRINT_TAIL)) 1202 trace_seq_puts(s, "}\n"); 1203 else 1204 trace_seq_printf(s, "} /* %ps */\n", (void *)func); 1205 } 1206 1207 /* Overrun */ 1208 if (flags & TRACE_GRAPH_PRINT_OVERRUN) 1209 trace_seq_printf(s, " (Overruns: %u)\n", 1210 trace->overrun); 1211 1212 print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, 1213 cpu, pid, flags); 1214 1215 return trace_handle_return(s); 1216 } 1217 1218 static enum print_line_t 1219 print_graph_comment(struct trace_seq *s, struct trace_entry *ent, 1220 struct trace_iterator *iter, u32 flags) 1221 { 1222 struct trace_array *tr = iter->tr; 1223 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 1224 struct fgraph_data *data = iter->private; 1225 struct trace_event *event; 1226 int depth = 0; 1227 int ret; 1228 int i; 1229 1230 if (data) 1231 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth; 1232 1233 print_graph_prologue(iter, s, 0, 0, flags); 1234 1235 /* No time */ 1236 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 1237 1238 /* Indentation */ 1239 if (depth > 0) 1240 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) 1241 trace_seq_putc(s, ' '); 1242 1243 /* The comment */ 1244 trace_seq_puts(s, "/* "); 1245 1246 switch (iter->ent->type) { 1247 case TRACE_BPUTS: 1248 ret = trace_print_bputs_msg_only(iter); 1249 if (ret != TRACE_TYPE_HANDLED) 1250 return ret; 1251 break; 1252 case TRACE_BPRINT: 1253 ret = trace_print_bprintk_msg_only(iter); 1254 if (ret != TRACE_TYPE_HANDLED) 1255 return ret; 1256 break; 1257 case TRACE_PRINT: 1258 ret = trace_print_printk_msg_only(iter); 1259 if (ret != TRACE_TYPE_HANDLED) 1260 return ret; 1261 break; 1262 default: 1263 event = ftrace_find_event(ent->type); 1264 if (!event) 1265 return TRACE_TYPE_UNHANDLED; 1266 1267 ret = event->funcs->trace(iter, sym_flags, event); 1268 if (ret != TRACE_TYPE_HANDLED) 1269 return ret; 1270 } 1271 1272 if (trace_seq_has_overflowed(s)) 1273 goto out; 1274 1275 /* Strip ending newline */ 1276 if (s->buffer[s->seq.len - 1] == '\n') { 1277 s->buffer[s->seq.len - 1] = '\0'; 1278 s->seq.len--; 1279 } 1280 1281 trace_seq_puts(s, " */\n"); 1282 out: 1283 return trace_handle_return(s); 1284 } 1285 1286 1287 enum print_line_t 1288 print_graph_function_flags(struct trace_iterator *iter, u32 flags) 1289 { 1290 struct ftrace_graph_ent_entry *field; 1291 struct fgraph_data *data = iter->private; 1292 struct trace_entry *entry = iter->ent; 1293 struct trace_seq *s = &iter->seq; 1294 int cpu = iter->cpu; 1295 int ret; 1296 1297 if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) { 1298 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0; 1299 return TRACE_TYPE_HANDLED; 1300 } 1301 1302 /* 1303 * If the last output failed, there's a possibility we need 1304 * to print out the missing entry which would never go out. 1305 */ 1306 if (data && data->failed) { 1307 field = &data->ent.ent; 1308 iter->cpu = data->cpu; 1309 ret = print_graph_entry(field, s, iter, flags); 1310 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { 1311 per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; 1312 ret = TRACE_TYPE_NO_CONSUME; 1313 } 1314 iter->cpu = cpu; 1315 return ret; 1316 } 1317 1318 switch (entry->type) { 1319 case TRACE_GRAPH_ENT: { 1320 /* 1321 * print_graph_entry() may consume the current event, 1322 * thus @field may become invalid, so we need to save it. 1323 * sizeof(struct ftrace_graph_ent_entry) is very small, 1324 * it can be safely saved at the stack. 1325 */ 1326 struct ftrace_graph_ent_entry saved; 1327 trace_assign_type(field, entry); 1328 saved = *field; 1329 return print_graph_entry(&saved, s, iter, flags); 1330 } 1331 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1332 case TRACE_GRAPH_RETADDR_ENT: { 1333 struct fgraph_retaddr_ent_entry saved; 1334 struct fgraph_retaddr_ent_entry *rfield; 1335 1336 trace_assign_type(rfield, entry); 1337 saved = *rfield; 1338 return print_graph_entry((struct ftrace_graph_ent_entry *)&saved, s, iter, flags); 1339 } 1340 #endif 1341 case TRACE_GRAPH_RET: { 1342 struct ftrace_graph_ret_entry *field; 1343 trace_assign_type(field, entry); 1344 return print_graph_return(&field->ret, s, entry, iter, flags); 1345 } 1346 case TRACE_STACK: 1347 case TRACE_FN: 1348 /* dont trace stack and functions as comments */ 1349 return TRACE_TYPE_UNHANDLED; 1350 1351 default: 1352 return print_graph_comment(s, entry, iter, flags); 1353 } 1354 1355 return TRACE_TYPE_HANDLED; 1356 } 1357 1358 static enum print_line_t 1359 print_graph_function(struct trace_iterator *iter) 1360 { 1361 return print_graph_function_flags(iter, tracer_flags.val); 1362 } 1363 1364 static enum print_line_t 1365 print_graph_function_event(struct trace_iterator *iter, int flags, 1366 struct trace_event *event) 1367 { 1368 return print_graph_function(iter); 1369 } 1370 1371 static void print_lat_header(struct seq_file *s, u32 flags) 1372 { 1373 static const char spaces[] = " " /* 16 spaces */ 1374 " " /* 4 spaces */ 1375 " "; /* 17 spaces */ 1376 int size = 0; 1377 1378 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1379 size += 16; 1380 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1381 size += 16; 1382 if (flags & TRACE_GRAPH_PRINT_CPU) 1383 size += 4; 1384 if (flags & TRACE_GRAPH_PRINT_PROC) 1385 size += 17; 1386 1387 seq_printf(s, "#%.*s _-----=> irqs-off \n", size, spaces); 1388 seq_printf(s, "#%.*s / _----=> need-resched \n", size, spaces); 1389 seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces); 1390 seq_printf(s, "#%.*s|| / _--=> preempt-depth \n", size, spaces); 1391 seq_printf(s, "#%.*s||| / \n", size, spaces); 1392 } 1393 1394 static void __print_graph_headers_flags(struct trace_array *tr, 1395 struct seq_file *s, u32 flags) 1396 { 1397 int lat = tr->trace_flags & TRACE_ITER_LATENCY_FMT; 1398 1399 if (lat) 1400 print_lat_header(s, flags); 1401 1402 /* 1st line */ 1403 seq_putc(s, '#'); 1404 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1405 seq_puts(s, " TIME "); 1406 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1407 seq_puts(s, " REL TIME "); 1408 if (flags & TRACE_GRAPH_PRINT_CPU) 1409 seq_puts(s, " CPU"); 1410 if (flags & TRACE_GRAPH_PRINT_PROC) 1411 seq_puts(s, " TASK/PID "); 1412 if (lat) 1413 seq_puts(s, "|||| "); 1414 if (flags & TRACE_GRAPH_PRINT_DURATION) 1415 seq_puts(s, " DURATION "); 1416 seq_puts(s, " FUNCTION CALLS\n"); 1417 1418 /* 2nd line */ 1419 seq_putc(s, '#'); 1420 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1421 seq_puts(s, " | "); 1422 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1423 seq_puts(s, " | "); 1424 if (flags & TRACE_GRAPH_PRINT_CPU) 1425 seq_puts(s, " | "); 1426 if (flags & TRACE_GRAPH_PRINT_PROC) 1427 seq_puts(s, " | | "); 1428 if (lat) 1429 seq_puts(s, "|||| "); 1430 if (flags & TRACE_GRAPH_PRINT_DURATION) 1431 seq_puts(s, " | | "); 1432 seq_puts(s, " | | | |\n"); 1433 } 1434 1435 static void print_graph_headers(struct seq_file *s) 1436 { 1437 print_graph_headers_flags(s, tracer_flags.val); 1438 } 1439 1440 void print_graph_headers_flags(struct seq_file *s, u32 flags) 1441 { 1442 struct trace_iterator *iter = s->private; 1443 struct trace_array *tr = iter->tr; 1444 1445 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 1446 return; 1447 1448 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) { 1449 /* print nothing if the buffers are empty */ 1450 if (trace_empty(iter)) 1451 return; 1452 1453 print_trace_header(s, iter); 1454 } 1455 1456 __print_graph_headers_flags(tr, s, flags); 1457 } 1458 1459 void graph_trace_open(struct trace_iterator *iter) 1460 { 1461 /* pid and depth on the last trace processed */ 1462 struct fgraph_data *data; 1463 gfp_t gfpflags; 1464 int cpu; 1465 1466 iter->private = NULL; 1467 1468 /* We can be called in atomic context via ftrace_dump() */ 1469 gfpflags = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL; 1470 1471 data = kzalloc(sizeof(*data), gfpflags); 1472 if (!data) 1473 goto out_err; 1474 1475 data->cpu_data = alloc_percpu_gfp(struct fgraph_cpu_data, gfpflags); 1476 if (!data->cpu_data) 1477 goto out_err_free; 1478 1479 for_each_possible_cpu(cpu) { 1480 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 1481 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); 1482 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore); 1483 int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1484 1485 *pid = -1; 1486 *depth = 0; 1487 *ignore = 0; 1488 *depth_irq = -1; 1489 } 1490 1491 iter->private = data; 1492 1493 return; 1494 1495 out_err_free: 1496 kfree(data); 1497 out_err: 1498 pr_warn("function graph tracer: not enough memory\n"); 1499 } 1500 1501 void graph_trace_close(struct trace_iterator *iter) 1502 { 1503 struct fgraph_data *data = iter->private; 1504 1505 if (data) { 1506 free_percpu(data->cpu_data); 1507 kfree(data); 1508 } 1509 } 1510 1511 static int 1512 func_graph_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1513 { 1514 if (bit == TRACE_GRAPH_PRINT_IRQS) 1515 ftrace_graph_skip_irqs = !set; 1516 1517 if (bit == TRACE_GRAPH_SLEEP_TIME) 1518 ftrace_graph_sleep_time_control(set); 1519 1520 if (bit == TRACE_GRAPH_GRAPH_TIME) 1521 ftrace_graph_graph_time_control(set); 1522 1523 return 0; 1524 } 1525 1526 static struct trace_event_functions graph_functions = { 1527 .trace = print_graph_function_event, 1528 }; 1529 1530 static struct trace_event graph_trace_entry_event = { 1531 .type = TRACE_GRAPH_ENT, 1532 .funcs = &graph_functions, 1533 }; 1534 1535 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1536 static struct trace_event graph_trace_retaddr_entry_event = { 1537 .type = TRACE_GRAPH_RETADDR_ENT, 1538 .funcs = &graph_functions, 1539 }; 1540 #endif 1541 1542 static struct trace_event graph_trace_ret_event = { 1543 .type = TRACE_GRAPH_RET, 1544 .funcs = &graph_functions 1545 }; 1546 1547 static struct tracer graph_trace __tracer_data = { 1548 .name = "function_graph", 1549 .update_thresh = graph_trace_update_thresh, 1550 .open = graph_trace_open, 1551 .pipe_open = graph_trace_open, 1552 .close = graph_trace_close, 1553 .pipe_close = graph_trace_close, 1554 .init = graph_trace_init, 1555 .reset = graph_trace_reset, 1556 .print_line = print_graph_function, 1557 .print_header = print_graph_headers, 1558 .flags = &tracer_flags, 1559 .set_flag = func_graph_set_flag, 1560 .allow_instances = true, 1561 #ifdef CONFIG_FTRACE_SELFTEST 1562 .selftest = trace_selftest_startup_function_graph, 1563 #endif 1564 }; 1565 1566 1567 static ssize_t 1568 graph_depth_write(struct file *filp, const char __user *ubuf, size_t cnt, 1569 loff_t *ppos) 1570 { 1571 unsigned long val; 1572 int ret; 1573 1574 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 1575 if (ret) 1576 return ret; 1577 1578 fgraph_max_depth = val; 1579 1580 *ppos += cnt; 1581 1582 return cnt; 1583 } 1584 1585 static ssize_t 1586 graph_depth_read(struct file *filp, char __user *ubuf, size_t cnt, 1587 loff_t *ppos) 1588 { 1589 char buf[15]; /* More than enough to hold UINT_MAX + "\n"*/ 1590 int n; 1591 1592 n = sprintf(buf, "%d\n", fgraph_max_depth); 1593 1594 return simple_read_from_buffer(ubuf, cnt, ppos, buf, n); 1595 } 1596 1597 static const struct file_operations graph_depth_fops = { 1598 .open = tracing_open_generic, 1599 .write = graph_depth_write, 1600 .read = graph_depth_read, 1601 .llseek = generic_file_llseek, 1602 }; 1603 1604 static __init int init_graph_tracefs(void) 1605 { 1606 int ret; 1607 1608 ret = tracing_init_dentry(); 1609 if (ret) 1610 return 0; 1611 1612 trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL, 1613 NULL, &graph_depth_fops); 1614 1615 return 0; 1616 } 1617 fs_initcall(init_graph_tracefs); 1618 1619 static __init int init_graph_trace(void) 1620 { 1621 max_bytes_for_cpu = snprintf(NULL, 0, "%u", nr_cpu_ids - 1); 1622 1623 if (!register_trace_event(&graph_trace_entry_event)) { 1624 pr_warn("Warning: could not register graph trace events\n"); 1625 return 1; 1626 } 1627 1628 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1629 if (!register_trace_event(&graph_trace_retaddr_entry_event)) { 1630 pr_warn("Warning: could not register graph trace retaddr events\n"); 1631 return 1; 1632 } 1633 #endif 1634 1635 if (!register_trace_event(&graph_trace_ret_event)) { 1636 pr_warn("Warning: could not register graph trace events\n"); 1637 return 1; 1638 } 1639 1640 return register_tracer(&graph_trace); 1641 } 1642 1643 core_initcall(init_graph_trace); 1644