163 #define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset]))
169 #define SHADOW_STACK_TASK_VARS(ret_stack) \  argument
170 	((unsigned long *)(&(ret_stack)[SHADOW_STACK_OFFSET - FGRAPH_ARRAY_SIZE]))
251 	return t->ret_stack[offset];  in get_fgraph_entry()
257 	return __get_offset(t->ret_stack[offset]);  in get_frame_offset()
264 	return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK;  in get_bitmap_bits()
271 	t->ret_stack[offset] = (bitmap << FGRAPH_INDEX_SHIFT) |  in set_bitmap()
278 	unsigned long val = t->ret_stack[offset];  in get_data_type_data()
283 	return (void *)&t->ret_stack[offset];  in get_data_type_data()
309 	unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);  in ret_stack_set_task_var()
317 	unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);  in ret_stack_get_task_var()
322 static void ret_stack_init_task_vars(unsigned long *ret_stack)  in ret_stack_init_task_vars()  argument
324 	unsigned long *gvals = SHADOW_STACK_TASK_VARS(ret_stack);  in ret_stack_init_task_vars()
360 	data = &current->ret_stack[curr_ret_stack];  in fgraph_reserve_data()
369 	current->ret_stack[curr_ret_stack - 1] = val;  in fgraph_reserve_data()
375 	current->ret_stack[curr_ret_stack - 1] = val;  in fgraph_reserve_data()
466 	struct ftrace_ret_stack *ret_stack = NULL;  in fgraph_retrieve_parent_data()  local
476 		ret_stack = get_ret_stack(current, offset, &next_offset);  in fgraph_retrieve_parent_data()
477 		if (!ret_stack || --depth < 0)  in fgraph_retrieve_parent_data()
482 	if (!ret_stack)  in fgraph_retrieve_parent_data()
566 	struct ftrace_ret_stack *ret_stack;  in ftrace_push_return_trace()  local
573 	if (!current->ret_stack)  in ftrace_push_return_trace()
597 	ret_stack = RET_STACK(current, offset);  in ftrace_push_return_trace()
601 	current->ret_stack[offset] = val;  in ftrace_push_return_trace()
602 	ret_stack->ret = ret;  in ftrace_push_return_trace()
625 	current->ret_stack[offset] = val;  in ftrace_push_return_trace()
627 	ret_stack->ret = ret;  in ftrace_push_return_trace()
628 	ret_stack->func = func;  in ftrace_push_return_trace()
630 	ret_stack->fp = frame_pointer;  in ftrace_push_return_trace()
632 	ret_stack->retp = retp;  in ftrace_push_return_trace()
725 	struct ftrace_ret_stack *ret_stack;  in ftrace_pop_return_trace()  local
727 	ret_stack = get_ret_stack(current, current->curr_ret_stack, offset);  in ftrace_pop_return_trace()
729 	if (unlikely(!ret_stack)) {  in ftrace_pop_return_trace()
753 	if (unlikely(ret_stack->fp != frame_pointer)) {  in ftrace_pop_return_trace()
757 		     ret_stack->fp,  in ftrace_pop_return_trace()
759 		     (void *)ret_stack->func,  in ftrace_pop_return_trace()
760 		     ret_stack->ret);  in ftrace_pop_return_trace()
767 	*ret = ret_stack->ret;  in ftrace_pop_return_trace()
768 	trace->func = ret_stack->func;  in ftrace_pop_return_trace()
778 	return ret_stack;  in ftrace_pop_return_trace()
813 	struct ftrace_ret_stack *ret_stack;  in __ftrace_return_to_handler()  local
820 	ret_stack = ftrace_pop_return_trace(&trace, &ret, frame_pointer, &offset);  in __ftrace_return_to_handler()
822 	if (unlikely(!ret_stack)) {  in __ftrace_return_to_handler()
898 	struct ftrace_ret_stack *ret_stack = NULL;  in ftrace_graph_get_ret_stack()  local
905 		ret_stack = get_ret_stack(task, offset, &offset);  in ftrace_graph_get_ret_stack()
906 	} while (ret_stack && --idx >= 0);  in ftrace_graph_get_ret_stack()
908 	return ret_stack;  in ftrace_graph_get_ret_stack()
921 	struct ftrace_ret_stack *ret_stack = NULL;  in ftrace_graph_top_ret_addr()  local
928 		ret_stack = get_ret_stack(task, offset, &offset);  in ftrace_graph_top_ret_addr()
929 	} while (ret_stack && ret_stack->ret == return_handler);  in ftrace_graph_top_ret_addr()
931 	return ret_stack ? ret_stack->ret : 0;  in ftrace_graph_top_ret_addr()
957 	struct ftrace_ret_stack *ret_stack;  in ftrace_graph_ret_addr()  local
969 		ret_stack = get_ret_stack(task, i, &i);  in ftrace_graph_ret_addr()
970 		if (!ret_stack)  in ftrace_graph_ret_addr()
980 		if (ret_stack->retp == retp &&  in ftrace_graph_ret_addr()
981 		    ret_stack->ret != return_handler) {  in ftrace_graph_ret_addr()
983 			return ret_stack->ret;  in ftrace_graph_ret_addr()
1058 		if (t->ret_stack == NULL) {  in alloc_retstack_tasklist()
1065 			t->ret_stack = ret_stack_list[start++];  in alloc_retstack_tasklist()
1106 graph_init_task(struct task_struct *t, unsigned long *ret_stack)  in graph_init_task()  argument
1109 	ret_stack_init_task_vars(ret_stack);  in graph_init_task()
1115 	t->ret_stack = ret_stack;  in graph_init_task()
1130 	if (t->ret_stack)  in ftrace_graph_init_idle_task()
1131 		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));  in ftrace_graph_init_idle_task()
1134 		unsigned long *ret_stack;  in ftrace_graph_init_idle_task()  local
1139 		ret_stack = per_cpu(idle_ret_stack, cpu);  in ftrace_graph_init_idle_task()
1140 		if (!ret_stack) {  in ftrace_graph_init_idle_task()
1141 			ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);  in ftrace_graph_init_idle_task()
1142 			if (!ret_stack)  in ftrace_graph_init_idle_task()
1144 			per_cpu(idle_ret_stack, cpu) = ret_stack;  in ftrace_graph_init_idle_task()
1146 		graph_init_task(t, ret_stack);  in ftrace_graph_init_idle_task()
1154 	t->ret_stack = NULL;  in ftrace_graph_init_task()
1159 		unsigned long *ret_stack;  in ftrace_graph_init_task()  local
1164 		ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);  in ftrace_graph_init_task()
1165 		if (!ret_stack)  in ftrace_graph_init_task()
1167 		graph_init_task(t, ret_stack);  in ftrace_graph_init_task()
1173 	unsigned long *ret_stack = t->ret_stack;  in ftrace_graph_exit_task()  local
1175 	t->ret_stack = NULL;  in ftrace_graph_exit_task()
1179 	if (ret_stack) {  in ftrace_graph_exit_task()
1182 		kmem_cache_free(fgraph_stack_cachep, ret_stack);  in ftrace_graph_exit_task()
1240 		if (!idle_task(cpu)->ret_stack)  in start_graph_tracing()
1265 		if (idle_task(cpu)->ret_stack)  in init_task_vars()
1271 		if (t->ret_stack)  in init_task_vars()
1316 	if (!idle_task(cpu)->ret_stack)  in fgraph_cpu_init()