460 	struct ftrace_profile_page *pg;  in function_stat_next()  local
462 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);  in function_stat_next()
468 	if ((void *)rec >= (void *)&pg->records[pg->index]) {  in function_stat_next()
469 		pg = pg->next;  in function_stat_next()
470 		if (!pg)  in function_stat_next()
472 		rec = &pg->records[0];  in function_stat_next()
593 	struct ftrace_profile_page *pg;  in ftrace_profile_reset()  local
595 	pg = stat->pages = stat->start;  in ftrace_profile_reset()
597 	while (pg) {  in ftrace_profile_reset()
598 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);  in ftrace_profile_reset()
599 		pg->index = 0;  in ftrace_profile_reset()
600 		pg = pg->next;  in ftrace_profile_reset()
609 	struct ftrace_profile_page *pg;  in ftrace_profile_pages_init()  local
635 	pg = stat->start = stat->pages;  in ftrace_profile_pages_init()
640 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);  in ftrace_profile_pages_init()
641 		if (!pg->next)  in ftrace_profile_pages_init()
643 		pg = pg->next;  in ftrace_profile_pages_init()
649 	pg = stat->start;  in ftrace_profile_pages_init()
650 	while (pg) {  in ftrace_profile_pages_init()
651 		unsigned long tmp = (unsigned long)pg;  in ftrace_profile_pages_init()
653 		pg = pg->next;  in ftrace_profile_pages_init()
1570 #define do_for_each_ftrace_rec(pg, rec)					\  argument
1571 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1573 		for (_____i = 0; _____i < pg->index; _____i++) {	\
1574 			rec = &pg->records[_____i];
1595 	struct ftrace_page *pg;  in lookup_rec()  local
1602 	for (pg = ftrace_pages_start; pg; pg = pg->next) {  in lookup_rec()
1603 		if (pg->index == 0 ||  in lookup_rec()
1604 		    end < pg->records[0].ip ||  in lookup_rec()
1605 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))  in lookup_rec()
1607 		rec = bsearch(&key, pg->records, pg->index,  in lookup_rec()
1743 	struct ftrace_page *pg;  in __ftrace_hash_rec_update()  local
1762 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_rec_update()
1980 	struct ftrace_page *pg;  in __ftrace_hash_update_ipmodify()  local
2008 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_update_ipmodify()
2054 	do_for_each_ftrace_rec(pg, rec) {  in __ftrace_hash_update_ipmodify()
2723 	struct ftrace_page *pg;  in ftrace_replace_code()  local
2731 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_replace_code()
2748 	struct ftrace_page	*pg;  member
2770 	iter->pg = ftrace_pages_start;  in ftrace_rec_iter_start()
2774 	while (iter->pg && !iter->pg->index)  in ftrace_rec_iter_start()
2775 		iter->pg = iter->pg->next;  in ftrace_rec_iter_start()
2777 	if (!iter->pg)  in ftrace_rec_iter_start()
2793 	if (iter->index >= iter->pg->index) {  in ftrace_rec_iter_next()
2794 		iter->pg = iter->pg->next;  in ftrace_rec_iter_next()
2798 		while (iter->pg && !iter->pg->index)  in ftrace_rec_iter_next()
2799 			iter->pg = iter->pg->next;  in ftrace_rec_iter_next()
2802 	if (!iter->pg)  in ftrace_rec_iter_next()
2816 	return &iter->pg->records[iter->index];  in ftrace_rec_iter_record()
3153 		struct ftrace_page *pg;  in ftrace_shutdown()  local
3156 		do_for_each_ftrace_rec(pg, rec) {  in ftrace_shutdown()
3746 	struct ftrace_page *pg;  in ftrace_update_code()  local
3769 	for (pg = new_pgs; pg; pg = pg->next) {  in ftrace_update_code()
3771 		for (i = 0; i < pg->index; i++) {  in ftrace_update_code()
3777 			p = &pg->records[i];  in ftrace_update_code()
3802 static int ftrace_allocate_records(struct ftrace_page *pg, int count)  in ftrace_allocate_records()  argument
3816 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);  in ftrace_allocate_records()
3818 	if (!pg->records) {  in ftrace_allocate_records()
3830 	pg->order = order;  in ftrace_allocate_records()
3840 	struct ftrace_page *pg = pages;  in ftrace_free_pages()  local
3842 	while (pg) {  in ftrace_free_pages()
3843 		if (pg->records) {  in ftrace_free_pages()
3844 			free_pages((unsigned long)pg->records, pg->order);  in ftrace_free_pages()
3845 			ftrace_number_of_pages -= 1 << pg->order;  in ftrace_free_pages()
3847 		pages = pg->next;  in ftrace_free_pages()
3848 		kfree(pg);  in ftrace_free_pages()
3849 		pg = pages;  in ftrace_free_pages()
3858 	struct ftrace_page *pg;  in ftrace_allocate_pages()  local
3864 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);  in ftrace_allocate_pages()
3865 	if (!pg)  in ftrace_allocate_pages()
3874 		cnt = ftrace_allocate_records(pg, num_to_init);  in ftrace_allocate_pages()
3882 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);  in ftrace_allocate_pages()
3883 		if (!pg->next)  in ftrace_allocate_pages()
3886 		pg = pg->next;  in ftrace_allocate_pages()
3903 	struct ftrace_page		*pg;  member
4133 	if (iter->idx >= iter->pg->index) {  in t_func_next()
4134 		if (iter->pg->next) {  in t_func_next()
4135 			iter->pg = iter->pg->next;  in t_func_next()
4140 		rec = &iter->pg->records[iter->idx++];  in t_func_next()
4242 	iter->pg = ftrace_pages_start;  in t_start()
4312 	struct ftrace_page *pg;  in ftrace_check_work_func()  local
4316 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_check_work_func()
4482 	iter->pg = ftrace_pages_start;  in ftrace_avail_open()
4506 	iter->pg = ftrace_pages_start;  in ftrace_enabled_open()
4531 	iter->pg = ftrace_pages_start;  in ftrace_touched_open()
4555 	iter->pg = ftrace_pages_start;  in ftrace_avail_addrs_open()
4641 		iter->pg = ftrace_pages_start;  in ftrace_regex_open()
4769 	struct ftrace_page *pg;  in add_rec_by_index()  local
4776 	do_for_each_ftrace_rec(pg, rec) {  in add_rec_by_index()
4777 		if (pg->index <= index) {  in add_rec_by_index()
4778 			index -= pg->index;  in add_rec_by_index()
4782 		rec = &pg->records[index];  in add_rec_by_index()
4853 	struct ftrace_page *pg;  in match_records()  local
4883 	do_for_each_ftrace_rec(pg, rec) {  in match_records()
6871 	struct ftrace_page *pg;  in ftrace_graph_set_hash()  local
6887 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_graph_set_hash()
7066 	struct ftrace_page *pg;  in ftrace_process_locs()  local
7124 	pg = start_pg;  in ftrace_process_locs()
7154 		end_offset = (pg->index+1) * sizeof(pg->records[0]);  in ftrace_process_locs()
7155 		if (end_offset > PAGE_SIZE << pg->order) {  in ftrace_process_locs()
7157 			if (WARN_ON(!pg->next))  in ftrace_process_locs()
7159 			pg = pg->next;  in ftrace_process_locs()
7162 		rec = &pg->records[pg->index++];  in ftrace_process_locs()
7166 	if (pg->next) {  in ftrace_process_locs()
7167 		pg_unuse = pg->next;  in ftrace_process_locs()
7168 		pg->next = NULL;  in ftrace_process_locs()
7172 	ftrace_pages = pg;  in ftrace_process_locs()
7197 		pg_remaining = (ENTRIES_PER_PAGE << pg->order) - pg->index;  in ftrace_process_locs()
7203 			for (pg = pg_unuse; pg; pg = pg->next)  in ftrace_process_locs()
7204 				remaining += 1 << pg->order;  in ftrace_process_locs()
7332 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)  in clear_mod_from_hash()  argument
7341 	for (i = 0; i < pg->index; i++) {  in clear_mod_from_hash()
7342 		rec = &pg->records[i];  in clear_mod_from_hash()
7355 static void clear_mod_from_hashes(struct ftrace_page *pg)  in clear_mod_from_hashes()  argument
7364 		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);  in clear_mod_from_hashes()
7365 		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);  in clear_mod_from_hashes()
7394 	struct ftrace_page *pg;  in ftrace_release_mod()  local
7414 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {  in ftrace_release_mod()
7415 		rec = &pg->records[0];  in ftrace_release_mod()
7421 			if (WARN_ON(pg == ftrace_pages_start))  in ftrace_release_mod()
7425 			if (pg == ftrace_pages)  in ftrace_release_mod()
7428 			ftrace_update_tot_cnt -= pg->index;  in ftrace_release_mod()
7429 			*last_pg = pg->next;  in ftrace_release_mod()
7431 			pg->next = tmp_page;  in ftrace_release_mod()
7432 			tmp_page = pg;  in ftrace_release_mod()
7434 			last_pg = &pg->next;  in ftrace_release_mod()
7442 	for (pg = tmp_page; pg; pg = tmp_page) {  in ftrace_release_mod()
7445 		clear_mod_from_hashes(pg);  in ftrace_release_mod()
7447 		if (pg->records) {  in ftrace_release_mod()
7448 			free_pages((unsigned long)pg->records, pg->order);  in ftrace_release_mod()
7449 			ftrace_number_of_pages -= 1 << pg->order;  in ftrace_release_mod()
7451 		tmp_page = pg->next;  in ftrace_release_mod()
7452 		kfree(pg);  in ftrace_release_mod()
7460 	struct ftrace_page *pg;  in ftrace_module_enable()  local
7483 	do_for_each_ftrace_rec(pg, rec) {  in ftrace_module_enable()
7773 	struct ftrace_page *pg;  in ftrace_free_mem()  local
7793 	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {  in ftrace_free_mem()
7794 		if (end < pg->records[0].ip ||  in ftrace_free_mem()
7795 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))  in ftrace_free_mem()
7798 		rec = bsearch(&key, pg->records, pg->index,  in ftrace_free_mem()
7810 		pg->index--;  in ftrace_free_mem()
7812 		if (!pg->index) {  in ftrace_free_mem()
7813 			*last_pg = pg->next;  in ftrace_free_mem()
7814 			pg->next = tmp_page;  in ftrace_free_mem()
7815 			tmp_page = pg;  in ftrace_free_mem()
7816 			pg = container_of(last_pg, struct ftrace_page, next);  in ftrace_free_mem()
7818 				ftrace_pages = pg;  in ftrace_free_mem()
7822 			(pg->index - (rec - pg->records)) * sizeof(*rec));  in ftrace_free_mem()