386 static void rb_init_page(struct buffer_data_page *bpage)  in rb_init_page()  argument
388 	local_set(&bpage->commit, 0);  in rb_init_page()
391 static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage)  in rb_page_commit()  argument
393 	return local_read(&bpage->page->commit);  in rb_page_commit()
396 static void free_buffer_page(struct buffer_page *bpage)  in free_buffer_page()  argument
399 	if (!bpage->range)  in free_buffer_page()
400 		free_pages((unsigned long)bpage->page, bpage->order);  in free_buffer_page()
401 	kfree(bpage);  in free_buffer_page()
1354 static inline void rb_inc_page(struct buffer_page **bpage)  in rb_inc_page()  argument
1356 	struct list_head *p = rb_list_head((*bpage)->list.next);  in rb_inc_page()
1358 	*bpage = list_entry(p, struct buffer_page, list);  in rb_inc_page()
1476 			  struct buffer_page *bpage)  in rb_check_bpage()  argument
1478 	unsigned long val = (unsigned long)bpage;  in rb_check_bpage()
2089 				  struct buffer_page *bpage)  in rb_meta_buffer_update()  argument
2093 	if (meta->head_buffer == (unsigned long)bpage->page)  in rb_meta_buffer_update()
2094 		cpu_buffer->head_page = bpage;  in rb_meta_buffer_update()
2096 	if (meta->commit_buffer == (unsigned long)bpage->page) {  in rb_meta_buffer_update()
2097 		cpu_buffer->commit_page = bpage;  in rb_meta_buffer_update()
2098 		cpu_buffer->tail_page = bpage;  in rb_meta_buffer_update()
2107 	struct buffer_page *bpage, *tmp;  in __rb_allocate_pages()  local
2148 		bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),  in __rb_allocate_pages()
2150 		if (!bpage)  in __rb_allocate_pages()
2153 		rb_check_bpage(cpu_buffer, bpage);  in __rb_allocate_pages()
2159 		list_add_tail(&bpage->list, pages);  in __rb_allocate_pages()
2163 			bpage->page = rb_range_buffer(cpu_buffer, i + 1);  in __rb_allocate_pages()
2164 			if (!bpage->page)  in __rb_allocate_pages()
2168 				rb_meta_buffer_update(cpu_buffer, bpage);  in __rb_allocate_pages()
2169 			bpage->range = 1;  in __rb_allocate_pages()
2170 			bpage->id = i + 1;  in __rb_allocate_pages()
2177 			bpage->page = page_address(page);  in __rb_allocate_pages()
2178 			rb_init_page(bpage->page);  in __rb_allocate_pages()
2180 		bpage->order = cpu_buffer->buffer->subbuf_order;  in __rb_allocate_pages()
2191 	list_for_each_entry_safe(bpage, tmp, pages, list) {  in __rb_allocate_pages()
2192 		list_del_init(&bpage->list);  in __rb_allocate_pages()
2193 		free_buffer_page(bpage);  in __rb_allocate_pages()
2231 	struct buffer_page *bpage;  in rb_allocate_cpu_buffer()  local
2252 	bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),  in rb_allocate_cpu_buffer()
2254 	if (!bpage)  in rb_allocate_cpu_buffer()
2257 	rb_check_bpage(cpu_buffer, bpage);  in rb_allocate_cpu_buffer()
2259 	cpu_buffer->reader_page = bpage;  in rb_allocate_cpu_buffer()
2268 		bpage->page = rb_range_buffer(cpu_buffer, 0);  in rb_allocate_cpu_buffer()
2269 		if (!bpage->page)  in rb_allocate_cpu_buffer()
2272 			rb_meta_buffer_update(cpu_buffer, bpage);  in rb_allocate_cpu_buffer()
2273 		bpage->range = 1;  in rb_allocate_cpu_buffer()
2280 		bpage->page = page_address(page);  in rb_allocate_cpu_buffer()
2281 		rb_init_page(bpage->page);  in rb_allocate_cpu_buffer()
2334 	struct buffer_page *bpage, *tmp;  in rb_free_cpu_buffer()  local
2343 		list_for_each_entry_safe(bpage, tmp, head, list) {  in rb_free_cpu_buffer()
2344 			list_del_init(&bpage->list);  in rb_free_cpu_buffer()
2345 			free_buffer_page(bpage);  in rb_free_cpu_buffer()
2347 		bpage = list_entry(head, struct buffer_page, list);  in rb_free_cpu_buffer()
2348 		free_buffer_page(bpage);  in rb_free_cpu_buffer()
2594 static inline unsigned long rb_page_entries(struct buffer_page *bpage)  in rb_page_entries()  argument
2596 	return local_read(&bpage->entries) & RB_WRITE_MASK;  in rb_page_entries()
2599 static inline unsigned long rb_page_write(struct buffer_page *bpage)  in rb_page_write()  argument
2601 	return local_read(&bpage->write) & RB_WRITE_MASK;  in rb_page_write()
2786 		struct buffer_page *bpage, *tmp;  in rb_insert_pages()  local
2787 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,  in rb_insert_pages()
2789 			list_del_init(&bpage->list);  in rb_insert_pages()
2790 			free_buffer_page(bpage);  in rb_insert_pages()
3017 		struct buffer_page *bpage, *tmp;  in ring_buffer_resize()  local
3025 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,  in ring_buffer_resize()
3027 			list_del_init(&bpage->list);  in ring_buffer_resize()
3028 			free_buffer_page(bpage);  in ring_buffer_resize()
3049 static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)  in __rb_page_index()  argument
3051 	return bpage->page->data + index;  in __rb_page_index()
3121 static __always_inline unsigned rb_page_size(struct buffer_page *bpage)  in rb_page_size()  argument
3123 	return rb_page_commit(bpage) & ~RB_MISSED_MASK;  in rb_page_size()
3724 	struct buffer_page *bpage;  in rb_try_to_discard()  local
3732 	bpage = READ_ONCE(cpu_buffer->tail_page);  in rb_try_to_discard()
3738 	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {  in rb_try_to_discard()
3740 			local_read(&bpage->write) & ~RB_WRITE_MASK;  in rb_try_to_discard()
3774 		if (local_try_cmpxchg(&bpage->write, &old_index, new_index)) {  in rb_try_to_discard()
4191 static void dump_buffer_page(struct buffer_data_page *bpage,  in dump_buffer_page()  argument
4199 	ts = bpage->time_stamp;  in dump_buffer_page()
4204 		event = (struct ring_buffer_event *)(bpage->data + e);  in dump_buffer_page()
4254 		dump_buffer_page(bpage, info, tail);			\
4270 	struct buffer_data_page *bpage;  in check_buffer()  local
4275 	bpage = info->tail_page->page;  in check_buffer()
4279 		tail = local_read(&bpage->commit);  in check_buffer()
4290 	if (tail <= 8 || tail > local_read(&bpage->commit))  in check_buffer()
4299 	ret = rb_read_data_buffer(bpage, tail, cpu_buffer->cpu, &ts, &delta);  in check_buffer()
4619 	struct buffer_page *bpage = cpu_buffer->commit_page;  in rb_decrement_entry()  local
4625 	if (likely(bpage->page == (void *)addr)) {  in rb_decrement_entry()
4626 		local_dec(&bpage->entries);  in rb_decrement_entry()
4634 	rb_inc_page(&bpage);  in rb_decrement_entry()
4635 	start = bpage;  in rb_decrement_entry()
4637 		if (bpage->page == (void *)addr) {  in rb_decrement_entry()
4638 			local_dec(&bpage->entries);  in rb_decrement_entry()
4641 		rb_inc_page(&bpage);  in rb_decrement_entry()
4642 	} while (bpage != start);  in rb_decrement_entry()
4938 	struct buffer_page *bpage;  in ring_buffer_oldest_event_ts()  local
4951 		bpage = cpu_buffer->reader_page;  in ring_buffer_oldest_event_ts()
4953 		bpage = rb_set_head_page(cpu_buffer);  in ring_buffer_oldest_event_ts()
4954 	if (bpage)  in ring_buffer_oldest_event_ts()
4955 		ret = bpage->page->time_stamp;  in ring_buffer_oldest_event_ts()
6378 	struct buffer_data_read_page *bpage = NULL;  in ring_buffer_alloc_read_page()  local
6385 	bpage = kzalloc(sizeof(*bpage), GFP_KERNEL);  in ring_buffer_alloc_read_page()
6386 	if (!bpage)  in ring_buffer_alloc_read_page()
6389 	bpage->order = buffer->subbuf_order;  in ring_buffer_alloc_read_page()
6395 		bpage->data = cpu_buffer->free_page;  in ring_buffer_alloc_read_page()
6402 	if (bpage->data)  in ring_buffer_alloc_read_page()
6409 		kfree(bpage);  in ring_buffer_alloc_read_page()
6413 	bpage->data = page_address(page);  in ring_buffer_alloc_read_page()
6416 	rb_init_page(bpage->data);  in ring_buffer_alloc_read_page()
6418 	return bpage;  in ring_buffer_alloc_read_page()
6434 	struct buffer_data_page *bpage = data_page->data;  in ring_buffer_free_read_page()  local
6435 	struct page *page = virt_to_page(bpage);  in ring_buffer_free_read_page()
6455 		cpu_buffer->free_page = bpage;  in ring_buffer_free_read_page()
6456 		bpage = NULL;  in ring_buffer_free_read_page()
6463 	free_pages((unsigned long)bpage, data_page->order);  in ring_buffer_free_read_page()
6508 	struct buffer_data_page *bpage;  in ring_buffer_read_page()  local
6534 	bpage = data_page->data;  in ring_buffer_read_page()
6535 	if (!bpage)  in ring_buffer_read_page()
6599 			memcpy(bpage->data + pos, rpage->data + rpos, size);  in ring_buffer_read_page()
6616 		local_set(&bpage->commit, pos);  in ring_buffer_read_page()
6617 		bpage->time_stamp = save_timestamp;  in ring_buffer_read_page()
6627 		rb_init_page(bpage);  in ring_buffer_read_page()
6628 		bpage = reader->page;  in ring_buffer_read_page()
6633 		data_page->data = bpage;  in ring_buffer_read_page()
6641 			local_set(&bpage->commit, reader->real_end);  in ring_buffer_read_page()
6647 	commit = local_read(&bpage->commit);  in ring_buffer_read_page()
6656 			memcpy(&bpage->data[commit], &missed_events,  in ring_buffer_read_page()
6658 			local_add(RB_MISSED_STORED, &bpage->commit);  in ring_buffer_read_page()
6661 		local_add(RB_MISSED_EVENTS, &bpage->commit);  in ring_buffer_read_page()
6668 		memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);  in ring_buffer_read_page()
6742 	struct buffer_page *bpage, *tmp;  in ring_buffer_subbuf_order_set()  local
6862 		list_for_each_entry_safe(bpage, tmp, &old_pages, list) {  in ring_buffer_subbuf_order_set()
6863 			list_del_init(&bpage->list);  in ring_buffer_subbuf_order_set()
6864 			free_buffer_page(bpage);  in ring_buffer_subbuf_order_set()
6889 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {  in ring_buffer_subbuf_order_set()
6890 			list_del_init(&bpage->list);  in ring_buffer_subbuf_order_set()
6891 			free_buffer_page(bpage);  in ring_buffer_subbuf_order_set()
7289 			struct buffer_data_page *bpage = reader->page;  in ring_buffer_map_get_reader()  local
7297 				local_set(&bpage->commit, reader->real_end);  in ring_buffer_map_get_reader()
7304 				memcpy(&bpage->data[commit], &missed_events,  in ring_buffer_map_get_reader()
7306 				local_add(RB_MISSED_STORED, &bpage->commit);  in ring_buffer_map_get_reader()
7308 			local_add(RB_MISSED_EVENTS, &bpage->commit);  in ring_buffer_map_get_reader()