Lines Matching refs:page_pool
85 bool page_pool_get_stats(const struct page_pool *pool, in page_pool_get_stats()
161 static bool page_pool_producer_lock(struct page_pool *pool) in page_pool_producer_lock()
174 static void page_pool_producer_unlock(struct page_pool *pool, in page_pool_producer_unlock()
186 CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users); in page_pool_struct_check()
187 CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page); in page_pool_struct_check()
188 CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset); in page_pool_struct_check()
189 CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag, in page_pool_struct_check()
193 static int page_pool_init(struct page_pool *pool, in page_pool_init()
320 static void page_pool_uninit(struct page_pool *pool) in page_pool_uninit()
338 struct page_pool *
341 struct page_pool *pool; in page_pool_create_percpu()
371 struct page_pool *page_pool_create(const struct page_pool_params *params) in page_pool_create()
377 static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem);
379 static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) in page_pool_refill_alloc_cache()
432 static netmem_ref __page_pool_get_cached(struct page_pool *pool) in __page_pool_get_cached()
448 static void __page_pool_dma_sync_for_device(const struct page_pool *pool, in __page_pool_dma_sync_for_device()
462 page_pool_dma_sync_for_device(const struct page_pool *pool, in page_pool_dma_sync_for_device()
470 static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) in page_pool_dma_map()
501 static struct page *__page_pool_alloc_page_order(struct page_pool *pool, in __page_pool_alloc_page_order()
527 static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, in __page_pool_alloc_pages_slow()
585 netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_netmems()
604 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_pages()
615 s32 page_pool_inflight(const struct page_pool *pool, bool strict) in page_pool_inflight()
634 void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) in page_pool_set_pp_info()
656 static __always_inline void __page_pool_release_page_dma(struct page_pool *pool, in __page_pool_release_page_dma()
681 void page_pool_return_page(struct page_pool *pool, netmem_ref netmem) in page_pool_return_page()
708 static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem) in page_pool_recycle_in_ring()
731 struct page_pool *pool) in page_pool_recycle_in_cache()
758 __page_pool_put_page(struct page_pool *pool, netmem_ref netmem, in __page_pool_put_page()
803 static bool page_pool_napi_local(const struct page_pool *pool) in page_pool_napi_local()
826 void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, in page_pool_put_unrefed_netmem()
842 void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, in page_pool_put_unrefed_page()
850 static void page_pool_recycle_ring_bulk(struct page_pool *pool, in page_pool_recycle_ring_bulk()
911 struct page_pool *pool = NULL; in page_pool_put_netmem_bulk()
918 struct page_pool *netmem_pp; in page_pool_put_netmem_bulk()
949 static netmem_ref page_pool_drain_frag(struct page_pool *pool, in page_pool_drain_frag()
967 static void page_pool_free_frag(struct page_pool *pool) in page_pool_free_frag()
980 netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, in page_pool_alloc_frag_netmem()
1025 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, in page_pool_alloc_frag()
1033 static void page_pool_empty_ring(struct page_pool *pool) in page_pool_empty_ring()
1048 static void __page_pool_destroy(struct page_pool *pool) in __page_pool_destroy()
1064 static void page_pool_empty_alloc_cache_once(struct page_pool *pool) in page_pool_empty_alloc_cache_once()
1081 static void page_pool_scrub(struct page_pool *pool) in page_pool_scrub()
1092 static int page_pool_release(struct page_pool *pool) in page_pool_release()
1107 struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw); in page_pool_release_retry()
1136 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), in page_pool_use_xdp_mem()
1144 void page_pool_disable_direct_recycling(struct page_pool *pool) in page_pool_disable_direct_recycling()
1162 void page_pool_destroy(struct page_pool *pool) in page_pool_destroy()
1186 void page_pool_update_nid(struct page_pool *pool, int new_nid) in page_pool_update_nid()
1209 void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov) in net_mp_niov_set_page_pool()