Home
last modified time | relevance | path

Searched refs:umem (Results 1 – 25 of 105) sorted by relevance

12345

/linux-6.15/net/xdp/
H A Dxdp_umem.c26 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true); in xdp_umem_unpin_pages()
29 umem->pgs = NULL; in xdp_umem_unpin_pages()
34 if (umem->user) { in xdp_umem_unaccount_pages()
35 atomic_long_sub(umem->npgs, &umem->user->locked_vm); in xdp_umem_unaccount_pages()
64 kfree(umem); in xdp_umem_release()
81 if (!umem) in xdp_put_umem()
100 umem->pgs = kvcalloc(umem->npgs, sizeof(*umem->pgs), GFP_KERNEL | __GFP_NOWARN); in xdp_umem_pin_pages()
234 err = xdp_umem_addr_map(umem, umem->pgs, umem->npgs); in xdp_umem_reg()
252 umem = kzalloc(sizeof(*umem), GFP_KERNEL); in xdp_umem_create()
253 if (!umem) in xdp_umem_create()
[all …]
H A Dxsk_buff_pool.c58 struct xdp_umem *umem) in xp_create_and_assign_umem() argument
79 pool->addrs_cnt = umem->size; in xp_create_and_assign_umem()
80 pool->heads_cnt = umem->chunks; in xp_create_and_assign_umem()
82 pool->headroom = umem->headroom; in xp_create_and_assign_umem()
86 pool->frame_len = umem->chunk_size - umem->headroom - in xp_create_and_assign_umem()
88 pool->umem = umem; in xp_create_and_assign_umem()
89 pool->addrs = umem->addrs; in xp_create_and_assign_umem()
106 xskb->xdp.frame_sz = umem->chunk_size - umem->headroom; in xp_create_and_assign_umem()
149 if (pool->umem->zc) { in xp_disable_drv_zc()
235 pool->umem->zc = true; in xp_assign_dev()
[all …]
H A Dxsk_diag.c50 struct xdp_umem *umem = xs->umem; in xsk_diag_put_umem() local
54 if (!umem) in xsk_diag_put_umem()
57 du.id = umem->id; in xsk_diag_put_umem()
58 du.size = umem->size; in xsk_diag_put_umem()
59 du.num_pages = umem->npgs; in xsk_diag_put_umem()
60 du.chunk_size = umem->chunk_size; in xsk_diag_put_umem()
61 du.headroom = umem->headroom; in xsk_diag_put_umem()
65 if (umem->zc) in xsk_diag_put_umem()
67 du.refs = refcount_read(&umem->users); in xsk_diag_put_umem()
/linux-6.15/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/
H A Dumem.c46 list_for_each_entry(umem, &master->umem, head) { in nvkm_umem_search()
67 if (!umem->map) in nvkm_umem_unmap()
70 if (umem->io) { in nvkm_umem_unmap()
75 umem->bar = NULL; in nvkm_umem_unmap()
79 umem->map = NULL; in nvkm_umem_unmap()
94 if (umem->map) in nvkm_umem_map()
98 int ret = nvkm_mem_map_host(umem->memory, &umem->map); in nvkm_umem_map()
131 return umem; in nvkm_umem_dtor()
164 if (!(umem = kzalloc(sizeof(*umem), GFP_KERNEL))) in nvkm_umem_new()
167 umem->mmu = mmu; in nvkm_umem_new()
[all …]
/linux-6.15/drivers/infiniband/core/
H A Dumem.c93 if (umem->is_odp) { in ib_umem_find_best_pgsz()
186 umem = kzalloc(sizeof(*umem), GFP_KERNEL); in ib_umem_get()
187 if (!umem) in ib_umem_get()
196 umem->iova = addr; in ib_umem_get()
268 kfree(umem); in ib_umem_get()
280 if (!umem) in ib_umem_release()
284 if (umem->is_odp) in ib_umem_release()
287 __ib_umem_release(umem->ibdev, umem, 1); in ib_umem_release()
289 atomic64_sub(ib_umem_num_pages(umem), &umem->owning_mm->pinned_vm); in ib_umem_release()
291 kfree(umem); in ib_umem_release()
[all …]
H A Dumem_odp.c121 struct ib_umem *umem; in ib_umem_odp_alloc_implicit() local
131 umem = &umem_odp->umem; in ib_umem_odp_alloc_implicit()
132 umem->ibdev = device; in ib_umem_odp_alloc_implicit()
169 struct ib_umem *umem; in ib_umem_odp_alloc_child() local
178 umem = &odp_data->umem; in ib_umem_odp_alloc_child()
179 umem->ibdev = root->umem.ibdev; in ib_umem_odp_alloc_child()
180 umem->length = size; in ib_umem_odp_alloc_child()
182 umem->writable = root->umem.writable; in ib_umem_odp_alloc_child()
183 umem->owning_mm = root->umem.owning_mm; in ib_umem_odp_alloc_child()
200 mmput(umem->owning_mm); in ib_umem_odp_alloc_child()
[all …]
H A Dumem_dmabuf.c40 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, in ib_umem_dmabuf_map_pages()
66 umem_dmabuf->umem.sgt_append.sgt.nents = nmap; in ib_umem_dmabuf_map_pages()
125 struct ib_umem *umem; in ib_umem_dmabuf_get_with_dma_device() local
148 umem = &umem_dmabuf->umem; in ib_umem_dmabuf_get_with_dma_device()
149 umem->ibdev = device; in ib_umem_dmabuf_get_with_dma_device()
150 umem->length = size; in ib_umem_dmabuf_get_with_dma_device()
151 umem->address = offset; in ib_umem_dmabuf_get_with_dma_device()
152 umem->writable = ib_access_writable(access); in ib_umem_dmabuf_get_with_dma_device()
153 umem->is_dmabuf = 1; in ib_umem_dmabuf_get_with_dma_device()
155 if (!ib_umem_num_pages(umem)) in ib_umem_dmabuf_get_with_dma_device()
[all …]
/linux-6.15/include/rdma/
H A Dib_umem.h32 struct ib_umem umem; member
46 return container_of(umem, struct ib_umem_dmabuf, umem); in to_ib_umem_dmabuf()
52 return umem->address & ~PAGE_MASK; in ib_umem_offset()
58 return (sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem)) & in ib_umem_dma_offset()
65 return (size_t)((ALIGN(umem->iova + umem->length, pgsz) - in ib_umem_num_dma_blocks()
66 ALIGN_DOWN(umem->iova, pgsz))) / in ib_umem_num_dma_blocks()
72 return ib_umem_num_dma_blocks(umem, PAGE_SIZE); in ib_umem_num_pages()
76 struct ib_umem *umem, in __rdma_umem_block_iter_start() argument
80 umem->sgt_append.sgt.nents, pgsz); in __rdma_umem_block_iter_start()
109 void ib_umem_release(struct ib_umem *umem);
[all …]
H A Dib_umem_odp.h13 struct ib_umem umem; member
47 static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem) in to_ib_umem_odp() argument
49 return container_of(umem, struct ib_umem_odp, umem); in to_ib_umem_odp()
/linux-6.15/tools/testing/selftests/bpf/
H A Dxsk.c94 return umem ? umem->fd : -EINVAL; in xsk_umem__fd()
237 umem = calloc(1, sizeof(*umem)); in xsk_umem__create()
238 if (!umem) in xsk_umem__create()
277 free(umem); in xsk_umem__create()
478 struct xsk_umem *umem = ctx->umem; in xsk_put_ctx() local
529 ctx->umem = umem; in xsk_create_ctx()
713 if (!umem) in xsk_socket__create()
726 if (!umem) in xsk_umem__delete()
733 if (!err && umem->fill_save && umem->comp_save) { in xsk_umem__delete()
741 free(umem); in xsk_umem__delete()
[all …]
H A Dxskxceiver.c175 return !!ifobj->umem->umem; in is_umem_valid()
185 return umem->num_frames * umem->frame_size; in umem_size()
210 &umem->fq, &umem->cq, &cfg); in xsk_configure_umem()
216 umem->base_addr = umem_size(umem); in xsk_configure_umem()
217 umem->next_buffer = umem_size(umem); in xsk_configure_umem()
228 umem->next_buffer += umem->frame_size; in umem_alloc_buffer()
229 if (umem->next_buffer >= umem->base_addr + umem_size(umem)) in umem_alloc_buffer()
230 umem->next_buffer = umem->base_addr; in umem_alloc_buffer()
267 xsk->umem = umem; in __xsk_configure_socket()
323 xsk_umem__delete(umem->umem); in ifobj_zc_avail()
[all …]
/linux-6.15/drivers/infiniband/sw/siw/
H A Dsiw_mem.c70 if (umem->base_mem) in siw_umem_release()
77 kfree(umem->page_chunk); in siw_umem_release()
78 kfree(umem); in siw_umem_release()
360 struct siw_umem *umem; in siw_umem_get() local
374 umem = kzalloc(sizeof(*umem), GFP_KERNEL); in siw_umem_get()
375 if (!umem) in siw_umem_get()
378 umem->page_chunk = in siw_umem_get()
380 if (!umem->page_chunk) { in siw_umem_get()
412 umem->num_pages++; in siw_umem_get()
419 return umem; in siw_umem_get()
[all …]
H A Dsiw_mem.h11 void siw_umem_release(struct siw_umem *umem);
59 static inline struct page *siw_get_upage(struct siw_umem *umem, u64 addr) in siw_get_upage() argument
61 unsigned int page_idx = (addr - umem->fp_addr) >> PAGE_SHIFT, in siw_get_upage()
65 if (likely(page_idx < umem->num_pages)) in siw_get_upage()
66 return umem->page_chunk[chunk_idx].plist[page_in_chunk]; in siw_get_upage()
/linux-6.15/drivers/infiniband/hw/mlx4/
H A Dmr.c77 mr->umem = NULL; in mlx4_ib_get_dma_mr()
91 struct ib_umem *umem) in mlx4_ib_umem_write_mtt() argument
155 if (IS_ERR(mr->umem)) { in mlx4_ib_reg_user_mr()
156 err = PTR_ERR(mr->umem); in mlx4_ib_reg_user_mr()
188 ib_umem_release(mr->umem); in mlx4_ib_reg_user_mr()
245 if (IS_ERR(mmr->umem)) { in mlx4_ib_rereg_user_mr()
246 err = PTR_ERR(mmr->umem); in mlx4_ib_rereg_user_mr()
248 mmr->umem = NULL; in mlx4_ib_rereg_user_mr()
344 if (mr->umem) in mlx4_ib_dereg_mr()
345 ib_umem_release(mr->umem); in mlx4_ib_dereg_mr()
[all …]
H A Ddoorbell.c40 struct ib_umem *umem; member
67 page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, in mlx4_ib_db_map_user()
69 if (IS_ERR(page->umem)) { in mlx4_ib_db_map_user()
70 err = PTR_ERR(page->umem); in mlx4_ib_db_map_user()
78 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + in mlx4_ib_db_map_user()
95 ib_umem_release(db->u.user_page->umem); in mlx4_ib_db_unmap_user()
/linux-6.15/lib/tests/
H A Dusercopy_kunit.c36 char __user *umem; member
50 char __user *umem = priv->umem; in usercopy_test_check_nonzero_user() local
66 umem += start; in usercopy_test_check_nonzero_user()
88 KUNIT_EXPECT_EQ_MSG(test, copy_to_user(umem, kmem, size), 0, in usercopy_test_check_nonzero_user()
94 int retval = check_zeroed_user(umem + start, len); in usercopy_test_check_nonzero_user()
109 char __user *umem = priv->umem; in usercopy_test_copy_struct_from_user() local
122 KUNIT_ASSERT_EQ_MSG(test, copy_to_user(umem, umem_src, size), 0, in usercopy_test_copy_struct_from_user()
163 KUNIT_EXPECT_EQ_MSG(test, clear_user(umem + ksize, usize - ksize), 0, in usercopy_test_copy_struct_from_user()
179 char __user *usermem = priv->umem; in usercopy_test_valid()
220 char __user *usermem = priv->umem; in usercopy_test_invalid()
[all …]
/linux-6.15/drivers/infiniband/hw/mlx5/
H A Dmem.c40 void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas, in mlx5_ib_populate_pas() argument
45 rdma_umem_for_each_dma_block (umem, &biter, page_size) { in mlx5_ib_populate_pas()
58 struct ib_umem *umem, unsigned long pgsz_bitmap, in __mlx5_umem_find_best_quantized_pgoff() argument
66 page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, pgoff_bitmask); in __mlx5_umem_find_best_quantized_pgoff()
77 page_offset = ib_umem_dma_offset(umem, page_size); in __mlx5_umem_find_best_quantized_pgoff()
80 page_offset = ib_umem_dma_offset(umem, page_size); in __mlx5_umem_find_best_quantized_pgoff()
H A Ddoorbell.c42 struct ib_umem *umem; member
69 page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, in mlx5_ib_db_map_user()
71 if (IS_ERR(page->umem)) { in mlx5_ib_db_map_user()
72 err = PTR_ERR(page->umem); in mlx5_ib_db_map_user()
82 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + in mlx5_ib_db_map_user()
100 ib_umem_release(db->u.user_page->umem); in mlx5_ib_db_unmap_user()
H A Dmr.c1067 mr->umem = NULL; in mlx5_ib_get_dma_mr()
1116 umem->iova = iova; in mlx5_umem_dmabuf_default_pgsz()
1130 if (umem->is_dmabuf) in alloc_cacheable_mr()
1162 mr->umem = umem; in alloc_cacheable_mr()
1314 mr->umem = umem; in reg_create()
1551 if (IS_ERR(umem)) in mlx5_ib_reg_user_mr()
1859 umem = mr->umem; in mlx5_ib_rereg_user_mr()
1860 mr->umem = NULL; in mlx5_ib_rereg_user_mr()
2133 if (mr->umem) { in __mlx5_ib_dereg_mr()
2266 mr->umem = NULL; in mlx5_ib_alloc_pi_mr()
[all …]
/linux-6.15/drivers/infiniband/sw/rxe/
H A Drxe_odp.c45 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); in rxe_odp_do_pagefault_and_lock()
66 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); in rxe_odp_init_pages()
69 ret = rxe_odp_do_pagefault_and_lock(mr, mr->umem->address, in rxe_odp_init_pages()
70 mr->umem->length, in rxe_odp_init_pages()
109 mr->umem = &umem_odp->umem; in rxe_odp_mr_init_user()
113 mr->page_offset = ib_umem_offset(&umem_odp->umem); in rxe_odp_mr_init_user()
152 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); in rxe_odp_map_range_and_lock()
187 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); in __rxe_odp_mr_copy()
225 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); in rxe_odp_mr_copy()
232 if (unlikely(!mr->umem->is_odp)) in rxe_odp_mr_copy()
[all …]
/linux-6.15/drivers/infiniband/hw/vmw_pvrdma/
H A Dpvrdma_mr.c117 struct ib_umem *umem; in pvrdma_reg_user_mr() local
129 umem = ib_umem_get(pd->device, start, length, access_flags); in pvrdma_reg_user_mr()
130 if (IS_ERR(umem)) { in pvrdma_reg_user_mr()
133 return ERR_CAST(umem); in pvrdma_reg_user_mr()
136 npages = ib_umem_num_dma_blocks(umem, PAGE_SIZE); in pvrdma_reg_user_mr()
152 mr->umem = umem; in pvrdma_reg_user_mr()
161 ret = pvrdma_page_dir_insert_umem(&mr->pdir, mr->umem, 0); in pvrdma_reg_user_mr()
190 ib_umem_release(umem); in pvrdma_reg_user_mr()
257 mr->umem = NULL; in pvrdma_alloc_mr()
294 ib_umem_release(mr->umem); in pvrdma_dereg_mr()
H A Dpvrdma_srq.c149 srq->umem = ib_umem_get(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); in pvrdma_create_srq()
150 if (IS_ERR(srq->umem)) { in pvrdma_create_srq()
151 ret = PTR_ERR(srq->umem); in pvrdma_create_srq()
155 srq->npages = ib_umem_num_dma_blocks(srq->umem, PAGE_SIZE); in pvrdma_create_srq()
171 pvrdma_page_dir_insert_umem(&srq->pdir, srq->umem, 0); in pvrdma_create_srq()
209 ib_umem_release(srq->umem); in pvrdma_create_srq()
229 ib_umem_release(srq->umem); in pvrdma_free_srq()
/linux-6.15/drivers/infiniband/hw/mana/
H A Dmr.c127 mr->umem = ib_umem_get(ibdev, start, length, access_flags); in mana_ib_reg_user_mr()
128 if (IS_ERR(mr->umem)) { in mana_ib_reg_user_mr()
129 err = PTR_ERR(mr->umem); in mana_ib_reg_user_mr()
135 err = mana_ib_create_dma_region(dev, mr->umem, &dma_region_handle, iova); in mana_ib_reg_user_mr()
169 ib_umem_release(mr->umem); in mana_ib_reg_user_mr()
206 mr->umem = &umem_dmabuf->umem; in mana_ib_reg_user_mr_dmabuf()
208 err = mana_ib_create_dma_region(dev, mr->umem, &dma_region_handle, iova); in mana_ib_reg_user_mr_dmabuf()
238 ib_umem_release(mr->umem); in mana_ib_reg_user_mr_dmabuf()
290 if (mr->umem) in mana_ib_dereg_mr()
291 ib_umem_release(mr->umem); in mana_ib_dereg_mr()
/linux-6.15/drivers/infiniband/hw/hns/
H A Dhns_roce_db.c32 page->umem = ib_umem_get(context->ibucontext.device, page_addr, in hns_roce_db_map_user()
34 if (IS_ERR(page->umem)) { in hns_roce_db_map_user()
35 ret = PTR_ERR(page->umem); in hns_roce_db_map_user()
44 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset; in hns_roce_db_map_user()
45 db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset; in hns_roce_db_map_user()
63 ib_umem_release(db->u.user_page->umem); in hns_roce_db_unmap_user()
/linux-6.15/drivers/vdpa/vdpa_user/
H A Dvduse_dev.c1038 if (!dev->umem) in vduse_dev_dereg_umem()
1051 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); in vduse_dev_dereg_umem()
1054 kfree(dev->umem); in vduse_dev_dereg_umem()
1055 dev->umem = NULL; in vduse_dev_dereg_umem()
1078 if (dev->umem) in vduse_dev_reg_umem()
1085 umem = kzalloc(sizeof(*umem), GFP_KERNEL); in vduse_dev_reg_umem()
1115 dev->umem = umem; in vduse_dev_reg_umem()
1124 kfree(umem); in vduse_dev_reg_umem()
1333 if (copy_from_user(&umem, argp, sizeof(umem))) in vduse_dev_ioctl()
1343 umem.uaddr, umem.size); in vduse_dev_ioctl()
[all …]

12345