61 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);  in amdgpu_bo_destroy()  local
63 	amdgpu_bo_kunmap(bo);  in amdgpu_bo_destroy()
65 	if (bo->tbo.base.import_attach)  in amdgpu_bo_destroy()
66 		drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);  in amdgpu_bo_destroy()
67 	drm_gem_object_release(&bo->tbo.base);  in amdgpu_bo_destroy()
68 	amdgpu_bo_unref(&bo->parent);  in amdgpu_bo_destroy()
69 	kvfree(bo);  in amdgpu_bo_destroy()
74 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);  in amdgpu_bo_user_destroy()  local
77 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_user_destroy()
92 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)  in amdgpu_bo_is_amdgpu_bo()  argument
94 	if (bo->destroy == &amdgpu_bo_destroy ||  in amdgpu_bo_is_amdgpu_bo()
95 	    bo->destroy == &amdgpu_bo_user_destroy)  in amdgpu_bo_is_amdgpu_bo()
376 			   struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo,  in amdgpu_bo_create_isp_user()  argument
384 	*bo = gem_to_amdgpu_bo(gem_obj);  in amdgpu_bo_create_isp_user()
385 	if (!(*bo)) {  in amdgpu_bo_create_isp_user()
390 	r = amdgpu_bo_reserve(*bo, false);  in amdgpu_bo_create_isp_user()
396 	r = amdgpu_bo_pin(*bo, domain);  in amdgpu_bo_create_isp_user()
402 	r = amdgpu_ttm_alloc_gart(&(*bo)->tbo);  in amdgpu_bo_create_isp_user()
404 		dev_err(adev->dev, "%p bind failed\n", *bo);  in amdgpu_bo_create_isp_user()
409 		*gpu_addr = amdgpu_bo_gpu_offset(*bo);  in amdgpu_bo_create_isp_user()
411 	amdgpu_bo_unreserve(*bo);  in amdgpu_bo_create_isp_user()
416 	amdgpu_bo_unpin(*bo);  in amdgpu_bo_create_isp_user()
418 	amdgpu_bo_unreserve(*bo);  in amdgpu_bo_create_isp_user()
419 	amdgpu_bo_unref(bo);  in amdgpu_bo_create_isp_user()
504 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,  in amdgpu_bo_free_kernel()  argument
507 	if (*bo == NULL)  in amdgpu_bo_free_kernel()
510 	WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);  in amdgpu_bo_free_kernel()
512 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {  in amdgpu_bo_free_kernel()
514 			amdgpu_bo_kunmap(*bo);  in amdgpu_bo_free_kernel()
516 		amdgpu_bo_unpin(*bo);  in amdgpu_bo_free_kernel()
517 		amdgpu_bo_unreserve(*bo);  in amdgpu_bo_free_kernel()
519 	amdgpu_bo_unref(bo);  in amdgpu_bo_free_kernel()
539 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo)  in amdgpu_bo_free_isp_user()  argument
541 	if (bo == NULL)  in amdgpu_bo_free_isp_user()
544 	if (amdgpu_bo_reserve(bo, true) == 0) {  in amdgpu_bo_free_isp_user()
545 		amdgpu_bo_unpin(bo);  in amdgpu_bo_free_isp_user()
546 		amdgpu_bo_unreserve(bo);  in amdgpu_bo_free_isp_user()
548 	amdgpu_bo_unref(&bo);  in amdgpu_bo_free_isp_user()
640 	struct amdgpu_bo *bo;  in amdgpu_bo_create()  local
666 	bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);  in amdgpu_bo_create()
667 	if (bo == NULL)  in amdgpu_bo_create()
669 	drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);  in amdgpu_bo_create()
670 	bo->tbo.base.funcs = &amdgpu_gem_object_funcs;  in amdgpu_bo_create()
671 	bo->vm_bo = NULL;  in amdgpu_bo_create()
672 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :  in amdgpu_bo_create()
674 	bo->allowed_domains = bo->preferred_domains;  in amdgpu_bo_create()
677 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)  in amdgpu_bo_create()
678 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;  in amdgpu_bo_create()
680 	bo->flags = bp->flags;  in amdgpu_bo_create()
684 		bo->xcp_id = bp->xcp_id_plus1 - 1;  in amdgpu_bo_create()
687 		bo->xcp_id = 0;  in amdgpu_bo_create()
689 	if (!amdgpu_bo_support_uswc(bo->flags))  in amdgpu_bo_create()
690 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;  in amdgpu_bo_create()
692 	bo->tbo.bdev = &adev->mman.bdev;  in amdgpu_bo_create()
695 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in amdgpu_bo_create()
697 		amdgpu_bo_placement_from_domain(bo, bp->domain);  in amdgpu_bo_create()
699 		bo->tbo.priority = 2;  in amdgpu_bo_create()
701 		bo->tbo.priority = 1;  in amdgpu_bo_create()
706 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,  in amdgpu_bo_create()
707 				 &bo->placement, page_align, &ctx,  NULL,  in amdgpu_bo_create()
713 	    amdgpu_res_cpu_visible(adev, bo->tbo.resource))  in amdgpu_bo_create()
720 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {  in amdgpu_bo_create()
723 		r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence);  in amdgpu_bo_create()
727 		dma_resv_add_fence(bo->tbo.base.resv, fence,  in amdgpu_bo_create()
732 		amdgpu_bo_unreserve(bo);  in amdgpu_bo_create()
733 	*bo_ptr = bo;  in amdgpu_bo_create()
735 	trace_amdgpu_bo_create(bo);  in amdgpu_bo_create()
739 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_create()
745 		dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_bo_create()
746 	amdgpu_bo_unref(&bo);  in amdgpu_bo_create()
821 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)  in amdgpu_bo_kmap()  argument
826 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)  in amdgpu_bo_kmap()
829 	r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,  in amdgpu_bo_kmap()
834 	kptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
841 	r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);  in amdgpu_bo_kmap()
846 		*ptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
860 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)  in amdgpu_bo_kptr()  argument
864 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);  in amdgpu_bo_kptr()
873 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)  in amdgpu_bo_kunmap()  argument
875 	if (bo->kmap.bo)  in amdgpu_bo_kunmap()
876 		ttm_bo_kunmap(&bo->kmap);  in amdgpu_bo_kunmap()
888 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)  in amdgpu_bo_ref()  argument
890 	if (bo == NULL)  in amdgpu_bo_ref()
893 	drm_gem_object_get(&bo->tbo.base);  in amdgpu_bo_ref()
894 	return bo;  in amdgpu_bo_ref()
903 void amdgpu_bo_unref(struct amdgpu_bo **bo)  in amdgpu_bo_unref()  argument
905 	if ((*bo) == NULL)  in amdgpu_bo_unref()
908 	drm_gem_object_put(&(*bo)->tbo.base);  in amdgpu_bo_unref()
909 	*bo = NULL;  in amdgpu_bo_unref()
928 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)  in amdgpu_bo_pin()  argument
930 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_pin()
934 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))  in amdgpu_bo_pin()
938 	if (bo->preferred_domains & domain)  in amdgpu_bo_pin()
939 		domain = bo->preferred_domains & domain;  in amdgpu_bo_pin()
942 	if (bo->tbo.base.import_attach) {  in amdgpu_bo_pin()
949 	if (bo->tbo.pin_count) {  in amdgpu_bo_pin()
950 		uint32_t mem_type = bo->tbo.resource->mem_type;  in amdgpu_bo_pin()
951 		uint32_t mem_flags = bo->tbo.resource->placement;  in amdgpu_bo_pin()
957 		    (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&  in amdgpu_bo_pin()
961 		ttm_bo_pin(&bo->tbo);  in amdgpu_bo_pin()
970 	if (bo->tbo.base.import_attach)  in amdgpu_bo_pin()
971 		dma_buf_pin(bo->tbo.base.import_attach);  in amdgpu_bo_pin()
974 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))  in amdgpu_bo_pin()
975 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_pin()
976 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_bo_pin()
977 	for (i = 0; i < bo->placement.num_placement; i++) {  in amdgpu_bo_pin()
978 		if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&  in amdgpu_bo_pin()
979 		    bo->placements[i].mem_type == TTM_PL_VRAM)  in amdgpu_bo_pin()
980 			bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;  in amdgpu_bo_pin()
983 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_pin()
985 		dev_err(adev->dev, "%p pin failed\n", bo);  in amdgpu_bo_pin()
989 	ttm_bo_pin(&bo->tbo);  in amdgpu_bo_pin()
991 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {  in amdgpu_bo_pin()
992 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_pin()
993 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_pin()
995 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {  in amdgpu_bo_pin()
996 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_pin()
1013 void amdgpu_bo_unpin(struct amdgpu_bo *bo)  in amdgpu_bo_unpin()  argument
1015 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_unpin()
1017 	ttm_bo_unpin(&bo->tbo);  in amdgpu_bo_unpin()
1018 	if (bo->tbo.pin_count)  in amdgpu_bo_unpin()
1021 	if (bo->tbo.base.import_attach)  in amdgpu_bo_unpin()
1022 		dma_buf_unpin(bo->tbo.base.import_attach);  in amdgpu_bo_unpin()
1024 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {  in amdgpu_bo_unpin()
1025 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_unpin()
1026 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_unpin()
1028 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {  in amdgpu_bo_unpin()
1029 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_unpin()
1117 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)  in amdgpu_bo_set_tiling_flags()  argument
1119 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_set_tiling_flags()
1122 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_set_tiling_flags()
1127 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_set_tiling_flags()
1140 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)  in amdgpu_bo_get_tiling_flags()  argument
1144 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_get_tiling_flags()
1145 	dma_resv_assert_held(bo->tbo.base.resv);  in amdgpu_bo_get_tiling_flags()
1146 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_get_tiling_flags()
1165 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,  in amdgpu_bo_set_metadata()  argument
1171 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_set_metadata()
1172 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_set_metadata()
1212 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,  in amdgpu_bo_get_metadata()  argument
1221 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_get_metadata()
1222 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_get_metadata()
1250 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,  in amdgpu_bo_move_notify()  argument
1254 	struct ttm_resource *old_mem = bo->resource;  in amdgpu_bo_move_notify()
1257 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_move_notify()
1260 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_move_notify()
1281 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_release_notify()  argument
1283 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_release_notify()
1288 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_release_notify()
1291 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_release_notify()
1304 	if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))  in amdgpu_bo_release_notify()
1309 	if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||  in amdgpu_bo_release_notify()
1314 	r = dma_resv_reserve_fences(&bo->base._resv, 1);  in amdgpu_bo_release_notify()
1318 	r = amdgpu_fill_buffer(abo, 0, &bo->base._resv, &fence, true);  in amdgpu_bo_release_notify()
1322 	amdgpu_vram_mgr_set_cleared(bo->resource);  in amdgpu_bo_release_notify()
1323 	dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL);  in amdgpu_bo_release_notify()
1327 	dma_resv_unlock(&bo->base._resv);  in amdgpu_bo_release_notify()
1341 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_fault_reserve_notify()  argument
1343 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_fault_reserve_notify()
1345 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_fault_reserve_notify()
1351 	if (amdgpu_res_cpu_visible(adev, bo->resource))  in amdgpu_bo_fault_reserve_notify()
1366 	r = ttm_bo_validate(bo, &abo->placement, &ctx);  in amdgpu_bo_fault_reserve_notify()
1373 	if (bo->resource->mem_type == TTM_PL_VRAM &&  in amdgpu_bo_fault_reserve_notify()
1374 	    !amdgpu_res_cpu_visible(adev, bo->resource))  in amdgpu_bo_fault_reserve_notify()
1377 	ttm_bo_move_to_lru_tail_unlocked(bo);  in amdgpu_bo_fault_reserve_notify()
1389 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,  in amdgpu_bo_fence()  argument
1392 	struct dma_resv *resv = bo->tbo.base.resv;  in amdgpu_bo_fence()
1444 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)  in amdgpu_bo_sync_wait()  argument
1446 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_sync_wait()
1448 	return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,  in amdgpu_bo_sync_wait()
1462 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)  in amdgpu_bo_gpu_offset()  argument
1464 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);  in amdgpu_bo_gpu_offset()
1465 	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&  in amdgpu_bo_gpu_offset()
1466 		     !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);  in amdgpu_bo_gpu_offset()
1467 	WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);  in amdgpu_bo_gpu_offset()
1468 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&  in amdgpu_bo_gpu_offset()
1469 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));  in amdgpu_bo_gpu_offset()
1471 	return amdgpu_bo_gpu_offset_no_check(bo);  in amdgpu_bo_gpu_offset()
1481 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)  in amdgpu_bo_gpu_offset_no_check()  argument
1483 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_gpu_offset_no_check()
1486 	if (bo->tbo.resource->mem_type == TTM_PL_TT)  in amdgpu_bo_gpu_offset_no_check()
1487 		offset = amdgpu_gmc_agp_addr(&bo->tbo);  in amdgpu_bo_gpu_offset_no_check()
1490 		offset = (bo->tbo.resource->start << PAGE_SHIFT) +  in amdgpu_bo_gpu_offset_no_check()
1491 			amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);  in amdgpu_bo_gpu_offset_no_check()
1508 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo)  in amdgpu_bo_mem_stats_placement()  argument
1510 	uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK;  in amdgpu_bo_mem_stats_placement()
1556 #define amdgpu_bo_print_flag(m, bo, flag)		        \  argument
1558 		if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) {	\
1575 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)  in amdgpu_bo_print_info()  argument
1577 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_print_info()
1584 	if (dma_resv_trylock(bo->tbo.base.resv)) {  in amdgpu_bo_print_info()
1585 		if (!bo->tbo.resource) {  in amdgpu_bo_print_info()
1588 			switch (bo->tbo.resource->mem_type) {  in amdgpu_bo_print_info()
1590 				if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))  in amdgpu_bo_print_info()
1619 		dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_bo_print_info()
1624 	size = amdgpu_bo_size(bo);  in amdgpu_bo_print_info()
1628 	pin_count = READ_ONCE(bo->tbo.pin_count);  in amdgpu_bo_print_info()
1632 	dma_buf = READ_ONCE(bo->tbo.base.dma_buf);  in amdgpu_bo_print_info()
1633 	attachment = READ_ONCE(bo->tbo.base.import_attach);  in amdgpu_bo_print_info()
1640 	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);  in amdgpu_bo_print_info()
1641 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);  in amdgpu_bo_print_info()
1642 	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);  in amdgpu_bo_print_info()
1643 	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);  in amdgpu_bo_print_info()
1644 	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);  in amdgpu_bo_print_info()
1645 	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);  in amdgpu_bo_print_info()
1646 	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);  in amdgpu_bo_print_info()