213 svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)  in svm_migrate_get_vram_page()  argument
218 	svm_range_bo_ref(prange->svm_bo);  in svm_migrate_get_vram_page()
219 	page->zone_device_data = prange->svm_bo;  in svm_migrate_get_vram_page()
277 svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,  in svm_migrate_copy_to_vram()  argument
291 	pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,  in svm_migrate_copy_to_vram()
292 		 prange->last, ttm_res_offset);  in svm_migrate_copy_to_vram()
297 	amdgpu_res_first(prange->ttm_res, ttm_res_offset,  in svm_migrate_copy_to_vram()
305 			svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
387 svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,  in svm_migrate_vma_to_vram()  argument
391 	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);  in svm_migrate_vma_to_vram()
422 				      0, node->id, prange->prefetch_loc,  in svm_migrate_vma_to_vram()
423 				      prange->preferred_loc, trigger);  in svm_migrate_vma_to_vram()
428 			__func__, r, prange->start, prange->last);  in svm_migrate_vma_to_vram()
435 			 prange->start, prange->last);  in svm_migrate_vma_to_vram()
444 	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);  in svm_migrate_vma_to_vram()
463 		pdd = svm_range_get_pdd_by_node(prange, node);  in svm_migrate_vma_to_vram()
487 svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,  in svm_migrate_ram_to_vram()  argument
498 	if (start_mgr < prange->start || last_mgr > prange->last) {  in svm_migrate_ram_to_vram()
500 			 start_mgr, last_mgr, prange->start, prange->last);  in svm_migrate_ram_to_vram()
504 	node = svm_range_get_node_by_id(prange, best_loc);  in svm_migrate_ram_to_vram()
511 		prange->svms, start_mgr, last_mgr, prange->start, prange->last,  in svm_migrate_ram_to_vram()
518 					prange->npages * PAGE_SIZE,  in svm_migrate_ram_to_vram()
526 	r = svm_range_vram_node_new(node, prange, true);  in svm_migrate_ram_to_vram()
531 	ttm_res_offset = (start_mgr - prange->start + prange->offset) << PAGE_SHIFT;  in svm_migrate_ram_to_vram()
541 		r = svm_migrate_vma_to_vram(node, prange, vma, addr, next, trigger, ttm_res_offset);  in svm_migrate_ram_to_vram()
553 		prange->actual_loc = best_loc;  in svm_migrate_ram_to_vram()
554 		prange->vram_pages += mpages;  in svm_migrate_ram_to_vram()
555 	} else if (!prange->actual_loc) {  in svm_migrate_ram_to_vram()
559 		svm_range_vram_node_free(prange);  in svm_migrate_ram_to_vram()
564 					prange->npages * PAGE_SIZE,  in svm_migrate_ram_to_vram()
581 svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,  in svm_migrate_copy_to_ram()  argument
593 	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,  in svm_migrate_copy_to_ram()
594 		 prange->last);  in svm_migrate_copy_to_ram()
607 				 prange->svms, prange->start, prange->last);  in svm_migrate_copy_to_ram()
633 				 prange->svms, prange->start, prange->last);  in svm_migrate_copy_to_ram()
685 svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,  in svm_migrate_vma_to_ram()  argument
689 	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);  in svm_migrate_vma_to_ram()
725 				      node->id, 0, prange->prefetch_loc,  in svm_migrate_vma_to_ram()
726 				      prange->preferred_loc, trigger);  in svm_migrate_vma_to_ram()
731 			__func__, r, prange->start, prange->last);  in svm_migrate_vma_to_ram()
738 			 prange->start, prange->last);  in svm_migrate_vma_to_ram()
748 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
769 		pdd = svm_range_get_pdd_by_node(prange, node);  in svm_migrate_vma_to_ram()
791 int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,  in svm_migrate_vram_to_ram()  argument
804 	if (!prange->actual_loc) {  in svm_migrate_vram_to_ram()
806 			 prange->start, prange->last);  in svm_migrate_vram_to_ram()
810 	if (start_mgr < prange->start || last_mgr > prange->last) {  in svm_migrate_vram_to_ram()
812 			 start_mgr, last_mgr, prange->start, prange->last);  in svm_migrate_vram_to_ram()
816 	node = svm_range_get_node_by_id(prange, prange->actual_loc);  in svm_migrate_vram_to_ram()
818 		pr_debug("failed to get kfd node by id 0x%x\n", prange->actual_loc);  in svm_migrate_vram_to_ram()
822 		 prange->svms, prange, start_mgr, last_mgr,  in svm_migrate_vram_to_ram()
823 		 prange->actual_loc);  in svm_migrate_vram_to_ram()
833 			pr_debug("failed to find vma for prange %p\n", prange);  in svm_migrate_vram_to_ram()
839 		r = svm_migrate_vma_to_ram(node, prange, vma, addr, next, trigger,  in svm_migrate_vram_to_ram()
842 			pr_debug("failed %ld to migrate prange %p\n", r, prange);  in svm_migrate_vram_to_ram()
851 		prange->vram_pages -= mpages;  in svm_migrate_vram_to_ram()
856 		if (prange->vram_pages == 0 && prange->ttm_res) {  in svm_migrate_vram_to_ram()
857 			prange->actual_loc = 0;  in svm_migrate_vram_to_ram()
858 			svm_range_vram_node_free(prange);  in svm_migrate_vram_to_ram()
883 svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,  in svm_migrate_vram_to_vram()  argument
894 	pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);  in svm_migrate_vram_to_vram()
897 		r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,  in svm_migrate_vram_to_vram()
901 	} while (prange->actual_loc && --retries);  in svm_migrate_vram_to_vram()
903 	if (prange->actual_loc)  in svm_migrate_vram_to_vram()
906 	return svm_migrate_ram_to_vram(prange, best_loc, start, last, mm, trigger);  in svm_migrate_vram_to_vram()
910 svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,  in svm_migrate_to_vram()  argument
914 	if  (!prange->actual_loc || prange->actual_loc == best_loc)  in svm_migrate_to_vram()
915 		return svm_migrate_ram_to_vram(prange, best_loc, start, last,  in svm_migrate_to_vram()
919 		return svm_migrate_vram_to_vram(prange, best_loc, start, last,  in svm_migrate_to_vram()
939 	struct svm_range *prange;  in svm_migrate_to_ram()  local
975 	prange = svm_range_from_addr(&p->svms, addr, NULL);  in svm_migrate_to_ram()
976 	if (!prange) {  in svm_migrate_to_ram()
982 	mutex_lock(&prange->migrate_mutex);  in svm_migrate_to_ram()
984 	if (!prange->actual_loc)  in svm_migrate_to_ram()
988 	size = 1UL << prange->granularity;  in svm_migrate_to_ram()
989 	start = max(ALIGN_DOWN(addr, size), prange->start);  in svm_migrate_to_ram()
990 	last = min(ALIGN(addr + 1, size) - 1, prange->last);  in svm_migrate_to_ram()
992 	r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, start, last,  in svm_migrate_to_ram()
996 			r, prange->svms, prange, start, last);  in svm_migrate_to_ram()
999 	mutex_unlock(&prange->migrate_mutex);  in svm_migrate_to_ram()