183 			struct amdgpu_device *bo_adev = prange->svm_bo->node->adev;  in svm_range_dma_map_dev()
421 			pdd = kfd_get_process_device_data(svm_bo->node, p);  in svm_range_bo_release()
467 svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)  in svm_range_validate_svm_bo()  argument
485 		if (prange->svm_bo->node != node) {  in svm_range_validate_svm_bo()
553 svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,  in svm_range_vram_node_new()  argument
570 	if (svm_range_validate_svm_bo(node, prange))  in svm_range_vram_node_new()
584 	svm_bo->node = node;  in svm_range_vram_node_new()
601 	if (node->xcp)  in svm_range_vram_node_new()
602 		bp.xcp_id_plus1 = node->xcp->id + 1;  in svm_range_vram_node_new()
604 	r = amdgpu_bo_create_user(node->adev, &bp, &ubo);  in svm_range_vram_node_new()
649 	pdd = svm_range_get_pdd_by_node(prange, node);  in svm_range_vram_node_new()
695 svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node)  in svm_range_get_pdd_by_node()  argument
701 	return kfd_get_process_device_data(node, p);  in svm_range_get_pdd_by_node()
880 	struct interval_tree_node *node;  in svm_range_debug_dump()  local
895 	node = interval_tree_iter_first(&svms->objects, 0, ~0ULL);  in svm_range_debug_dump()
896 	while (node) {  in svm_range_debug_dump()
897 		prange = container_of(node, struct svm_range, it_node);  in svm_range_debug_dump()
902 		node = interval_tree_iter_next(node, 0, ~0ULL);  in svm_range_debug_dump()
1193 svm_range_get_pte_flags(struct kfd_node *node,  in svm_range_get_pte_flags()  argument
1199 	uint32_t gc_ip_version = KFD_GC_VERSION(node);  in svm_range_get_pte_flags()
1207 		bo_node = prange->svm_bo->node;  in svm_range_get_pte_flags()
1212 			if (bo_node == node) {  in svm_range_get_pte_flags()
1218 				if (svm_nodes_in_same_hive(node, bo_node))  in svm_range_get_pte_flags()
1228 			if (bo_node == node) {  in svm_range_get_pte_flags()
1231 				if (node->adev->gmc.xgmi.connected_to_cpu)  in svm_range_get_pte_flags()
1236 				if (svm_nodes_in_same_hive(node, bo_node))  in svm_range_get_pte_flags()
1248 			mtype_local = (gc_ip_version < IP_VERSION(9, 5, 0) && !node->adev->rev_id) ?  in svm_range_get_pte_flags()
1256 			if (bo_node->adev == node->adev &&  in svm_range_get_pte_flags()
1257 			    (!bo_node->xcp || !node->xcp || bo_node->xcp->mem_id == node->xcp->mem_id))  in svm_range_get_pte_flags()
1262 			else if (svm_nodes_in_same_hive(bo_node, node) && !ext_coherent)  in svm_range_get_pte_flags()
1266 				 !svm_nodes_in_same_hive(bo_node, node))  in svm_range_get_pte_flags()
1272 		} else if (node->adev->flags & AMD_IS_APU) {  in svm_range_get_pte_flags()
1310 	pte_flags |= amdgpu_gem_va_map_flags(node->adev, mapping_flags);  in svm_range_get_pte_flags()
1474 		bo_adev = prange->svm_bo->node->adev;  in svm_range_map_to_gpus()
2120 	struct interval_tree_node *node;  in svm_range_add()  local
2134 	node = interval_tree_iter_first(&svms->objects, start, last);  in svm_range_add()
2135 	while (node) {  in svm_range_add()
2139 		pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start,  in svm_range_add()
2140 			 node->last);  in svm_range_add()
2142 		prange = container_of(node, struct svm_range, it_node);  in svm_range_add()
2143 		next = interval_tree_iter_next(node, start, last);  in svm_range_add()
2144 		next_start = min(node->last, last) + 1;  in svm_range_add()
2149 		} else if (node->start < start || node->last > last) {  in svm_range_add()
2166 			if (node->start < start) {  in svm_range_add()
2173 			if (node->last > last) {  in svm_range_add()
2188 		if (node->start > start) {  in svm_range_add()
2189 			r = svm_range_split_new(svms, start, node->start - 1,  in svm_range_add()
2196 		node = next;  in svm_range_add()
2621 	struct interval_tree_node *node;  in svm_range_from_addr()  local
2625 	node = interval_tree_iter_first(&svms->objects, addr, addr);  in svm_range_from_addr()
2626 	if (!node)  in svm_range_from_addr()
2629 	prange = container_of(node, struct svm_range, it_node);  in svm_range_from_addr()
2631 		 addr, prange->start, prange->last, node->start, node->last);  in svm_range_from_addr()
2673 				struct kfd_node *node,  in svm_range_best_restore_location()  argument
2683 	r = kfd_process_gpuid_from_node(p, node, &gpuid, gpuidx);  in svm_range_best_restore_location()
2689 	if (node->adev->apu_prefer_gtt)  in svm_range_best_restore_location()
2697 		if (preferred_node && svm_nodes_in_same_hive(node, preferred_node))  in svm_range_best_restore_location()
2710 		if (bo_node && svm_nodes_in_same_hive(node, bo_node))  in svm_range_best_restore_location()
2725 	struct interval_tree_node *node;  in svm_range_get_range_boundaries()  local
2743 	node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX);  in svm_range_get_range_boundaries()
2744 	if (node) {  in svm_range_get_range_boundaries()
2745 		end_limit = min(end_limit, node->start);  in svm_range_get_range_boundaries()
2747 		rb_node = rb_prev(&node->rb);  in svm_range_get_range_boundaries()
2755 		node = container_of(rb_node, struct interval_tree_node, rb);  in svm_range_get_range_boundaries()
2756 		if (node->last >= addr) {  in svm_range_get_range_boundaries()
2760 		start_limit = max(start_limit, node->last + 1);  in svm_range_get_range_boundaries()
2778 	struct interval_tree_node *node;  in svm_range_check_vm_userptr()  local
2796 		node = interval_tree_iter_first(&vm->va, 0, ~0ULL);  in svm_range_check_vm_userptr()
2797 		while (node) {  in svm_range_check_vm_userptr()
2798 			mapping = container_of((struct rb_node *)node,  in svm_range_check_vm_userptr()
2806 				node = interval_tree_iter_next(node, 0, ~0ULL);  in svm_range_check_vm_userptr()
2825 svm_range *svm_range_create_unregistered_range(struct kfd_node *node,  in svm_range_create_unregistered_range()  argument
2860 	if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx)) {  in svm_range_create_unregistered_range()
2914 svm_range_count_fault(struct kfd_node *node, struct kfd_process *p,  in svm_range_count_fault()  argument
2927 		r = kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx);  in svm_range_count_fault()
2964 	struct kfd_node *node;  in svm_range_restore_pages()  local
2992 	node = kfd_node_by_irq_ids(adev, node_id, vmid);  in svm_range_restore_pages()
2993 	if (!node) {  in svm_range_restore_pages()
3000 	if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx)) {  in svm_range_restore_pages()
3055 		prange = svm_range_create_unregistered_range(node, p, mm, addr);  in svm_range_restore_pages()
3070 		amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid);  in svm_range_restore_pages()
3101 	best_loc = svm_range_best_restore_location(prange, node, &gpuidx);  in svm_range_restore_pages()
3113 	kfd_smi_event_page_fault_start(node, p->lead_thread->pid, addr,  in svm_range_restore_pages()
3156 	kfd_smi_event_page_fault_end(node, p->lead_thread->pid, addr,  in svm_range_restore_pages()
3166 		svm_range_count_fault(node, p, gpuidx);  in svm_range_restore_pages()
3174 		amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid);  in svm_range_restore_pages()
3325 	struct interval_tree_node *node;  in svm_range_check_vm()  local
3340 		node = interval_tree_iter_first(&vm->va, start, last);  in svm_range_check_vm()
3341 		if (node) {  in svm_range_check_vm()
3344 			mapping = container_of((struct rb_node *)node,  in svm_range_check_vm()
3760 	struct interval_tree_node *node;  in svm_range_get_attr()  local
3821 	node = interval_tree_iter_first(&svms->objects, start, last);  in svm_range_get_attr()
3822 	if (!node) {  in svm_range_get_attr()
3838 	while (node) {  in svm_range_get_attr()
3841 		prange = container_of(node, struct svm_range, it_node);  in svm_range_get_attr()
3842 		next = interval_tree_iter_next(node, start, last);  in svm_range_get_attr()
3880 		node = next;  in svm_range_get_attr()