567 static int kfd_gws_init(struct kfd_node *node)  in kfd_gws_init()  argument
570 	struct kfd_dev *kfd = node->kfd;  in kfd_gws_init()
571 	uint32_t mes_rev = node->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;  in kfd_gws_init()
573 	if (node->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)  in kfd_gws_init()
576 	if (hws_gws_support || (KFD_IS_SOC15(node) &&  in kfd_gws_init()
577 		((KFD_GC_VERSION(node) == IP_VERSION(9, 0, 1)  in kfd_gws_init()
579 		(KFD_GC_VERSION(node) <= IP_VERSION(9, 4, 0)  in kfd_gws_init()
581 		(KFD_GC_VERSION(node) == IP_VERSION(9, 4, 1)  in kfd_gws_init()
583 		(KFD_GC_VERSION(node) == IP_VERSION(9, 4, 2)  in kfd_gws_init()
585 		(KFD_GC_VERSION(node) == IP_VERSION(9, 4, 3) ||  in kfd_gws_init()
586 		 KFD_GC_VERSION(node) == IP_VERSION(9, 4, 4)) ||  in kfd_gws_init()
587 		(KFD_GC_VERSION(node) == IP_VERSION(9, 5, 0)) ||  in kfd_gws_init()
588 		(KFD_GC_VERSION(node) >= IP_VERSION(10, 3, 0)  in kfd_gws_init()
589 			&& KFD_GC_VERSION(node) < IP_VERSION(11, 0, 0)  in kfd_gws_init()
591 		(KFD_GC_VERSION(node) >= IP_VERSION(11, 0, 0)  in kfd_gws_init()
592 			&& KFD_GC_VERSION(node) < IP_VERSION(12, 0, 0)  in kfd_gws_init()
594 		(KFD_GC_VERSION(node) >= IP_VERSION(12, 0, 0))))) {  in kfd_gws_init()
595 		if (KFD_GC_VERSION(node) >= IP_VERSION(12, 0, 0))  in kfd_gws_init()
596 			node->adev->gds.gws_size = 64;  in kfd_gws_init()
597 		ret = amdgpu_amdkfd_alloc_gws(node->adev,  in kfd_gws_init()
598 				node->adev->gds.gws_size, &node->gws);  in kfd_gws_init()
610 static int kfd_init_node(struct kfd_node *node)  in kfd_init_node()  argument
614 	if (kfd_interrupt_init(node)) {  in kfd_init_node()
619 	node->dqm = device_queue_manager_init(node);  in kfd_init_node()
620 	if (!node->dqm) {  in kfd_init_node()
625 	if (kfd_gws_init(node)) {  in kfd_init_node()
627 			node->adev->gds.gws_size);  in kfd_init_node()
631 	if (kfd_resume(node))  in kfd_init_node()
634 	if (kfd_topology_add_device(node)) {  in kfd_init_node()
639 	kfd_smi_init(node);  in kfd_init_node()
646 	device_queue_manager_uninit(node->dqm);  in kfd_init_node()
648 	kfd_interrupt_exit(node);  in kfd_init_node()
650 	if (node->gws)  in kfd_init_node()
651 		amdgpu_amdkfd_free_gws(node->adev, node->gws);  in kfd_init_node()
654 	kfree(node);  in kfd_init_node()
683 static void kfd_setup_interrupt_bitmap(struct kfd_node *node,  in kfd_setup_interrupt_bitmap()  argument
686 	struct amdgpu_device *adev = node->adev;  in kfd_setup_interrupt_bitmap()
687 	uint32_t xcc_mask = node->xcc_mask;  in kfd_setup_interrupt_bitmap()
712 		node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2));  in kfd_setup_interrupt_bitmap()
715 							node->interrupt_bitmap);  in kfd_setup_interrupt_bitmap()
722 	struct kfd_node *node;  in kgd2kfd_device_init()  local
855 		node = kzalloc(sizeof(struct kfd_node), GFP_KERNEL);  in kgd2kfd_device_init()
856 		if (!node)  in kgd2kfd_device_init()
859 		node->node_id = i;  in kgd2kfd_device_init()
860 		node->adev = kfd->adev;  in kgd2kfd_device_init()
861 		node->kfd = kfd;  in kgd2kfd_device_init()
862 		node->kfd2kgd = kfd->kfd2kgd;  in kgd2kfd_device_init()
863 		node->vm_info.vmid_num_kfd = vmid_num_kfd;  in kgd2kfd_device_init()
864 		node->xcp = amdgpu_get_next_xcp(kfd->adev->xcp_mgr, &xcp_idx);  in kgd2kfd_device_init()
866 		if (node->xcp) {  in kgd2kfd_device_init()
867 			amdgpu_xcp_get_inst_details(node->xcp, AMDGPU_XCP_GFX,  in kgd2kfd_device_init()
868 						    &node->xcc_mask);  in kgd2kfd_device_init()
871 			node->xcc_mask =  in kgd2kfd_device_init()
875 		if (node->xcp) {  in kgd2kfd_device_init()
877 				node->node_id, node->xcp->mem_id,  in kgd2kfd_device_init()
878 				KFD_XCP_MEMORY_SIZE(node->adev, node->node_id) >> 20);  in kgd2kfd_device_init()
888 			node->vm_info.first_vmid_kfd = (i%2 == 0) ?  in kgd2kfd_device_init()
891 			node->vm_info.last_vmid_kfd = (i%2 == 0) ?  in kgd2kfd_device_init()
894 			node->compute_vmid_bitmap =  in kgd2kfd_device_init()
895 				((0x1 << (node->vm_info.last_vmid_kfd + 1)) - 1) -  in kgd2kfd_device_init()
896 				((0x1 << (node->vm_info.first_vmid_kfd)) - 1);  in kgd2kfd_device_init()
898 			node->vm_info.first_vmid_kfd = first_vmid_kfd;  in kgd2kfd_device_init()
899 			node->vm_info.last_vmid_kfd = last_vmid_kfd;  in kgd2kfd_device_init()
900 			node->compute_vmid_bitmap =  in kgd2kfd_device_init()
903 		node->max_proc_per_quantum = max_proc_per_quantum;  in kgd2kfd_device_init()
904 		atomic_set(&node->sram_ecc_flag, 0);  in kgd2kfd_device_init()
907 					&node->local_mem_info, node->xcp);  in kgd2kfd_device_init()
910 			kfd_setup_interrupt_bitmap(node, i);  in kgd2kfd_device_init()
913 		if (kfd_init_node(node)) {  in kgd2kfd_device_init()
918 		spin_lock_init(&node->watch_points_lock);  in kgd2kfd_device_init()
920 		kfd->nodes[i] = node;  in kgd2kfd_device_init()
930 		node->dqm->sched_policy);  in kgd2kfd_device_init()
968 	struct kfd_node *node;  in kgd2kfd_pre_reset()  local
975 		node = kfd->nodes[i];  in kgd2kfd_pre_reset()
976 		kfd_smi_event_update_gpu_reset(node, false, reset_context);  in kgd2kfd_pre_reset()
996 	struct kfd_node *node;  in kgd2kfd_post_reset()  local
1013 		node = kfd->nodes[i];  in kgd2kfd_post_reset()
1014 		atomic_set(&node->sram_ecc_flag, 0);  in kgd2kfd_post_reset()
1015 		kfd_smi_event_update_gpu_reset(node, true, NULL);  in kgd2kfd_post_reset()
1029 	struct kfd_node *node;  in kgd2kfd_suspend()  local
1045 		node = kfd->nodes[i];  in kgd2kfd_suspend()
1046 		node->dqm->ops.stop(node->dqm);  in kgd2kfd_suspend()
1075 static int kfd_resume(struct kfd_node *node)  in kfd_resume()  argument
1079 	err = node->dqm->ops.start(node->dqm);  in kfd_resume()
1083 			node->adev->pdev->vendor, node->adev->pdev->device);  in kfd_resume()
1094 	struct kfd_node *node;  in kgd2kfd_interrupt()  local
1105 		node = kfd->nodes[i];  in kgd2kfd_interrupt()
1106 		spin_lock_irqsave(&node->interrupt_lock, flags);  in kgd2kfd_interrupt()
1108 		if (node->interrupts_active  in kgd2kfd_interrupt()
1109 		    && interrupt_is_wanted(node, ih_ring_entry,  in kgd2kfd_interrupt()
1111 		    && enqueue_ih_ring_entry(node,  in kgd2kfd_interrupt()
1113 			queue_work(node->kfd->ih_wq, &node->interrupt_work);  in kgd2kfd_interrupt()
1114 			spin_unlock_irqrestore(&node->interrupt_lock, flags);  in kgd2kfd_interrupt()
1117 		spin_unlock_irqrestore(&node->interrupt_lock, flags);  in kgd2kfd_interrupt()
1257 int kfd_gtt_sa_allocate(struct kfd_node *node, unsigned int size,  in kfd_gtt_sa_allocate()  argument
1261 	struct kfd_dev *kfd = node->kfd;  in kfd_gtt_sa_allocate()
1361 int kfd_gtt_sa_free(struct kfd_node *node, struct kfd_mem_obj *mem_obj)  in kfd_gtt_sa_free()  argument
1363 	struct kfd_dev *kfd = node->kfd;  in kfd_gtt_sa_free()
1395 void kfd_inc_compute_active(struct kfd_node *node)  in kfd_inc_compute_active()  argument
1397 	if (atomic_inc_return(&node->kfd->compute_profile) == 1)  in kfd_inc_compute_active()
1398 		amdgpu_amdkfd_set_compute_idle(node->adev, false);  in kfd_inc_compute_active()
1401 void kfd_dec_compute_active(struct kfd_node *node)  in kfd_dec_compute_active()  argument
1403 	int count = atomic_dec_return(&node->kfd->compute_profile);  in kfd_dec_compute_active()
1406 		amdgpu_amdkfd_set_compute_idle(node->adev, true);  in kfd_dec_compute_active()
1410 static bool kfd_compute_active(struct kfd_node *node)  in kfd_compute_active()  argument
1412 	if (atomic_read(&node->kfd->compute_profile))  in kfd_compute_active()
1434 unsigned int kfd_get_num_sdma_engines(struct kfd_node *node)  in kfd_get_num_sdma_engines()  argument
1437 	if (!node->adev->gmc.xgmi.supported)  in kfd_get_num_sdma_engines()
1438 		return node->adev->sdma.num_instances/(int)node->kfd->num_nodes;  in kfd_get_num_sdma_engines()
1440 	return min(node->adev->sdma.num_instances/(int)node->kfd->num_nodes, 2);  in kfd_get_num_sdma_engines()
1443 unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_node *node)  in kfd_get_num_xgmi_sdma_engines()  argument
1446 	return node->adev->sdma.num_instances/(int)node->kfd->num_nodes -  in kfd_get_num_xgmi_sdma_engines()
1447 		kfd_get_num_sdma_engines(node);  in kfd_get_num_xgmi_sdma_engines()
1473 	struct kfd_node *node;  in kgd2kfd_start_sched()  local
1484 	node = kfd->nodes[node_id];  in kgd2kfd_start_sched()
1486 	ret = node->dqm->ops.unhalt(node->dqm);  in kgd2kfd_start_sched()
1495 	struct kfd_node *node;  in kgd2kfd_stop_sched()  local
1506 	node = kfd->nodes[node_id];  in kgd2kfd_stop_sched()
1507 	return node->dqm->ops.halt(node->dqm);  in kgd2kfd_stop_sched()
1512 	struct kfd_node *node;  in kgd2kfd_compute_active()  local
1523 	node = kfd->nodes[node_id];  in kgd2kfd_compute_active()
1525 	return kfd_compute_active(node);  in kgd2kfd_compute_active()