107 	struct qcm_process_device *qpd;  in kfd_sdma_activity_worker()  local
120 	qpd = &pdd->qpd;  in kfd_sdma_activity_worker()
121 	if (!dqm || !qpd)  in kfd_sdma_activity_worker()
152 	list_for_each_entry(q, &qpd->queues_list, list) {  in kfd_sdma_activity_worker()
214 	list_for_each_entry(q, &qpd->queues_list, list) {  in kfd_sdma_activity_worker()
285 	if (pdd->qpd.queue_count == 0) {  in kfd_get_cu_occupancy()
311 		    kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,  in kfd_get_cu_occupancy()
794 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_reserve_ib_mem()  local
803 	if (qpd->ib_kaddr || !qpd->ib_base)  in kfd_process_device_reserve_ib_mem()
807 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,  in kfd_process_device_reserve_ib_mem()
812 	qpd->ib_mem = mem;  in kfd_process_device_reserve_ib_mem()
813 	qpd->ib_kaddr = kaddr;  in kfd_process_device_reserve_ib_mem()
820 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_destroy_ib_mem()  local
822 	if (!qpd->ib_kaddr || !qpd->ib_base)  in kfd_process_device_destroy_ib_mem()
825 	kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);  in kfd_process_device_destroy_ib_mem()
1065 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)  in kfd_process_destroy_pdds()
1066 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,  in kfd_process_destroy_pdds()
1340 		struct qcm_process_device *qpd = &p->pdds[i]->qpd;  in kfd_process_init_cwsr_apu()  local
1342 		if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)  in kfd_process_init_cwsr_apu()
1346 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,  in kfd_process_init_cwsr_apu()
1350 		if (IS_ERR_VALUE(qpd->tba_addr)) {  in kfd_process_init_cwsr_apu()
1351 			int err = qpd->tba_addr;  in kfd_process_init_cwsr_apu()
1355 			qpd->tba_addr = 0;  in kfd_process_init_cwsr_apu()
1356 			qpd->cwsr_kaddr = NULL;  in kfd_process_init_cwsr_apu()
1360 		memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);  in kfd_process_init_cwsr_apu()
1362 		kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);  in kfd_process_init_cwsr_apu()
1364 		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;  in kfd_process_init_cwsr_apu()
1366 			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);  in kfd_process_init_cwsr_apu()
1377 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_init_cwsr_dgpu()  local
1385 	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)  in kfd_process_device_init_cwsr_dgpu()
1389 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,  in kfd_process_device_init_cwsr_dgpu()
1394 	qpd->cwsr_mem = mem;  in kfd_process_device_init_cwsr_dgpu()
1395 	qpd->cwsr_kaddr = kaddr;  in kfd_process_device_init_cwsr_dgpu()
1396 	qpd->tba_addr = qpd->cwsr_base;  in kfd_process_device_init_cwsr_dgpu()
1398 	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);  in kfd_process_device_init_cwsr_dgpu()
1400 	kfd_process_set_trap_debug_flag(&pdd->qpd,  in kfd_process_device_init_cwsr_dgpu()
1403 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;  in kfd_process_device_init_cwsr_dgpu()
1405 		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);  in kfd_process_device_init_cwsr_dgpu()
1413 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_destroy_cwsr_dgpu()  local
1415 	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)  in kfd_process_device_destroy_cwsr_dgpu()
1418 	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);  in kfd_process_device_destroy_cwsr_dgpu()
1421 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,  in kfd_process_set_trap_handler()  argument
1425 	if (qpd->cwsr_kaddr) {  in kfd_process_set_trap_handler()
1430 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);  in kfd_process_set_trap_handler()
1435 		qpd->tba_addr = tba_addr;  in kfd_process_set_trap_handler()
1436 		qpd->tma_addr = tma_addr;  in kfd_process_set_trap_handler()
1494 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,  in kfd_process_set_trap_debug_flag()  argument
1497 	if (qpd->cwsr_kaddr) {  in kfd_process_set_trap_debug_flag()
1499 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);  in kfd_process_set_trap_debug_flag()
1622 	INIT_LIST_HEAD(&pdd->qpd.queues_list);  in kfd_create_process_device_data()
1623 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);  in kfd_create_process_device_data()
1624 	pdd->qpd.dqm = dev->dqm;  in kfd_create_process_device_data()
1625 	pdd->qpd.pqm = &p->pqm;  in kfd_create_process_device_data()
1626 	pdd->qpd.evicted = 0;  in kfd_create_process_device_data()
1627 	pdd->qpd.mapped_gws_queue = false;  in kfd_create_process_device_data()
1887 							    &pdd->qpd);  in kfd_process_evict_queues()
1916 							      &pdd->qpd))  in kfd_process_evict_queues()
1939 							      &pdd->qpd);  in kfd_process_restore_queues()
2117 	struct qcm_process_device *qpd;  in kfd_reserved_mem_mmap()  local
2127 	qpd = &pdd->qpd;  in kfd_reserved_mem_mmap()
2129 	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in kfd_reserved_mem_mmap()
2131 	if (!qpd->cwsr_kaddr) {  in kfd_reserved_mem_mmap()
2141 			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),  in kfd_reserved_mem_mmap()