Lines Matching refs:dqm
49 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
52 static int execute_queues_cpsch(struct device_queue_manager *dqm,
56 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
62 static int map_queues_cpsch(struct device_queue_manager *dqm);
64 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
67 static inline void deallocate_hqd(struct device_queue_manager *dqm,
69 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
70 static int allocate_sdma_queue(struct device_queue_manager *dqm,
73 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
83 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) in is_pipe_enabled() argument
86 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec in is_pipe_enabled()
87 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; in is_pipe_enabled()
90 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) in is_pipe_enabled()
92 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) in is_pipe_enabled()
97 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) in get_cp_queues_num() argument
99 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, in get_cp_queues_num()
103 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) in get_queues_per_pipe() argument
105 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; in get_queues_per_pipe()
108 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) in get_pipes_per_mec() argument
110 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; in get_pipes_per_mec()
113 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) in get_num_all_sdma_engines() argument
115 return kfd_get_num_sdma_engines(dqm->dev) + in get_num_all_sdma_engines()
116 kfd_get_num_xgmi_sdma_engines(dqm->dev); in get_num_all_sdma_engines()
119 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) in get_num_sdma_queues() argument
121 return kfd_get_num_sdma_engines(dqm->dev) * in get_num_sdma_queues()
122 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; in get_num_sdma_queues()
125 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) in get_num_xgmi_sdma_queues() argument
127 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * in get_num_xgmi_sdma_queues()
128 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; in get_num_xgmi_sdma_queues()
131 static void init_sdma_bitmaps(struct device_queue_manager *dqm) in init_sdma_bitmaps() argument
133 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); in init_sdma_bitmaps()
134 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); in init_sdma_bitmaps()
136 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); in init_sdma_bitmaps()
137 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); in init_sdma_bitmaps()
140 bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap, in init_sdma_bitmaps()
141 dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap, in init_sdma_bitmaps()
145 void program_sh_mem_settings(struct device_queue_manager *dqm, in program_sh_mem_settings() argument
148 uint32_t xcc_mask = dqm->dev->xcc_mask; in program_sh_mem_settings()
152 dqm->dev->kfd2kgd->program_sh_mem_settings( in program_sh_mem_settings()
153 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, in program_sh_mem_settings()
158 static void kfd_hws_hang(struct device_queue_manager *dqm) in kfd_hws_hang() argument
165 list_for_each_entry(cur, &dqm->queues, list) { in kfd_hws_hang()
177 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); in kfd_hws_hang()
200 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, in add_queue_mes() argument
203 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; in add_queue_mes()
209 if (!dqm->sched_running || dqm->sched_halt) in add_queue_mes()
266 kfd_hws_hang(dqm); in add_queue_mes()
272 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, in remove_queue_mes() argument
275 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; in remove_queue_mes()
279 if (!dqm->sched_running || dqm->sched_halt) in remove_queue_mes()
297 kfd_hws_hang(dqm); in remove_queue_mes()
303 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm) in remove_all_kfd_queues_mes() argument
306 struct device *dev = dqm->dev->adev->dev; in remove_all_kfd_queues_mes()
311 list_for_each_entry(cur, &dqm->queues, list) { in remove_all_kfd_queues_mes()
315 retval = remove_queue_mes(dqm, q, qpd); in remove_all_kfd_queues_mes()
320 dqm->dev->id); in remove_all_kfd_queues_mes()
330 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) in add_all_kfd_queues_mes() argument
333 struct device *dev = dqm->dev->adev->dev; in add_all_kfd_queues_mes()
338 list_for_each_entry(cur, &dqm->queues, list) { in add_all_kfd_queues_mes()
343 retval = add_queue_mes(dqm, q, qpd); in add_all_kfd_queues_mes()
348 dqm->dev->id); in add_all_kfd_queues_mes()
357 static int suspend_all_queues_mes(struct device_queue_manager *dqm) in suspend_all_queues_mes() argument
359 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; in suspend_all_queues_mes()
371 kfd_hws_hang(dqm); in suspend_all_queues_mes()
377 static int resume_all_queues_mes(struct device_queue_manager *dqm) in resume_all_queues_mes() argument
379 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; in resume_all_queues_mes()
391 kfd_hws_hang(dqm); in resume_all_queues_mes()
397 static void increment_queue_count(struct device_queue_manager *dqm, in increment_queue_count() argument
401 dqm->active_queue_count++; in increment_queue_count()
404 dqm->active_cp_queue_count++; in increment_queue_count()
407 dqm->gws_queue_count++; in increment_queue_count()
412 static void decrement_queue_count(struct device_queue_manager *dqm, in decrement_queue_count() argument
416 dqm->active_queue_count--; in decrement_queue_count()
419 dqm->active_cp_queue_count--; in decrement_queue_count()
422 dqm->gws_queue_count--; in decrement_queue_count()
435 struct kfd_node *dev = qpd->dqm->dev; in allocate_doorbell()
462 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * in allocate_doorbell()
463 get_num_all_sdma_engines(qpd->dqm) + in allocate_doorbell()
506 struct kfd_node *dev = qpd->dqm->dev; in deallocate_doorbell()
517 static void program_trap_handler_settings(struct device_queue_manager *dqm, in program_trap_handler_settings() argument
520 uint32_t xcc_mask = dqm->dev->xcc_mask; in program_trap_handler_settings()
523 if (dqm->dev->kfd2kgd->program_trap_handler_settings) in program_trap_handler_settings()
525 dqm->dev->kfd2kgd->program_trap_handler_settings( in program_trap_handler_settings()
526 dqm->dev->adev, qpd->vmid, qpd->tba_addr, in program_trap_handler_settings()
530 static int allocate_vmid(struct device_queue_manager *dqm, in allocate_vmid() argument
535 struct device *dev = dqm->dev->adev->dev; in allocate_vmid()
538 for (i = dqm->dev->vm_info.first_vmid_kfd; in allocate_vmid()
539 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { in allocate_vmid()
540 if (!dqm->vmid_pasid[i]) { in allocate_vmid()
553 dqm->vmid_pasid[allocated_vmid] = pdd->pasid; in allocate_vmid()
555 set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid); in allocate_vmid()
560 program_sh_mem_settings(dqm, qpd); in allocate_vmid()
562 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) in allocate_vmid()
563 program_trap_handler_settings(dqm, qpd); in allocate_vmid()
568 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, in allocate_vmid()
574 if (dqm->dev->kfd2kgd->set_scratch_backing_va) in allocate_vmid()
575 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, in allocate_vmid()
584 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; in flush_texture_cache_nocpsch()
599 static void deallocate_vmid(struct device_queue_manager *dqm, in deallocate_vmid() argument
603 struct device *dev = dqm->dev->adev->dev; in deallocate_vmid()
613 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); in deallocate_vmid()
614 dqm->vmid_pasid[qpd->vmid] = 0; in deallocate_vmid()
620 static int create_queue_nocpsch(struct device_queue_manager *dqm, in create_queue_nocpsch() argument
629 dqm_lock(dqm); in create_queue_nocpsch()
631 if (dqm->total_queue_count >= max_num_of_queues_per_device) { in create_queue_nocpsch()
633 dqm->total_queue_count); in create_queue_nocpsch()
639 retval = allocate_vmid(dqm, qpd, q); in create_queue_nocpsch()
654 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in create_queue_nocpsch()
657 retval = allocate_hqd(dqm, q); in create_queue_nocpsch()
664 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); in create_queue_nocpsch()
667 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); in create_queue_nocpsch()
675 dqm_unlock(dqm); in create_queue_nocpsch()
677 dqm_lock(dqm); in create_queue_nocpsch()
693 if (!dqm->sched_running) { in create_queue_nocpsch()
712 increment_queue_count(dqm, qpd, q); in create_queue_nocpsch()
718 dqm->total_queue_count++; in create_queue_nocpsch()
720 dqm->total_queue_count); in create_queue_nocpsch()
729 deallocate_hqd(dqm, q); in create_queue_nocpsch()
732 deallocate_sdma_queue(dqm, q); in create_queue_nocpsch()
735 deallocate_vmid(dqm, qpd, q); in create_queue_nocpsch()
737 dqm_unlock(dqm); in create_queue_nocpsch()
741 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) in allocate_hqd() argument
748 for (pipe = dqm->next_pipe_to_allocate, i = 0; in allocate_hqd()
749 i < get_pipes_per_mec(dqm); in allocate_hqd()
750 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { in allocate_hqd()
752 if (!is_pipe_enabled(dqm, 0, pipe)) in allocate_hqd()
755 if (dqm->allocated_queues[pipe] != 0) { in allocate_hqd()
756 bit = ffs(dqm->allocated_queues[pipe]) - 1; in allocate_hqd()
757 dqm->allocated_queues[pipe] &= ~(1 << bit); in allocate_hqd()
770 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); in allocate_hqd()
775 static inline void deallocate_hqd(struct device_queue_manager *dqm, in deallocate_hqd() argument
778 dqm->allocated_queues[q->pipe] |= (1 << q->queue); in deallocate_hqd()
852 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, in destroy_queue_nocpsch_locked() argument
859 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in destroy_queue_nocpsch_locked()
863 deallocate_hqd(dqm, q); in destroy_queue_nocpsch_locked()
865 deallocate_sdma_queue(dqm, q); in destroy_queue_nocpsch_locked()
867 deallocate_sdma_queue(dqm, q); in destroy_queue_nocpsch_locked()
873 dqm->total_queue_count--; in destroy_queue_nocpsch_locked()
877 if (!dqm->sched_running) { in destroy_queue_nocpsch_locked()
893 dqm->dev); in destroy_queue_nocpsch_locked()
897 dbgdev_wave_reset_wavefronts(dqm->dev, in destroy_queue_nocpsch_locked()
902 deallocate_vmid(dqm, qpd, q); in destroy_queue_nocpsch_locked()
906 decrement_queue_count(dqm, qpd, q); in destroy_queue_nocpsch_locked()
911 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, in destroy_queue_nocpsch() argument
917 struct device *dev = dqm->dev->adev->dev; in destroy_queue_nocpsch()
920 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; in destroy_queue_nocpsch()
932 dqm_lock(dqm); in destroy_queue_nocpsch()
933 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); in destroy_queue_nocpsch()
936 dqm_unlock(dqm); in destroy_queue_nocpsch()
943 static int update_queue(struct device_queue_manager *dqm, struct queue *q, in update_queue() argument
947 struct device *dev = dqm->dev->adev->dev; in update_queue()
952 dqm_lock(dqm); in update_queue()
958 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in update_queue()
965 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { in update_queue()
966 if (!dqm->dev->kfd->shared_resources.enable_mes) in update_queue()
967 retval = unmap_queues_cpsch(dqm, in update_queue()
970 retval = remove_queue_mes(dqm, q, &pdd->qpd); in update_queue()
987 if (!dqm->sched_running) { in update_queue()
993 (dqm->dev->kfd->cwsr_enabled ? in update_queue()
1012 increment_queue_count(dqm, &pdd->qpd, q); in update_queue()
1014 decrement_queue_count(dqm, &pdd->qpd, q); in update_queue()
1017 dqm->gws_queue_count++; in update_queue()
1023 dqm->gws_queue_count--; in update_queue()
1029 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { in update_queue()
1030 if (!dqm->dev->kfd->shared_resources.enable_mes) in update_queue()
1031 retval = map_queues_cpsch(dqm); in update_queue()
1033 retval = add_queue_mes(dqm, q, &pdd->qpd); in update_queue()
1048 dqm_unlock(dqm); in update_queue()
1060 static int suspend_single_queue(struct device_queue_manager *dqm, in suspend_single_queue() argument
1084 if (dqm->dev->kfd->shared_resources.enable_mes) { in suspend_single_queue()
1085 int r = remove_queue_mes(dqm, q, &pdd->qpd); in suspend_single_queue()
1091 decrement_queue_count(dqm, &pdd->qpd, q); in suspend_single_queue()
1106 static int resume_single_queue(struct device_queue_manager *dqm, in resume_single_queue() argument
1124 if (dqm->dev->kfd->shared_resources.enable_mes) { in resume_single_queue()
1125 int r = add_queue_mes(dqm, q, &pdd->qpd); in resume_single_queue()
1132 increment_queue_count(dqm, qpd, q); in resume_single_queue()
1138 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, in evict_process_queues_nocpsch() argument
1146 dqm_lock(dqm); in evict_process_queues_nocpsch()
1163 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in evict_process_queues_nocpsch()
1166 decrement_queue_count(dqm, qpd, q); in evict_process_queues_nocpsch()
1168 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) in evict_process_queues_nocpsch()
1172 (dqm->dev->kfd->cwsr_enabled ? in evict_process_queues_nocpsch()
1184 dqm_unlock(dqm); in evict_process_queues_nocpsch()
1188 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, in evict_process_queues_cpsch() argument
1192 struct device *dev = dqm->dev->adev->dev; in evict_process_queues_cpsch()
1196 dqm_lock(dqm); in evict_process_queues_cpsch()
1221 decrement_queue_count(dqm, qpd, q); in evict_process_queues_cpsch()
1223 if (dqm->dev->kfd->shared_resources.enable_mes) { in evict_process_queues_cpsch()
1226 err = remove_queue_mes(dqm, q, qpd); in evict_process_queues_cpsch()
1235 if (!dqm->dev->kfd->shared_resources.enable_mes) in evict_process_queues_cpsch()
1236 retval = execute_queues_cpsch(dqm, in evict_process_queues_cpsch()
1243 dqm_unlock(dqm); in evict_process_queues_cpsch()
1247 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, in restore_process_queues_nocpsch() argument
1262 dqm_lock(dqm); in restore_process_queues_nocpsch()
1278 dqm->dev->kfd2kgd->set_vm_context_page_table_base( in restore_process_queues_nocpsch()
1279 dqm->dev->adev, in restore_process_queues_nocpsch()
1302 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in restore_process_queues_nocpsch()
1305 increment_queue_count(dqm, qpd, q); in restore_process_queues_nocpsch()
1307 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) in restore_process_queues_nocpsch()
1324 dqm_unlock(dqm); in restore_process_queues_nocpsch()
1328 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, in restore_process_queues_cpsch() argument
1332 struct device *dev = dqm->dev->adev->dev; in restore_process_queues_cpsch()
1339 dqm_lock(dqm); in restore_process_queues_cpsch()
1368 increment_queue_count(dqm, &pdd->qpd, q); in restore_process_queues_cpsch()
1370 if (dqm->dev->kfd->shared_resources.enable_mes) { in restore_process_queues_cpsch()
1371 retval = add_queue_mes(dqm, q, qpd); in restore_process_queues_cpsch()
1379 if (!dqm->dev->kfd->shared_resources.enable_mes) in restore_process_queues_cpsch()
1380 retval = execute_queues_cpsch(dqm, in restore_process_queues_cpsch()
1387 dqm_unlock(dqm); in restore_process_queues_cpsch()
1391 static int register_process(struct device_queue_manager *dqm, in register_process() argument
1409 dqm_lock(dqm); in register_process()
1410 list_add(&n->list, &dqm->queues); in register_process()
1416 retval = dqm->asic_ops.update_qpd(dqm, qpd); in register_process()
1418 dqm->processes_count++; in register_process()
1420 dqm_unlock(dqm); in register_process()
1425 kfd_inc_compute_active(dqm->dev); in register_process()
1430 static int unregister_process(struct device_queue_manager *dqm, in unregister_process() argument
1440 dqm_lock(dqm); in unregister_process()
1442 list_for_each_entry_safe(cur, next, &dqm->queues, list) { in unregister_process()
1446 dqm->processes_count--; in unregister_process()
1453 dqm_unlock(dqm); in unregister_process()
1459 kfd_dec_compute_active(dqm->dev); in unregister_process()
1465 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, in set_pasid_vmid_mapping() argument
1468 uint32_t xcc_mask = dqm->dev->xcc_mask; in set_pasid_vmid_mapping()
1472 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( in set_pasid_vmid_mapping()
1473 dqm->dev->adev, pasid, vmid, xcc_id); in set_pasid_vmid_mapping()
1481 static void init_interrupts(struct device_queue_manager *dqm) in init_interrupts() argument
1483 uint32_t xcc_mask = dqm->dev->xcc_mask; in init_interrupts()
1487 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { in init_interrupts()
1488 if (is_pipe_enabled(dqm, 0, i)) { in init_interrupts()
1489 dqm->dev->kfd2kgd->init_interrupts( in init_interrupts()
1490 dqm->dev->adev, i, xcc_id); in init_interrupts()
1496 static int initialize_nocpsch(struct device_queue_manager *dqm) in initialize_nocpsch() argument
1500 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); in initialize_nocpsch()
1502 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), in initialize_nocpsch()
1504 if (!dqm->allocated_queues) in initialize_nocpsch()
1507 mutex_init(&dqm->lock_hidden); in initialize_nocpsch()
1508 INIT_LIST_HEAD(&dqm->queues); in initialize_nocpsch()
1509 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; in initialize_nocpsch()
1510 dqm->active_cp_queue_count = 0; in initialize_nocpsch()
1511 dqm->gws_queue_count = 0; in initialize_nocpsch()
1513 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { in initialize_nocpsch()
1514 int pipe_offset = pipe * get_queues_per_pipe(dqm); in initialize_nocpsch()
1516 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) in initialize_nocpsch()
1518 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) in initialize_nocpsch()
1519 dqm->allocated_queues[pipe] |= 1 << queue; in initialize_nocpsch()
1522 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); in initialize_nocpsch()
1524 init_sdma_bitmaps(dqm); in initialize_nocpsch()
1529 static void uninitialize(struct device_queue_manager *dqm) in uninitialize() argument
1533 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); in uninitialize()
1535 kfree(dqm->allocated_queues); in uninitialize()
1537 kfree(dqm->mqd_mgrs[i]); in uninitialize()
1538 mutex_destroy(&dqm->lock_hidden); in uninitialize()
1541 static int start_nocpsch(struct device_queue_manager *dqm) in start_nocpsch() argument
1546 init_interrupts(dqm); in start_nocpsch()
1548 if (dqm->dev->adev->asic_type == CHIP_HAWAII) in start_nocpsch()
1549 r = pm_init(&dqm->packet_mgr, dqm); in start_nocpsch()
1551 dqm->sched_running = true; in start_nocpsch()
1556 static int stop_nocpsch(struct device_queue_manager *dqm) in stop_nocpsch() argument
1558 dqm_lock(dqm); in stop_nocpsch()
1559 if (!dqm->sched_running) { in stop_nocpsch()
1560 dqm_unlock(dqm); in stop_nocpsch()
1564 if (dqm->dev->adev->asic_type == CHIP_HAWAII) in stop_nocpsch()
1565 pm_uninit(&dqm->packet_mgr); in stop_nocpsch()
1566 dqm->sched_running = false; in stop_nocpsch()
1567 dqm_unlock(dqm); in stop_nocpsch()
1572 static int allocate_sdma_queue(struct device_queue_manager *dqm, in allocate_sdma_queue() argument
1575 struct device *dev = dqm->dev->adev->dev; in allocate_sdma_queue()
1579 if (bitmap_empty(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES)) { in allocate_sdma_queue()
1586 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { in allocate_sdma_queue()
1590 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); in allocate_sdma_queue()
1594 bit = find_first_bit(dqm->sdma_bitmap, in allocate_sdma_queue()
1595 get_num_sdma_queues(dqm)); in allocate_sdma_queue()
1596 clear_bit(bit, dqm->sdma_bitmap); in allocate_sdma_queue()
1601 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); in allocate_sdma_queue()
1603 kfd_get_num_sdma_engines(dqm->dev); in allocate_sdma_queue()
1605 if (bitmap_empty(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES)) { in allocate_sdma_queue()
1611 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { in allocate_sdma_queue()
1615 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); in allocate_sdma_queue()
1618 bit = find_first_bit(dqm->xgmi_sdma_bitmap, in allocate_sdma_queue()
1619 get_num_xgmi_sdma_queues(dqm)); in allocate_sdma_queue()
1620 clear_bit(bit, dqm->xgmi_sdma_bitmap); in allocate_sdma_queue()
1630 kfd_get_num_sdma_engines(dqm->dev) + in allocate_sdma_queue()
1631 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); in allocate_sdma_queue()
1633 kfd_get_num_xgmi_sdma_engines(dqm->dev); in allocate_sdma_queue()
1638 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) { in allocate_sdma_queue()
1639 num_queues = get_num_sdma_queues(dqm); in allocate_sdma_queue()
1640 num_engines = kfd_get_num_sdma_engines(dqm->dev); in allocate_sdma_queue()
1643 num_queues = get_num_xgmi_sdma_queues(dqm); in allocate_sdma_queue()
1644 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev); in allocate_sdma_queue()
1645 eng_offset = kfd_get_num_sdma_engines(dqm->dev); in allocate_sdma_queue()
1654 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap)) in allocate_sdma_queue()
1657 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap); in allocate_sdma_queue()
1677 static void deallocate_sdma_queue(struct device_queue_manager *dqm, in deallocate_sdma_queue() argument
1681 if (q->sdma_id >= get_num_sdma_queues(dqm)) in deallocate_sdma_queue()
1683 set_bit(q->sdma_id, dqm->sdma_bitmap); in deallocate_sdma_queue()
1685 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) in deallocate_sdma_queue()
1687 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); in deallocate_sdma_queue()
1695 static int set_sched_resources(struct device_queue_manager *dqm) in set_sched_resources() argument
1699 struct device *dev = dqm->dev->adev->dev; in set_sched_resources()
1701 res.vmid_mask = dqm->dev->compute_vmid_bitmap; in set_sched_resources()
1705 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) in set_sched_resources()
1706 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; in set_sched_resources()
1708 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) in set_sched_resources()
1726 dqm->dev->adev, i); in set_sched_resources()
1736 return pm_send_set_resources(&dqm->packet_mgr, &res); in set_sched_resources()
1739 static int initialize_cpsch(struct device_queue_manager *dqm) in initialize_cpsch() argument
1741 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); in initialize_cpsch()
1743 mutex_init(&dqm->lock_hidden); in initialize_cpsch()
1744 INIT_LIST_HEAD(&dqm->queues); in initialize_cpsch()
1745 dqm->active_queue_count = dqm->processes_count = 0; in initialize_cpsch()
1746 dqm->active_cp_queue_count = 0; in initialize_cpsch()
1747 dqm->gws_queue_count = 0; in initialize_cpsch()
1748 dqm->active_runlist = false; in initialize_cpsch()
1749 dqm->trap_debug_vmid = 0; in initialize_cpsch()
1751 init_sdma_bitmaps(dqm); in initialize_cpsch()
1753 update_dqm_wait_times(dqm); in initialize_cpsch()
1762 static int halt_cpsch(struct device_queue_manager *dqm) in halt_cpsch() argument
1766 dqm_lock(dqm); in halt_cpsch()
1767 if (!dqm->sched_running) { in halt_cpsch()
1768 dqm_unlock(dqm); in halt_cpsch()
1772 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n"); in halt_cpsch()
1774 if (!dqm->is_hws_hang) { in halt_cpsch()
1775 if (!dqm->dev->kfd->shared_resources.enable_mes) in halt_cpsch()
1776 ret = unmap_queues_cpsch(dqm, in halt_cpsch()
1780 ret = remove_all_kfd_queues_mes(dqm); in halt_cpsch()
1782 dqm->sched_halt = true; in halt_cpsch()
1783 dqm_unlock(dqm); in halt_cpsch()
1791 static int unhalt_cpsch(struct device_queue_manager *dqm) in unhalt_cpsch() argument
1795 dqm_lock(dqm); in unhalt_cpsch()
1796 if (!dqm->sched_running || !dqm->sched_halt) { in unhalt_cpsch()
1797 WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n"); in unhalt_cpsch()
1798 dqm_unlock(dqm); in unhalt_cpsch()
1801 dqm->sched_halt = false; in unhalt_cpsch()
1802 if (!dqm->dev->kfd->shared_resources.enable_mes) in unhalt_cpsch()
1803 ret = execute_queues_cpsch(dqm, in unhalt_cpsch()
1807 ret = add_all_kfd_queues_mes(dqm); in unhalt_cpsch()
1809 dqm_unlock(dqm); in unhalt_cpsch()
1814 static int start_cpsch(struct device_queue_manager *dqm) in start_cpsch() argument
1816 struct device *dev = dqm->dev->adev->dev; in start_cpsch()
1821 dqm_lock(dqm); in start_cpsch()
1823 if (!dqm->dev->kfd->shared_resources.enable_mes) { in start_cpsch()
1824 retval = pm_init(&dqm->packet_mgr, dqm); in start_cpsch()
1828 retval = set_sched_resources(dqm); in start_cpsch()
1835 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), in start_cpsch()
1836 &dqm->fence_mem); in start_cpsch()
1841 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; in start_cpsch()
1842 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; in start_cpsch()
1844 init_interrupts(dqm); in start_cpsch()
1847 dqm->sched_running = true; in start_cpsch()
1849 if (!dqm->dev->kfd->shared_resources.enable_mes) { in start_cpsch()
1850 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, in start_cpsch()
1853 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); in start_cpsch()
1857 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * in start_cpsch()
1858 dqm->dev->kfd->shared_resources.num_pipe_per_mec * in start_cpsch()
1859 NUM_XCC(dqm->dev->xcc_mask); in start_cpsch()
1861 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info); in start_cpsch()
1862 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL); in start_cpsch()
1864 if (!dqm->detect_hang_info) { in start_cpsch()
1869 dqm_unlock(dqm); in start_cpsch()
1873 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); in start_cpsch()
1876 if (!dqm->dev->kfd->shared_resources.enable_mes) in start_cpsch()
1877 pm_uninit(&dqm->packet_mgr); in start_cpsch()
1879 dqm_unlock(dqm); in start_cpsch()
1883 static int stop_cpsch(struct device_queue_manager *dqm) in stop_cpsch() argument
1885 dqm_lock(dqm); in stop_cpsch()
1886 if (!dqm->sched_running) { in stop_cpsch()
1887 dqm_unlock(dqm); in stop_cpsch()
1891 if (!dqm->dev->kfd->shared_resources.enable_mes) in stop_cpsch()
1892 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); in stop_cpsch()
1894 remove_all_kfd_queues_mes(dqm); in stop_cpsch()
1896 dqm->sched_running = false; in stop_cpsch()
1898 if (!dqm->dev->kfd->shared_resources.enable_mes) in stop_cpsch()
1899 pm_release_ib(&dqm->packet_mgr); in stop_cpsch()
1901 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); in stop_cpsch()
1902 if (!dqm->dev->kfd->shared_resources.enable_mes) in stop_cpsch()
1903 pm_uninit(&dqm->packet_mgr); in stop_cpsch()
1904 kfree(dqm->detect_hang_info); in stop_cpsch()
1905 dqm->detect_hang_info = NULL; in stop_cpsch()
1906 dqm_unlock(dqm); in stop_cpsch()
1911 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, in create_kernel_queue_cpsch() argument
1915 dqm_lock(dqm); in create_kernel_queue_cpsch()
1916 if (dqm->total_queue_count >= max_num_of_queues_per_device) { in create_kernel_queue_cpsch()
1918 dqm->total_queue_count); in create_kernel_queue_cpsch()
1919 dqm_unlock(dqm); in create_kernel_queue_cpsch()
1927 dqm->total_queue_count++; in create_kernel_queue_cpsch()
1929 dqm->total_queue_count); in create_kernel_queue_cpsch()
1932 increment_queue_count(dqm, qpd, kq->queue); in create_kernel_queue_cpsch()
1934 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, in create_kernel_queue_cpsch()
1936 dqm_unlock(dqm); in create_kernel_queue_cpsch()
1941 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, in destroy_kernel_queue_cpsch() argument
1945 dqm_lock(dqm); in destroy_kernel_queue_cpsch()
1947 decrement_queue_count(dqm, qpd, kq->queue); in destroy_kernel_queue_cpsch()
1949 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, in destroy_kernel_queue_cpsch()
1955 dqm->total_queue_count--; in destroy_kernel_queue_cpsch()
1957 dqm->total_queue_count); in destroy_kernel_queue_cpsch()
1958 dqm_unlock(dqm); in destroy_kernel_queue_cpsch()
1961 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, in create_queue_cpsch() argument
1969 if (dqm->total_queue_count >= max_num_of_queues_per_device) { in create_queue_cpsch()
1971 dqm->total_queue_count); in create_queue_cpsch()
1979 dqm_lock(dqm); in create_queue_cpsch()
1980 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); in create_queue_cpsch()
1981 dqm_unlock(dqm); in create_queue_cpsch()
1990 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in create_queue_cpsch()
1995 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); in create_queue_cpsch()
2004 dqm_lock(dqm); in create_queue_cpsch()
2026 increment_queue_count(dqm, qpd, q); in create_queue_cpsch()
2028 if (!dqm->dev->kfd->shared_resources.enable_mes) in create_queue_cpsch()
2029 retval = execute_queues_cpsch(dqm, in create_queue_cpsch()
2032 retval = add_queue_mes(dqm, q, qpd); in create_queue_cpsch()
2041 dqm->total_queue_count++; in create_queue_cpsch()
2044 dqm->total_queue_count); in create_queue_cpsch()
2046 dqm_unlock(dqm); in create_queue_cpsch()
2053 decrement_queue_count(dqm, qpd, q); in create_queue_cpsch()
2055 dqm_unlock(dqm); in create_queue_cpsch()
2061 dqm_lock(dqm); in create_queue_cpsch()
2062 deallocate_sdma_queue(dqm, q); in create_queue_cpsch()
2063 dqm_unlock(dqm); in create_queue_cpsch()
2069 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, in amdkfd_fence_wait_timeout() argument
2074 struct device *dev = dqm->dev->adev->dev; in amdkfd_fence_wait_timeout()
2075 uint64_t *fence_addr = dqm->fence_addr; in amdkfd_fence_wait_timeout()
2079 if (amdgpu_amdkfd_is_fed(dqm->dev->adev)) in amdkfd_fence_wait_timeout()
2100 static int map_queues_cpsch(struct device_queue_manager *dqm) in map_queues_cpsch() argument
2102 struct device *dev = dqm->dev->adev->dev; in map_queues_cpsch()
2105 if (!dqm->sched_running || dqm->sched_halt) in map_queues_cpsch()
2107 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) in map_queues_cpsch()
2109 if (dqm->active_runlist) in map_queues_cpsch()
2112 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); in map_queues_cpsch()
2118 dqm->active_runlist = true; in map_queues_cpsch()
2123 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, in set_queue_as_reset() argument
2128 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n", in set_queue_as_reset()
2134 decrement_queue_count(dqm, qpd, q); in set_queue_as_reset()
2138 static int detect_queue_hang(struct device_queue_manager *dqm) in detect_queue_hang() argument
2143 if (WARN_ON(dqm->detect_hang_count > 0)) in detect_queue_hang()
2146 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size); in detect_queue_hang()
2152 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) in detect_queue_hang()
2153 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; in detect_queue_hang()
2155 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) in detect_queue_hang()
2158 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue); in detect_queue_hang()
2160 for_each_inst(xcc_id, dqm->dev->xcc_mask) { in detect_queue_hang()
2161 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr( in detect_queue_hang()
2162 dqm->dev->adev, pipe, queue, xcc_id); in detect_queue_hang()
2173 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; in detect_queue_hang()
2174 dqm->detect_hang_count++; in detect_queue_hang()
2178 return dqm->detect_hang_count; in detect_queue_hang()
2181 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address) in find_queue_by_address() argument
2187 list_for_each_entry(cur, &dqm->queues, list) { in find_queue_by_address()
2198 static int reset_hung_queues(struct device_queue_manager *dqm) in reset_hung_queues() argument
2202 if (!dqm->detect_hang_info || dqm->is_hws_hang) in reset_hung_queues()
2206 if (!detect_queue_hang(dqm)) in reset_hung_queues()
2209 for (i = 0; i < dqm->detect_hang_count; i++) { in reset_hung_queues()
2210 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; in reset_hung_queues()
2211 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address); in reset_hung_queues()
2220 pdd = kfd_get_process_device_data(dqm->dev, q->process); in reset_hung_queues()
2226 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev, in reset_hung_queues()
2236 set_queue_as_reset(dqm, q, &pdd->qpd); in reset_hung_queues()
2240 if (reset_count == dqm->detect_hang_count) in reset_hung_queues()
2241 kfd_signal_reset_event(dqm->dev); in reset_hung_queues()
2246 dqm->detect_hang_count = 0; in reset_hung_queues()
2251 static bool sdma_has_hang(struct device_queue_manager *dqm) in sdma_has_hang() argument
2253 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); in sdma_has_hang()
2254 int engine_end = engine_start + get_num_all_sdma_engines(dqm); in sdma_has_hang()
2255 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; in sdma_has_hang()
2260 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j)) in sdma_has_hang()
2270 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm, in set_sdma_queue_as_reset() argument
2277 list_for_each_entry(cur, &dqm->queues, list) { in set_sdma_queue_as_reset()
2283 set_queue_as_reset(dqm, q, qpd); in set_sdma_queue_as_reset()
2292 static int reset_hung_queues_sdma(struct device_queue_manager *dqm) in reset_hung_queues_sdma() argument
2294 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); in reset_hung_queues_sdma()
2295 int engine_end = engine_start + get_num_all_sdma_engines(dqm); in reset_hung_queues_sdma()
2296 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; in reset_hung_queues_sdma()
2299 if (dqm->is_hws_hang) in reset_hung_queues_sdma()
2303 dqm->detect_hang_count = 0; in reset_hung_queues_sdma()
2307 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j); in reset_hung_queues_sdma()
2313 if (amdgpu_sdma_reset_engine(dqm->dev->adev, i) || in reset_hung_queues_sdma()
2314 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) || in reset_hung_queues_sdma()
2315 !set_sdma_queue_as_reset(dqm, doorbell_off)) { in reset_hung_queues_sdma()
2321 dqm->detect_hang_count++; in reset_hung_queues_sdma()
2327 if (dqm->detect_hang_count) in reset_hung_queues_sdma()
2328 kfd_signal_reset_event(dqm->dev); in reset_hung_queues_sdma()
2333 dqm->detect_hang_count = 0; in reset_hung_queues_sdma()
2338 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma) in reset_queues_on_hws_hang() argument
2346 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm); in reset_queues_on_hws_hang()
2357 static int unmap_queues_cpsch(struct device_queue_manager *dqm, in unmap_queues_cpsch() argument
2363 struct device *dev = dqm->dev->adev->dev; in unmap_queues_cpsch()
2367 if (!dqm->sched_running) in unmap_queues_cpsch()
2369 if (!dqm->active_runlist) in unmap_queues_cpsch()
2371 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) in unmap_queues_cpsch()
2375 retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr, in unmap_queues_cpsch()
2381 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); in unmap_queues_cpsch()
2385 *dqm->fence_addr = KFD_FENCE_INIT; in unmap_queues_cpsch()
2387 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, in unmap_queues_cpsch()
2390 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, in unmap_queues_cpsch()
2394 kfd_hws_hang(dqm); in unmap_queues_cpsch()
2406 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; in unmap_queues_cpsch()
2407 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) && in unmap_queues_cpsch()
2408 reset_queues_on_hws_hang(dqm, false)) in unmap_queues_cpsch()
2412 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true)) in unmap_queues_cpsch()
2417 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, in unmap_queues_cpsch()
2422 pm_release_ib(&dqm->packet_mgr); in unmap_queues_cpsch()
2423 dqm->active_runlist = false; in unmap_queues_cpsch()
2425 up_read(&dqm->dev->adev->reset_domain->sem); in unmap_queues_cpsch()
2429 dqm->is_hws_hang = true; in unmap_queues_cpsch()
2430 kfd_hws_hang(dqm); in unmap_queues_cpsch()
2431 up_read(&dqm->dev->adev->reset_domain->sem); in unmap_queues_cpsch()
2436 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid) in reset_queues_cpsch() argument
2440 dqm_lock(dqm); in reset_queues_cpsch()
2442 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, in reset_queues_cpsch()
2445 dqm_unlock(dqm); in reset_queues_cpsch()
2450 static int execute_queues_cpsch(struct device_queue_manager *dqm, in execute_queues_cpsch() argument
2457 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) in execute_queues_cpsch()
2459 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); in execute_queues_cpsch()
2461 retval = map_queues_cpsch(dqm); in execute_queues_cpsch()
2462 up_read(&dqm->dev->adev->reset_domain->sem); in execute_queues_cpsch()
2466 static int wait_on_destroy_queue(struct device_queue_manager *dqm, in wait_on_destroy_queue() argument
2482 dqm_unlock(dqm); in wait_on_destroy_queue()
2484 ret = wait_event_interruptible(dqm->destroy_wait, in wait_on_destroy_queue()
2488 dqm_lock(dqm); in wait_on_destroy_queue()
2494 static int destroy_queue_cpsch(struct device_queue_manager *dqm, in destroy_queue_cpsch() argument
2502 struct device *dev = dqm->dev->adev->dev; in destroy_queue_cpsch()
2515 dqm_lock(dqm); in destroy_queue_cpsch()
2517 retval = wait_on_destroy_queue(dqm, q); in destroy_queue_cpsch()
2520 dqm_unlock(dqm); in destroy_queue_cpsch()
2534 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in destroy_queue_cpsch()
2541 deallocate_sdma_queue(dqm, q); in destroy_queue_cpsch()
2546 decrement_queue_count(dqm, qpd, q); in destroy_queue_cpsch()
2548 if (!dqm->dev->kfd->shared_resources.enable_mes) { in destroy_queue_cpsch()
2549 retval = execute_queues_cpsch(dqm, in destroy_queue_cpsch()
2555 retval = remove_queue_mes(dqm, q, qpd); in destroy_queue_cpsch()
2565 dqm->total_queue_count--; in destroy_queue_cpsch()
2567 dqm->total_queue_count); in destroy_queue_cpsch()
2569 dqm_unlock(dqm); in destroy_queue_cpsch()
2585 dqm_unlock(dqm); in destroy_queue_cpsch()
2589 static bool set_cache_memory_policy(struct device_queue_manager *dqm, in set_cache_memory_policy() argument
2599 if (!dqm->asic_ops.set_cache_memory_policy) in set_cache_memory_policy()
2602 dqm_lock(dqm); in set_cache_memory_policy()
2604 retval = dqm->asic_ops.set_cache_memory_policy( in set_cache_memory_policy()
2605 dqm, in set_cache_memory_policy()
2616 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) in set_cache_memory_policy()
2617 program_sh_mem_settings(dqm, qpd); in set_cache_memory_policy()
2624 dqm_unlock(dqm); in set_cache_memory_policy()
2628 static int process_termination_nocpsch(struct device_queue_manager *dqm, in process_termination_nocpsch() argument
2636 dqm_lock(dqm); in process_termination_nocpsch()
2644 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in process_termination_nocpsch()
2646 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); in process_termination_nocpsch()
2649 dqm_unlock(dqm); in process_termination_nocpsch()
2651 dqm_lock(dqm); in process_termination_nocpsch()
2655 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { in process_termination_nocpsch()
2659 dqm->processes_count--; in process_termination_nocpsch()
2665 dqm_unlock(dqm); in process_termination_nocpsch()
2671 kfd_dec_compute_active(dqm->dev); in process_termination_nocpsch()
2676 static int get_wave_state(struct device_queue_manager *dqm, in get_wave_state() argument
2684 dqm_lock(dqm); in get_wave_state()
2686 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; in get_wave_state()
2691 dqm_unlock(dqm); in get_wave_state()
2695 dqm_unlock(dqm); in get_wave_state()
2706 static void get_queue_checkpoint_info(struct device_queue_manager *dqm, in get_queue_checkpoint_info() argument
2715 dqm_lock(dqm); in get_queue_checkpoint_info()
2716 mqd_mgr = dqm->mqd_mgrs[mqd_type]; in get_queue_checkpoint_info()
2723 dqm_unlock(dqm); in get_queue_checkpoint_info()
2726 static int checkpoint_mqd(struct device_queue_manager *dqm, in checkpoint_mqd() argument
2736 dqm_lock(dqm); in checkpoint_mqd()
2743 mqd_mgr = dqm->mqd_mgrs[mqd_type]; in checkpoint_mqd()
2752 dqm_unlock(dqm); in checkpoint_mqd()
2756 static int process_termination_cpsch(struct device_queue_manager *dqm, in process_termination_cpsch() argument
2761 struct device *dev = dqm->dev->adev->dev; in process_termination_cpsch()
2771 dqm_lock(dqm); in process_termination_cpsch()
2776 decrement_queue_count(dqm, qpd, kq->queue); in process_termination_cpsch()
2778 dqm->total_queue_count--; in process_termination_cpsch()
2785 deallocate_sdma_queue(dqm, q); in process_termination_cpsch()
2787 deallocate_sdma_queue(dqm, q); in process_termination_cpsch()
2790 decrement_queue_count(dqm, qpd, q); in process_termination_cpsch()
2792 if (dqm->dev->kfd->shared_resources.enable_mes) { in process_termination_cpsch()
2793 retval = remove_queue_mes(dqm, q, qpd); in process_termination_cpsch()
2800 dqm->total_queue_count--; in process_termination_cpsch()
2804 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { in process_termination_cpsch()
2808 dqm->processes_count--; in process_termination_cpsch()
2814 if (!dqm->dev->kfd->shared_resources.enable_mes) in process_termination_cpsch()
2815 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); in process_termination_cpsch()
2818 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { in process_termination_cpsch()
2819 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); in process_termination_cpsch()
2820 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); in process_termination_cpsch()
2822 up_read(&dqm->dev->adev->reset_domain->sem); in process_termination_cpsch()
2830 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( in process_termination_cpsch()
2834 dqm_unlock(dqm); in process_termination_cpsch()
2836 dqm_lock(dqm); in process_termination_cpsch()
2838 dqm_unlock(dqm); in process_termination_cpsch()
2844 kfd_dec_compute_active(dqm->dev); in process_termination_cpsch()
2849 static int init_mqd_managers(struct device_queue_manager *dqm) in init_mqd_managers() argument
2852 struct device *dev = dqm->dev->adev->dev; in init_mqd_managers()
2856 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); in init_mqd_managers()
2861 dqm->mqd_mgrs[i] = mqd_mgr; in init_mqd_managers()
2868 kfree(dqm->mqd_mgrs[j]); in init_mqd_managers()
2869 dqm->mqd_mgrs[j] = NULL; in init_mqd_managers()
2876 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) in allocate_hiq_sdma_mqd() argument
2879 struct kfd_node *dev = dqm->dev; in allocate_hiq_sdma_mqd()
2880 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; in allocate_hiq_sdma_mqd()
2881 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * in allocate_hiq_sdma_mqd()
2882 get_num_all_sdma_engines(dqm) * in allocate_hiq_sdma_mqd()
2884 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * in allocate_hiq_sdma_mqd()
2885 NUM_XCC(dqm->dev->xcc_mask)); in allocate_hiq_sdma_mqd()
2896 struct device_queue_manager *dqm; in device_queue_manager_init() local
2900 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL); in device_queue_manager_init()
2901 if (!dqm) in device_queue_manager_init()
2913 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; in device_queue_manager_init()
2916 dqm->sched_policy = sched_policy; in device_queue_manager_init()
2920 dqm->dev = dev; in device_queue_manager_init()
2921 switch (dqm->sched_policy) { in device_queue_manager_init()
2925 dqm->ops.create_queue = create_queue_cpsch; in device_queue_manager_init()
2926 dqm->ops.initialize = initialize_cpsch; in device_queue_manager_init()
2927 dqm->ops.start = start_cpsch; in device_queue_manager_init()
2928 dqm->ops.stop = stop_cpsch; in device_queue_manager_init()
2929 dqm->ops.halt = halt_cpsch; in device_queue_manager_init()
2930 dqm->ops.unhalt = unhalt_cpsch; in device_queue_manager_init()
2931 dqm->ops.destroy_queue = destroy_queue_cpsch; in device_queue_manager_init()
2932 dqm->ops.update_queue = update_queue; in device_queue_manager_init()
2933 dqm->ops.register_process = register_process; in device_queue_manager_init()
2934 dqm->ops.unregister_process = unregister_process; in device_queue_manager_init()
2935 dqm->ops.uninitialize = uninitialize; in device_queue_manager_init()
2936 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; in device_queue_manager_init()
2937 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; in device_queue_manager_init()
2938 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; in device_queue_manager_init()
2939 dqm->ops.process_termination = process_termination_cpsch; in device_queue_manager_init()
2940 dqm->ops.evict_process_queues = evict_process_queues_cpsch; in device_queue_manager_init()
2941 dqm->ops.restore_process_queues = restore_process_queues_cpsch; in device_queue_manager_init()
2942 dqm->ops.get_wave_state = get_wave_state; in device_queue_manager_init()
2943 dqm->ops.reset_queues = reset_queues_cpsch; in device_queue_manager_init()
2944 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; in device_queue_manager_init()
2945 dqm->ops.checkpoint_mqd = checkpoint_mqd; in device_queue_manager_init()
2949 dqm->ops.start = start_nocpsch; in device_queue_manager_init()
2950 dqm->ops.stop = stop_nocpsch; in device_queue_manager_init()
2951 dqm->ops.create_queue = create_queue_nocpsch; in device_queue_manager_init()
2952 dqm->ops.destroy_queue = destroy_queue_nocpsch; in device_queue_manager_init()
2953 dqm->ops.update_queue = update_queue; in device_queue_manager_init()
2954 dqm->ops.register_process = register_process; in device_queue_manager_init()
2955 dqm->ops.unregister_process = unregister_process; in device_queue_manager_init()
2956 dqm->ops.initialize = initialize_nocpsch; in device_queue_manager_init()
2957 dqm->ops.uninitialize = uninitialize; in device_queue_manager_init()
2958 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; in device_queue_manager_init()
2959 dqm->ops.process_termination = process_termination_nocpsch; in device_queue_manager_init()
2960 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; in device_queue_manager_init()
2961 dqm->ops.restore_process_queues = in device_queue_manager_init()
2963 dqm->ops.get_wave_state = get_wave_state; in device_queue_manager_init()
2964 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; in device_queue_manager_init()
2965 dqm->ops.checkpoint_mqd = checkpoint_mqd; in device_queue_manager_init()
2968 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); in device_queue_manager_init()
2975 device_queue_manager_init_cik(&dqm->asic_ops); in device_queue_manager_init()
2985 device_queue_manager_init_vi(&dqm->asic_ops); in device_queue_manager_init()
2990 device_queue_manager_init_v12(&dqm->asic_ops); in device_queue_manager_init()
2992 device_queue_manager_init_v11(&dqm->asic_ops); in device_queue_manager_init()
2994 device_queue_manager_init_v10(&dqm->asic_ops); in device_queue_manager_init()
2996 device_queue_manager_init_v9(&dqm->asic_ops); in device_queue_manager_init()
3004 if (init_mqd_managers(dqm)) in device_queue_manager_init()
3007 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { in device_queue_manager_init()
3012 if (!dqm->ops.initialize(dqm)) { in device_queue_manager_init()
3013 init_waitqueue_head(&dqm->destroy_wait); in device_queue_manager_init()
3014 return dqm; in device_queue_manager_init()
3018 kfree(dqm); in device_queue_manager_init()
3030 void device_queue_manager_uninit(struct device_queue_manager *dqm) in device_queue_manager_uninit() argument
3032 dqm->ops.stop(dqm); in device_queue_manager_uninit()
3033 dqm->ops.uninitialize(dqm); in device_queue_manager_uninit()
3034 if (!dqm->dev->kfd->shared_resources.enable_mes) in device_queue_manager_uninit()
3035 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); in device_queue_manager_uninit()
3036 kfree(dqm); in device_queue_manager_uninit()
3043 struct device_queue_manager *dqm = knode->dqm; in kfd_dqm_suspend_bad_queue_mes() local
3044 struct device *dev = dqm->dev->adev->dev; in kfd_dqm_suspend_bad_queue_mes()
3052 dqm_lock(dqm); in kfd_dqm_suspend_bad_queue_mes()
3059 ret = suspend_all_queues_mes(dqm); in kfd_dqm_suspend_bad_queue_mes()
3067 decrement_queue_count(dqm, qpd, q); in kfd_dqm_suspend_bad_queue_mes()
3069 ret = remove_queue_mes(dqm, q, qpd); in kfd_dqm_suspend_bad_queue_mes()
3075 ret = resume_all_queues_mes(dqm); in kfd_dqm_suspend_bad_queue_mes()
3085 dqm_unlock(dqm); in kfd_dqm_suspend_bad_queue_mes()
3090 static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm, in kfd_dqm_evict_pasid_mes() argument
3093 struct device *dev = dqm->dev->adev->dev; in kfd_dqm_evict_pasid_mes()
3097 dqm_lock(dqm); in kfd_dqm_evict_pasid_mes()
3103 dqm_unlock(dqm); in kfd_dqm_evict_pasid_mes()
3106 dqm_unlock(dqm); in kfd_dqm_evict_pasid_mes()
3108 ret = suspend_all_queues_mes(dqm); in kfd_dqm_evict_pasid_mes()
3114 ret = dqm->ops.evict_process_queues(dqm, qpd); in kfd_dqm_evict_pasid_mes()
3120 ret = resume_all_queues_mes(dqm); in kfd_dqm_evict_pasid_mes()
3130 struct device_queue_manager *dqm; in kfd_evict_process_device() local
3135 dqm = pdd->dev->dqm; in kfd_evict_process_device()
3139 if (dqm->dev->kfd->shared_resources.enable_mes) in kfd_evict_process_device()
3140 ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd); in kfd_evict_process_device()
3142 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); in kfd_evict_process_device()
3147 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, in reserve_debug_trap_vmid() argument
3151 struct device *dev = dqm->dev->adev->dev; in reserve_debug_trap_vmid()
3154 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { in reserve_debug_trap_vmid()
3155 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); in reserve_debug_trap_vmid()
3159 dqm_lock(dqm); in reserve_debug_trap_vmid()
3161 if (dqm->trap_debug_vmid != 0) { in reserve_debug_trap_vmid()
3167 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, in reserve_debug_trap_vmid()
3172 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; in reserve_debug_trap_vmid()
3173 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); in reserve_debug_trap_vmid()
3175 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; in reserve_debug_trap_vmid()
3176 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; in reserve_debug_trap_vmid()
3177 r = set_sched_resources(dqm); in reserve_debug_trap_vmid()
3181 r = map_queues_cpsch(dqm); in reserve_debug_trap_vmid()
3185 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); in reserve_debug_trap_vmid()
3188 dqm_unlock(dqm); in reserve_debug_trap_vmid()
3195 int release_debug_trap_vmid(struct device_queue_manager *dqm, in release_debug_trap_vmid() argument
3198 struct device *dev = dqm->dev->adev->dev; in release_debug_trap_vmid()
3203 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { in release_debug_trap_vmid()
3204 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); in release_debug_trap_vmid()
3208 dqm_lock(dqm); in release_debug_trap_vmid()
3209 trap_debug_vmid = dqm->trap_debug_vmid; in release_debug_trap_vmid()
3210 if (dqm->trap_debug_vmid == 0) { in release_debug_trap_vmid()
3216 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, in release_debug_trap_vmid()
3221 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; in release_debug_trap_vmid()
3222 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); in release_debug_trap_vmid()
3224 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; in release_debug_trap_vmid()
3225 dqm->trap_debug_vmid = 0; in release_debug_trap_vmid()
3226 r = set_sched_resources(dqm); in release_debug_trap_vmid()
3230 r = map_queues_cpsch(dqm); in release_debug_trap_vmid()
3237 dqm_unlock(dqm); in release_debug_trap_vmid()
3293 struct device_queue_manager *dqm = pdd->dev->dqm; in copy_context_work_handler() local
3300 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; in copy_context_work_handler()
3350 struct device_queue_manager *dqm = pdd->dev->dqm; in resume_queues() local
3351 struct device *dev = dqm->dev->adev->dev; in resume_queues()
3356 dqm_lock(dqm); in resume_queues()
3369 int err = resume_single_queue(dqm, &pdd->qpd, q); in resume_queues()
3382 if (dqm->dev->kfd->shared_resources.enable_mes) { in resume_queues()
3383 wake_up_all(&dqm->destroy_wait); in resume_queues()
3393 dqm_unlock(dqm); in resume_queues()
3397 r = execute_queues_cpsch(dqm, in resume_queues()
3417 wake_up_all(&dqm->destroy_wait); in resume_queues()
3421 dqm_unlock(dqm); in resume_queues()
3453 struct device_queue_manager *dqm = pdd->dev->dqm; in suspend_queues() local
3454 struct device *dev = dqm->dev->adev->dev; in suspend_queues()
3460 dqm_lock(dqm); in suspend_queues()
3469 int err = suspend_single_queue(dqm, pdd, q); in suspend_queues()
3470 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; in suspend_queues()
3491 dqm_unlock(dqm); in suspend_queues()
3494 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); in suspend_queues()
3498 r = execute_queues_cpsch(dqm, in suspend_queues()
3522 dqm_unlock(dqm); in suspend_queues()
3524 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); in suspend_queues()
3588 int debug_lock_and_unmap(struct device_queue_manager *dqm) in debug_lock_and_unmap() argument
3590 struct device *dev = dqm->dev->adev->dev; in debug_lock_and_unmap()
3593 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { in debug_lock_and_unmap()
3594 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); in debug_lock_and_unmap()
3598 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) in debug_lock_and_unmap()
3601 dqm_lock(dqm); in debug_lock_and_unmap()
3603 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); in debug_lock_and_unmap()
3605 dqm_unlock(dqm); in debug_lock_and_unmap()
3610 int debug_map_and_unlock(struct device_queue_manager *dqm) in debug_map_and_unlock() argument
3612 struct device *dev = dqm->dev->adev->dev; in debug_map_and_unlock()
3615 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { in debug_map_and_unlock()
3616 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); in debug_map_and_unlock()
3620 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) in debug_map_and_unlock()
3623 r = map_queues_cpsch(dqm); in debug_map_and_unlock()
3625 dqm_unlock(dqm); in debug_map_and_unlock()
3630 int debug_refresh_runlist(struct device_queue_manager *dqm) in debug_refresh_runlist() argument
3632 int r = debug_lock_and_unmap(dqm); in debug_refresh_runlist()
3637 return debug_map_and_unlock(dqm); in debug_refresh_runlist()
3640 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, in kfd_dqm_is_queue_in_process() argument
3650 dqm_lock(dqm); in kfd_dqm_is_queue_in_process()
3661 dqm_unlock(dqm); in kfd_dqm_is_queue_in_process()
3689 struct device_queue_manager *dqm = data; in dqm_debugfs_hqds() local
3690 uint32_t xcc_mask = dqm->dev->xcc_mask; in dqm_debugfs_hqds()
3696 if (!dqm->sched_running) { in dqm_debugfs_hqds()
3702 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, in dqm_debugfs_hqds()
3711 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, in dqm_debugfs_hqds()
3712 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), in dqm_debugfs_hqds()
3719 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { in dqm_debugfs_hqds()
3720 int pipe_offset = pipe * get_queues_per_pipe(dqm); in dqm_debugfs_hqds()
3722 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { in dqm_debugfs_hqds()
3724 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) in dqm_debugfs_hqds()
3727 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, in dqm_debugfs_hqds()
3744 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); in dqm_debugfs_hqds()
3746 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); in dqm_debugfs_hqds()
3749 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; in dqm_debugfs_hqds()
3751 r = dqm->dev->kfd2kgd->hqd_sdma_dump( in dqm_debugfs_hqds()
3752 dqm->dev->adev, pipe, queue, &dump, &n_regs); in dqm_debugfs_hqds()
3767 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) in dqm_debugfs_hang_hws() argument
3771 dqm_lock(dqm); in dqm_debugfs_hang_hws()
3772 r = pm_debugfs_hang_hws(&dqm->packet_mgr); in dqm_debugfs_hang_hws()
3774 dqm_unlock(dqm); in dqm_debugfs_hang_hws()
3777 dqm->active_runlist = true; in dqm_debugfs_hang_hws()
3778 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, in dqm_debugfs_hang_hws()
3780 dqm_unlock(dqm); in dqm_debugfs_hang_hws()