1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/ratelimit.h> 26 #include <linux/printk.h> 27 #include <linux/slab.h> 28 #include <linux/list.h> 29 #include <linux/types.h> 30 #include <linux/bitops.h> 31 #include <linux/sched.h> 32 #include "kfd_priv.h" 33 #include "kfd_device_queue_manager.h" 34 #include "kfd_mqd_manager.h" 35 #include "cik_regs.h" 36 #include "kfd_kernel_queue.h" 37 #include "amdgpu_amdkfd.h" 38 #include "amdgpu_reset.h" 39 #include "mes_v11_api_def.h" 40 #include "kfd_debug.h" 41 42 /* Size of the per-pipe EOP queue */ 43 #define CIK_HPD_EOP_BYTES_LOG2 11 44 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 45 46 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 47 u32 pasid, unsigned int vmid); 48 49 static int execute_queues_cpsch(struct device_queue_manager *dqm, 50 enum kfd_unmap_queues_filter filter, 51 uint32_t filter_param, 52 uint32_t grace_period); 53 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 54 enum kfd_unmap_queues_filter filter, 55 uint32_t filter_param, 56 uint32_t grace_period, 57 bool reset); 58 59 static int map_queues_cpsch(struct device_queue_manager *dqm); 60 61 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 62 struct queue *q); 63 64 static inline void deallocate_hqd(struct device_queue_manager *dqm, 65 struct queue *q); 66 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); 67 static int allocate_sdma_queue(struct device_queue_manager *dqm, 68 struct queue *q, const uint32_t *restore_sdma_id); 69 static void kfd_process_hw_exception(struct work_struct *work); 70 71 static inline 72 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 73 { 74 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) 75 return KFD_MQD_TYPE_SDMA; 76 return KFD_MQD_TYPE_CP; 77 } 78 79 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) 80 { 81 int i; 82 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec 83 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 84 85 /* queue is available for KFD usage if bit is 1 */ 86 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) 87 if (test_bit(pipe_offset + i, 88 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 89 return true; 90 return false; 91 } 92 93 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) 94 { 95 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, 96 AMDGPU_MAX_QUEUES); 97 } 98 99 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) 100 { 101 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; 102 } 103 104 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) 105 { 106 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; 107 } 108 109 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) 110 { 111 return kfd_get_num_sdma_engines(dqm->dev) + 112 kfd_get_num_xgmi_sdma_engines(dqm->dev); 113 } 114 115 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) 116 { 117 return kfd_get_num_sdma_engines(dqm->dev) * 118 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 119 } 120 121 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) 122 { 123 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * 124 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 125 } 126 127 static void init_sdma_bitmaps(struct device_queue_manager *dqm) 128 { 129 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); 130 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 131 132 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 133 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 134 135 /* Mask out the reserved queues */ 136 bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap, 137 dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap, 138 KFD_MAX_SDMA_QUEUES); 139 } 140 141 void program_sh_mem_settings(struct device_queue_manager *dqm, 142 struct qcm_process_device *qpd) 143 { 144 uint32_t xcc_mask = dqm->dev->xcc_mask; 145 int xcc_id; 146 147 for_each_inst(xcc_id, xcc_mask) 148 dqm->dev->kfd2kgd->program_sh_mem_settings( 149 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, 150 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, 151 qpd->sh_mem_bases, xcc_id); 152 } 153 154 static void kfd_hws_hang(struct device_queue_manager *dqm) 155 { 156 /* 157 * Issue a GPU reset if HWS is unresponsive 158 */ 159 schedule_work(&dqm->hw_exception_work); 160 } 161 162 static int convert_to_mes_queue_type(int queue_type) 163 { 164 int mes_queue_type; 165 166 switch (queue_type) { 167 case KFD_QUEUE_TYPE_COMPUTE: 168 mes_queue_type = MES_QUEUE_TYPE_COMPUTE; 169 break; 170 case KFD_QUEUE_TYPE_SDMA: 171 mes_queue_type = MES_QUEUE_TYPE_SDMA; 172 break; 173 default: 174 WARN(1, "Invalid queue type %d", queue_type); 175 mes_queue_type = -EINVAL; 176 break; 177 } 178 179 return mes_queue_type; 180 } 181 182 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 183 struct qcm_process_device *qpd) 184 { 185 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 186 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 187 struct mes_add_queue_input queue_input; 188 int r, queue_type; 189 uint64_t wptr_addr_off; 190 191 if (!down_read_trylock(&adev->reset_domain->sem)) 192 return -EIO; 193 194 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 195 queue_input.process_id = qpd->pqm->process->pasid; 196 queue_input.page_table_base_addr = qpd->page_table_base; 197 queue_input.process_va_start = 0; 198 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 199 /* MES unit for quantum is 100ns */ 200 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */ 201 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr; 202 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */ 203 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 204 queue_input.inprocess_gang_priority = q->properties.priority; 205 queue_input.gang_global_priority_level = 206 AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 207 queue_input.doorbell_offset = q->properties.doorbell_off; 208 queue_input.mqd_addr = q->gart_mqd_addr; 209 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 210 211 if (q->wptr_bo) { 212 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1); 213 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->wptr_bo) + wptr_addr_off; 214 } 215 216 queue_input.is_kfd_process = 1; 217 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); 218 queue_input.queue_size = q->properties.queue_size >> 2; 219 220 queue_input.paging = false; 221 queue_input.tba_addr = qpd->tba_addr; 222 queue_input.tma_addr = qpd->tma_addr; 223 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device); 224 queue_input.skip_process_ctx_clear = 225 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && 226 (qpd->pqm->process->debug_trap_enabled || 227 kfd_dbg_has_ttmps_always_setup(q->device)); 228 229 queue_type = convert_to_mes_queue_type(q->properties.type); 230 if (queue_type < 0) { 231 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", 232 q->properties.type); 233 return -EINVAL; 234 } 235 queue_input.queue_type = (uint32_t)queue_type; 236 237 queue_input.exclusively_scheduled = q->properties.is_gws; 238 239 amdgpu_mes_lock(&adev->mes); 240 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 241 amdgpu_mes_unlock(&adev->mes); 242 up_read(&adev->reset_domain->sem); 243 if (r) { 244 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n", 245 q->properties.doorbell_off); 246 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 247 kfd_hws_hang(dqm); 248 } 249 250 return r; 251 } 252 253 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, 254 struct qcm_process_device *qpd) 255 { 256 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 257 int r; 258 struct mes_remove_queue_input queue_input; 259 260 if (!down_read_trylock(&adev->reset_domain->sem)) 261 return -EIO; 262 263 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 264 queue_input.doorbell_offset = q->properties.doorbell_off; 265 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 266 267 amdgpu_mes_lock(&adev->mes); 268 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 269 amdgpu_mes_unlock(&adev->mes); 270 up_read(&adev->reset_domain->sem); 271 272 if (r) { 273 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", 274 q->properties.doorbell_off); 275 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 276 kfd_hws_hang(dqm); 277 } 278 279 return r; 280 } 281 282 static int remove_all_queues_mes(struct device_queue_manager *dqm) 283 { 284 struct device_process_node *cur; 285 struct device *dev = dqm->dev->adev->dev; 286 struct qcm_process_device *qpd; 287 struct queue *q; 288 int retval = 0; 289 290 list_for_each_entry(cur, &dqm->queues, list) { 291 qpd = cur->qpd; 292 list_for_each_entry(q, &qpd->queues_list, list) { 293 if (q->properties.is_active) { 294 retval = remove_queue_mes(dqm, q, qpd); 295 if (retval) { 296 dev_err(dev, "%s: Failed to remove queue %d for dev %d", 297 __func__, 298 q->properties.queue_id, 299 dqm->dev->id); 300 return retval; 301 } 302 } 303 } 304 } 305 306 return retval; 307 } 308 309 static void increment_queue_count(struct device_queue_manager *dqm, 310 struct qcm_process_device *qpd, 311 struct queue *q) 312 { 313 dqm->active_queue_count++; 314 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 315 q->properties.type == KFD_QUEUE_TYPE_DIQ) 316 dqm->active_cp_queue_count++; 317 318 if (q->properties.is_gws) { 319 dqm->gws_queue_count++; 320 qpd->mapped_gws_queue = true; 321 } 322 } 323 324 static void decrement_queue_count(struct device_queue_manager *dqm, 325 struct qcm_process_device *qpd, 326 struct queue *q) 327 { 328 dqm->active_queue_count--; 329 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 330 q->properties.type == KFD_QUEUE_TYPE_DIQ) 331 dqm->active_cp_queue_count--; 332 333 if (q->properties.is_gws) { 334 dqm->gws_queue_count--; 335 qpd->mapped_gws_queue = false; 336 } 337 } 338 339 /* 340 * Allocate a doorbell ID to this queue. 341 * If doorbell_id is passed in, make sure requested ID is valid then allocate it. 342 */ 343 static int allocate_doorbell(struct qcm_process_device *qpd, 344 struct queue *q, 345 uint32_t const *restore_id) 346 { 347 struct kfd_node *dev = qpd->dqm->dev; 348 349 if (!KFD_IS_SOC15(dev)) { 350 /* On pre-SOC15 chips we need to use the queue ID to 351 * preserve the user mode ABI. 352 */ 353 354 if (restore_id && *restore_id != q->properties.queue_id) 355 return -EINVAL; 356 357 q->doorbell_id = q->properties.queue_id; 358 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 359 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 360 /* For SDMA queues on SOC15 with 8-byte doorbell, use static 361 * doorbell assignments based on the engine and queue id. 362 * The doobell index distance between RLC (2*i) and (2*i+1) 363 * for a SDMA engine is 512. 364 */ 365 366 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx; 367 368 /* 369 * q->properties.sdma_engine_id corresponds to the virtual 370 * sdma engine number. However, for doorbell allocation, 371 * we need the physical sdma engine id in order to get the 372 * correct doorbell offset. 373 */ 374 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * 375 get_num_all_sdma_engines(qpd->dqm) + 376 q->properties.sdma_engine_id] 377 + (q->properties.sdma_queue_id & 1) 378 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET 379 + (q->properties.sdma_queue_id >> 1); 380 381 if (restore_id && *restore_id != valid_id) 382 return -EINVAL; 383 q->doorbell_id = valid_id; 384 } else { 385 /* For CP queues on SOC15 */ 386 if (restore_id) { 387 /* make sure that ID is free */ 388 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) 389 return -EINVAL; 390 391 q->doorbell_id = *restore_id; 392 } else { 393 /* or reserve a free doorbell ID */ 394 unsigned int found; 395 396 found = find_first_zero_bit(qpd->doorbell_bitmap, 397 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 398 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 399 pr_debug("No doorbells available"); 400 return -EBUSY; 401 } 402 set_bit(found, qpd->doorbell_bitmap); 403 q->doorbell_id = found; 404 } 405 } 406 407 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev, 408 qpd->proc_doorbells, 409 q->doorbell_id, 410 dev->kfd->device_info.doorbell_size); 411 return 0; 412 } 413 414 static void deallocate_doorbell(struct qcm_process_device *qpd, 415 struct queue *q) 416 { 417 unsigned int old; 418 struct kfd_node *dev = qpd->dqm->dev; 419 420 if (!KFD_IS_SOC15(dev) || 421 q->properties.type == KFD_QUEUE_TYPE_SDMA || 422 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 423 return; 424 425 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); 426 WARN_ON(!old); 427 } 428 429 static void program_trap_handler_settings(struct device_queue_manager *dqm, 430 struct qcm_process_device *qpd) 431 { 432 uint32_t xcc_mask = dqm->dev->xcc_mask; 433 int xcc_id; 434 435 if (dqm->dev->kfd2kgd->program_trap_handler_settings) 436 for_each_inst(xcc_id, xcc_mask) 437 dqm->dev->kfd2kgd->program_trap_handler_settings( 438 dqm->dev->adev, qpd->vmid, qpd->tba_addr, 439 qpd->tma_addr, xcc_id); 440 } 441 442 static int allocate_vmid(struct device_queue_manager *dqm, 443 struct qcm_process_device *qpd, 444 struct queue *q) 445 { 446 struct device *dev = dqm->dev->adev->dev; 447 int allocated_vmid = -1, i; 448 449 for (i = dqm->dev->vm_info.first_vmid_kfd; 450 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { 451 if (!dqm->vmid_pasid[i]) { 452 allocated_vmid = i; 453 break; 454 } 455 } 456 457 if (allocated_vmid < 0) { 458 dev_err(dev, "no more vmid to allocate\n"); 459 return -ENOSPC; 460 } 461 462 pr_debug("vmid allocated: %d\n", allocated_vmid); 463 464 dqm->vmid_pasid[allocated_vmid] = q->process->pasid; 465 466 set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid); 467 468 qpd->vmid = allocated_vmid; 469 q->properties.vmid = allocated_vmid; 470 471 program_sh_mem_settings(dqm, qpd); 472 473 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) 474 program_trap_handler_settings(dqm, qpd); 475 476 /* qpd->page_table_base is set earlier when register_process() 477 * is called, i.e. when the first queue is created. 478 */ 479 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, 480 qpd->vmid, 481 qpd->page_table_base); 482 /* invalidate the VM context after pasid and vmid mapping is set up */ 483 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 484 485 if (dqm->dev->kfd2kgd->set_scratch_backing_va) 486 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, 487 qpd->sh_hidden_private_base, qpd->vmid); 488 489 return 0; 490 } 491 492 static int flush_texture_cache_nocpsch(struct kfd_node *kdev, 493 struct qcm_process_device *qpd) 494 { 495 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; 496 int ret; 497 498 if (!qpd->ib_kaddr) 499 return -ENOMEM; 500 501 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); 502 if (ret) 503 return ret; 504 505 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, 506 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, 507 pmf->release_mem_size / sizeof(uint32_t)); 508 } 509 510 static void deallocate_vmid(struct device_queue_manager *dqm, 511 struct qcm_process_device *qpd, 512 struct queue *q) 513 { 514 struct device *dev = dqm->dev->adev->dev; 515 516 /* On GFX v7, CP doesn't flush TC at dequeue */ 517 if (q->device->adev->asic_type == CHIP_HAWAII) 518 if (flush_texture_cache_nocpsch(q->device, qpd)) 519 dev_err(dev, "Failed to flush TC\n"); 520 521 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 522 523 /* Release the vmid mapping */ 524 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 525 dqm->vmid_pasid[qpd->vmid] = 0; 526 527 qpd->vmid = 0; 528 q->properties.vmid = 0; 529 } 530 531 static int create_queue_nocpsch(struct device_queue_manager *dqm, 532 struct queue *q, 533 struct qcm_process_device *qpd, 534 const struct kfd_criu_queue_priv_data *qd, 535 const void *restore_mqd, const void *restore_ctl_stack) 536 { 537 struct mqd_manager *mqd_mgr; 538 int retval; 539 540 dqm_lock(dqm); 541 542 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 543 pr_warn("Can't create new usermode queue because %d queues were already created\n", 544 dqm->total_queue_count); 545 retval = -EPERM; 546 goto out_unlock; 547 } 548 549 if (list_empty(&qpd->queues_list)) { 550 retval = allocate_vmid(dqm, qpd, q); 551 if (retval) 552 goto out_unlock; 553 } 554 q->properties.vmid = qpd->vmid; 555 /* 556 * Eviction state logic: mark all queues as evicted, even ones 557 * not currently active. Restoring inactive queues later only 558 * updates the is_evicted flag but is a no-op otherwise. 559 */ 560 q->properties.is_evicted = !!qpd->evicted; 561 562 q->properties.tba_addr = qpd->tba_addr; 563 q->properties.tma_addr = qpd->tma_addr; 564 565 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 566 q->properties.type)]; 567 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 568 retval = allocate_hqd(dqm, q); 569 if (retval) 570 goto deallocate_vmid; 571 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", 572 q->pipe, q->queue); 573 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 574 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 575 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 576 if (retval) 577 goto deallocate_vmid; 578 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 579 } 580 581 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 582 if (retval) 583 goto out_deallocate_hqd; 584 585 /* Temporarily release dqm lock to avoid a circular lock dependency */ 586 dqm_unlock(dqm); 587 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 588 dqm_lock(dqm); 589 590 if (!q->mqd_mem_obj) { 591 retval = -ENOMEM; 592 goto out_deallocate_doorbell; 593 } 594 595 if (qd) 596 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 597 &q->properties, restore_mqd, restore_ctl_stack, 598 qd->ctl_stack_size); 599 else 600 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 601 &q->gart_mqd_addr, &q->properties); 602 603 if (q->properties.is_active) { 604 if (!dqm->sched_running) { 605 WARN_ONCE(1, "Load non-HWS mqd while stopped\n"); 606 goto add_queue_to_list; 607 } 608 609 if (WARN(q->process->mm != current->mm, 610 "should only run in user thread")) 611 retval = -EFAULT; 612 else 613 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 614 q->queue, &q->properties, current->mm); 615 if (retval) 616 goto out_free_mqd; 617 } 618 619 add_queue_to_list: 620 list_add(&q->list, &qpd->queues_list); 621 qpd->queue_count++; 622 if (q->properties.is_active) 623 increment_queue_count(dqm, qpd, q); 624 625 /* 626 * Unconditionally increment this counter, regardless of the queue's 627 * type or whether the queue is active. 628 */ 629 dqm->total_queue_count++; 630 pr_debug("Total of %d queues are accountable so far\n", 631 dqm->total_queue_count); 632 goto out_unlock; 633 634 out_free_mqd: 635 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 636 out_deallocate_doorbell: 637 deallocate_doorbell(qpd, q); 638 out_deallocate_hqd: 639 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 640 deallocate_hqd(dqm, q); 641 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 642 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 643 deallocate_sdma_queue(dqm, q); 644 deallocate_vmid: 645 if (list_empty(&qpd->queues_list)) 646 deallocate_vmid(dqm, qpd, q); 647 out_unlock: 648 dqm_unlock(dqm); 649 return retval; 650 } 651 652 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 653 { 654 bool set; 655 int pipe, bit, i; 656 657 set = false; 658 659 for (pipe = dqm->next_pipe_to_allocate, i = 0; 660 i < get_pipes_per_mec(dqm); 661 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { 662 663 if (!is_pipe_enabled(dqm, 0, pipe)) 664 continue; 665 666 if (dqm->allocated_queues[pipe] != 0) { 667 bit = ffs(dqm->allocated_queues[pipe]) - 1; 668 dqm->allocated_queues[pipe] &= ~(1 << bit); 669 q->pipe = pipe; 670 q->queue = bit; 671 set = true; 672 break; 673 } 674 } 675 676 if (!set) 677 return -EBUSY; 678 679 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); 680 /* horizontal hqd allocation */ 681 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); 682 683 return 0; 684 } 685 686 static inline void deallocate_hqd(struct device_queue_manager *dqm, 687 struct queue *q) 688 { 689 dqm->allocated_queues[q->pipe] |= (1 << q->queue); 690 } 691 692 #define SQ_IND_CMD_CMD_KILL 0x00000003 693 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001 694 695 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p) 696 { 697 int status = 0; 698 unsigned int vmid; 699 uint16_t queried_pasid; 700 union SQ_CMD_BITS reg_sq_cmd; 701 union GRBM_GFX_INDEX_BITS reg_gfx_index; 702 struct kfd_process_device *pdd; 703 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd; 704 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd; 705 uint32_t xcc_mask = dev->xcc_mask; 706 int xcc_id; 707 708 reg_sq_cmd.u32All = 0; 709 reg_gfx_index.u32All = 0; 710 711 pr_debug("Killing all process wavefronts\n"); 712 713 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) { 714 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n"); 715 return -EOPNOTSUPP; 716 } 717 718 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. 719 * ATC_VMID15_PASID_MAPPING 720 * to check which VMID the current process is mapped to. 721 */ 722 723 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) { 724 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info 725 (dev->adev, vmid, &queried_pasid); 726 727 if (status && queried_pasid == p->pasid) { 728 pr_debug("Killing wave fronts of vmid %d and pasid 0x%x\n", 729 vmid, p->pasid); 730 break; 731 } 732 } 733 734 if (vmid > last_vmid_to_scan) { 735 dev_err(dev->adev->dev, "Didn't find vmid for pasid 0x%x\n", p->pasid); 736 return -EFAULT; 737 } 738 739 /* taking the VMID for that process on the safe way using PDD */ 740 pdd = kfd_get_process_device_data(dev, p); 741 if (!pdd) 742 return -EFAULT; 743 744 reg_gfx_index.bits.sh_broadcast_writes = 1; 745 reg_gfx_index.bits.se_broadcast_writes = 1; 746 reg_gfx_index.bits.instance_broadcast_writes = 1; 747 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST; 748 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL; 749 reg_sq_cmd.bits.vm_id = vmid; 750 751 for_each_inst(xcc_id, xcc_mask) 752 dev->kfd2kgd->wave_control_execute( 753 dev->adev, reg_gfx_index.u32All, 754 reg_sq_cmd.u32All, xcc_id); 755 756 return 0; 757 } 758 759 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked 760 * to avoid asynchronized access 761 */ 762 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 763 struct qcm_process_device *qpd, 764 struct queue *q) 765 { 766 int retval; 767 struct mqd_manager *mqd_mgr; 768 769 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 770 q->properties.type)]; 771 772 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 773 deallocate_hqd(dqm, q); 774 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 775 deallocate_sdma_queue(dqm, q); 776 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 777 deallocate_sdma_queue(dqm, q); 778 else { 779 pr_debug("q->properties.type %d is invalid\n", 780 q->properties.type); 781 return -EINVAL; 782 } 783 dqm->total_queue_count--; 784 785 deallocate_doorbell(qpd, q); 786 787 if (!dqm->sched_running) { 788 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n"); 789 return 0; 790 } 791 792 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 793 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 794 KFD_UNMAP_LATENCY_MS, 795 q->pipe, q->queue); 796 if (retval == -ETIME) 797 qpd->reset_wavefronts = true; 798 799 list_del(&q->list); 800 if (list_empty(&qpd->queues_list)) { 801 if (qpd->reset_wavefronts) { 802 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", 803 dqm->dev); 804 /* dbgdev_wave_reset_wavefronts has to be called before 805 * deallocate_vmid(), i.e. when vmid is still in use. 806 */ 807 dbgdev_wave_reset_wavefronts(dqm->dev, 808 qpd->pqm->process); 809 qpd->reset_wavefronts = false; 810 } 811 812 deallocate_vmid(dqm, qpd, q); 813 } 814 qpd->queue_count--; 815 if (q->properties.is_active) 816 decrement_queue_count(dqm, qpd, q); 817 818 return retval; 819 } 820 821 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 822 struct qcm_process_device *qpd, 823 struct queue *q) 824 { 825 int retval; 826 uint64_t sdma_val = 0; 827 struct device *dev = dqm->dev->adev->dev; 828 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 829 struct mqd_manager *mqd_mgr = 830 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 831 832 /* Get the SDMA queue stats */ 833 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 834 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 835 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 836 &sdma_val); 837 if (retval) 838 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 839 q->properties.queue_id); 840 } 841 842 dqm_lock(dqm); 843 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); 844 if (!retval) 845 pdd->sdma_past_activity_counter += sdma_val; 846 dqm_unlock(dqm); 847 848 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 849 850 return retval; 851 } 852 853 static int update_queue(struct device_queue_manager *dqm, struct queue *q, 854 struct mqd_update_info *minfo) 855 { 856 int retval = 0; 857 struct device *dev = dqm->dev->adev->dev; 858 struct mqd_manager *mqd_mgr; 859 struct kfd_process_device *pdd; 860 bool prev_active = false; 861 862 dqm_lock(dqm); 863 pdd = kfd_get_process_device_data(q->device, q->process); 864 if (!pdd) { 865 retval = -ENODEV; 866 goto out_unlock; 867 } 868 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 869 q->properties.type)]; 870 871 /* Save previous activity state for counters */ 872 prev_active = q->properties.is_active; 873 874 /* Make sure the queue is unmapped before updating the MQD */ 875 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 876 if (!dqm->dev->kfd->shared_resources.enable_mes) 877 retval = unmap_queues_cpsch(dqm, 878 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 879 else if (prev_active) 880 retval = remove_queue_mes(dqm, q, &pdd->qpd); 881 882 if (retval) { 883 dev_err(dev, "unmap queue failed\n"); 884 goto out_unlock; 885 } 886 } else if (prev_active && 887 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 888 q->properties.type == KFD_QUEUE_TYPE_SDMA || 889 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 890 891 if (!dqm->sched_running) { 892 WARN_ONCE(1, "Update non-HWS queue while stopped\n"); 893 goto out_unlock; 894 } 895 896 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 897 (dqm->dev->kfd->cwsr_enabled ? 898 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 899 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 900 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 901 if (retval) { 902 dev_err(dev, "destroy mqd failed\n"); 903 goto out_unlock; 904 } 905 } 906 907 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo); 908 909 /* 910 * check active state vs. the previous state and modify 911 * counter accordingly. map_queues_cpsch uses the 912 * dqm->active_queue_count to determine whether a new runlist must be 913 * uploaded. 914 */ 915 if (q->properties.is_active && !prev_active) { 916 increment_queue_count(dqm, &pdd->qpd, q); 917 } else if (!q->properties.is_active && prev_active) { 918 decrement_queue_count(dqm, &pdd->qpd, q); 919 } else if (q->gws && !q->properties.is_gws) { 920 if (q->properties.is_active) { 921 dqm->gws_queue_count++; 922 pdd->qpd.mapped_gws_queue = true; 923 } 924 q->properties.is_gws = true; 925 } else if (!q->gws && q->properties.is_gws) { 926 if (q->properties.is_active) { 927 dqm->gws_queue_count--; 928 pdd->qpd.mapped_gws_queue = false; 929 } 930 q->properties.is_gws = false; 931 } 932 933 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 934 if (!dqm->dev->kfd->shared_resources.enable_mes) 935 retval = map_queues_cpsch(dqm); 936 else if (q->properties.is_active) 937 retval = add_queue_mes(dqm, q, &pdd->qpd); 938 } else if (q->properties.is_active && 939 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 940 q->properties.type == KFD_QUEUE_TYPE_SDMA || 941 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 942 if (WARN(q->process->mm != current->mm, 943 "should only run in user thread")) 944 retval = -EFAULT; 945 else 946 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 947 q->pipe, q->queue, 948 &q->properties, current->mm); 949 } 950 951 out_unlock: 952 dqm_unlock(dqm); 953 return retval; 954 } 955 956 /* suspend_single_queue does not lock the dqm like the 957 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should 958 * lock the dqm before calling, and unlock after calling. 959 * 960 * The reason we don't lock the dqm is because this function may be 961 * called on multiple queues in a loop, so rather than locking/unlocking 962 * multiple times, we will just keep the dqm locked for all of the calls. 963 */ 964 static int suspend_single_queue(struct device_queue_manager *dqm, 965 struct kfd_process_device *pdd, 966 struct queue *q) 967 { 968 bool is_new; 969 970 if (q->properties.is_suspended) 971 return 0; 972 973 pr_debug("Suspending PASID %u queue [%i]\n", 974 pdd->process->pasid, 975 q->properties.queue_id); 976 977 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW); 978 979 if (is_new || q->properties.is_being_destroyed) { 980 pr_debug("Suspend: skip %s queue id %i\n", 981 is_new ? "new" : "destroyed", 982 q->properties.queue_id); 983 return -EBUSY; 984 } 985 986 q->properties.is_suspended = true; 987 if (q->properties.is_active) { 988 if (dqm->dev->kfd->shared_resources.enable_mes) { 989 int r = remove_queue_mes(dqm, q, &pdd->qpd); 990 991 if (r) 992 return r; 993 } 994 995 decrement_queue_count(dqm, &pdd->qpd, q); 996 q->properties.is_active = false; 997 } 998 999 return 0; 1000 } 1001 1002 /* resume_single_queue does not lock the dqm like the functions 1003 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should 1004 * lock the dqm before calling, and unlock after calling. 1005 * 1006 * The reason we don't lock the dqm is because this function may be 1007 * called on multiple queues in a loop, so rather than locking/unlocking 1008 * multiple times, we will just keep the dqm locked for all of the calls. 1009 */ 1010 static int resume_single_queue(struct device_queue_manager *dqm, 1011 struct qcm_process_device *qpd, 1012 struct queue *q) 1013 { 1014 struct kfd_process_device *pdd; 1015 1016 if (!q->properties.is_suspended) 1017 return 0; 1018 1019 pdd = qpd_to_pdd(qpd); 1020 1021 pr_debug("Restoring from suspend PASID %u queue [%i]\n", 1022 pdd->process->pasid, 1023 q->properties.queue_id); 1024 1025 q->properties.is_suspended = false; 1026 1027 if (QUEUE_IS_ACTIVE(q->properties)) { 1028 if (dqm->dev->kfd->shared_resources.enable_mes) { 1029 int r = add_queue_mes(dqm, q, &pdd->qpd); 1030 1031 if (r) 1032 return r; 1033 } 1034 1035 q->properties.is_active = true; 1036 increment_queue_count(dqm, qpd, q); 1037 } 1038 1039 return 0; 1040 } 1041 1042 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, 1043 struct qcm_process_device *qpd) 1044 { 1045 struct queue *q; 1046 struct mqd_manager *mqd_mgr; 1047 struct kfd_process_device *pdd; 1048 int retval, ret = 0; 1049 1050 dqm_lock(dqm); 1051 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1052 goto out; 1053 1054 pdd = qpd_to_pdd(qpd); 1055 pr_debug_ratelimited("Evicting PASID 0x%x queues\n", 1056 pdd->process->pasid); 1057 1058 pdd->last_evict_timestamp = get_jiffies_64(); 1059 /* Mark all queues as evicted. Deactivate all active queues on 1060 * the qpd. 1061 */ 1062 list_for_each_entry(q, &qpd->queues_list, list) { 1063 q->properties.is_evicted = true; 1064 if (!q->properties.is_active) 1065 continue; 1066 1067 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1068 q->properties.type)]; 1069 q->properties.is_active = false; 1070 decrement_queue_count(dqm, qpd, q); 1071 1072 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) 1073 continue; 1074 1075 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1076 (dqm->dev->kfd->cwsr_enabled ? 1077 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1078 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1079 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1080 if (retval && !ret) 1081 /* Return the first error, but keep going to 1082 * maintain a consistent eviction state 1083 */ 1084 ret = retval; 1085 } 1086 1087 out: 1088 dqm_unlock(dqm); 1089 return ret; 1090 } 1091 1092 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, 1093 struct qcm_process_device *qpd) 1094 { 1095 struct queue *q; 1096 struct device *dev = dqm->dev->adev->dev; 1097 struct kfd_process_device *pdd; 1098 int retval = 0; 1099 1100 dqm_lock(dqm); 1101 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1102 goto out; 1103 1104 pdd = qpd_to_pdd(qpd); 1105 1106 /* The debugger creates processes that temporarily have not acquired 1107 * all VMs for all devices and has no VMs itself. 1108 * Skip queue eviction on process eviction. 1109 */ 1110 if (!pdd->drm_priv) 1111 goto out; 1112 1113 pr_debug_ratelimited("Evicting PASID 0x%x queues\n", 1114 pdd->process->pasid); 1115 1116 /* Mark all queues as evicted. Deactivate all active queues on 1117 * the qpd. 1118 */ 1119 list_for_each_entry(q, &qpd->queues_list, list) { 1120 q->properties.is_evicted = true; 1121 if (!q->properties.is_active) 1122 continue; 1123 1124 q->properties.is_active = false; 1125 decrement_queue_count(dqm, qpd, q); 1126 1127 if (dqm->dev->kfd->shared_resources.enable_mes) { 1128 retval = remove_queue_mes(dqm, q, qpd); 1129 if (retval) { 1130 dev_err(dev, "Failed to evict queue %d\n", 1131 q->properties.queue_id); 1132 goto out; 1133 } 1134 } 1135 } 1136 pdd->last_evict_timestamp = get_jiffies_64(); 1137 if (!dqm->dev->kfd->shared_resources.enable_mes) 1138 retval = execute_queues_cpsch(dqm, 1139 qpd->is_debug ? 1140 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : 1141 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1142 USE_DEFAULT_GRACE_PERIOD); 1143 1144 out: 1145 dqm_unlock(dqm); 1146 return retval; 1147 } 1148 1149 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, 1150 struct qcm_process_device *qpd) 1151 { 1152 struct mm_struct *mm = NULL; 1153 struct queue *q; 1154 struct mqd_manager *mqd_mgr; 1155 struct kfd_process_device *pdd; 1156 uint64_t pd_base; 1157 uint64_t eviction_duration; 1158 int retval, ret = 0; 1159 1160 pdd = qpd_to_pdd(qpd); 1161 /* Retrieve PD base */ 1162 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1163 1164 dqm_lock(dqm); 1165 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1166 goto out; 1167 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1168 qpd->evicted--; 1169 goto out; 1170 } 1171 1172 pr_debug_ratelimited("Restoring PASID 0x%x queues\n", 1173 pdd->process->pasid); 1174 1175 /* Update PD Base in QPD */ 1176 qpd->page_table_base = pd_base; 1177 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1178 1179 if (!list_empty(&qpd->queues_list)) { 1180 dqm->dev->kfd2kgd->set_vm_context_page_table_base( 1181 dqm->dev->adev, 1182 qpd->vmid, 1183 qpd->page_table_base); 1184 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); 1185 } 1186 1187 /* Take a safe reference to the mm_struct, which may otherwise 1188 * disappear even while the kfd_process is still referenced. 1189 */ 1190 mm = get_task_mm(pdd->process->lead_thread); 1191 if (!mm) { 1192 ret = -EFAULT; 1193 goto out; 1194 } 1195 1196 /* Remove the eviction flags. Activate queues that are not 1197 * inactive for other reasons. 1198 */ 1199 list_for_each_entry(q, &qpd->queues_list, list) { 1200 q->properties.is_evicted = false; 1201 if (!QUEUE_IS_ACTIVE(q->properties)) 1202 continue; 1203 1204 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1205 q->properties.type)]; 1206 q->properties.is_active = true; 1207 increment_queue_count(dqm, qpd, q); 1208 1209 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) 1210 continue; 1211 1212 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 1213 q->queue, &q->properties, mm); 1214 if (retval && !ret) 1215 /* Return the first error, but keep going to 1216 * maintain a consistent eviction state 1217 */ 1218 ret = retval; 1219 } 1220 qpd->evicted = 0; 1221 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1222 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1223 out: 1224 if (mm) 1225 mmput(mm); 1226 dqm_unlock(dqm); 1227 return ret; 1228 } 1229 1230 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, 1231 struct qcm_process_device *qpd) 1232 { 1233 struct queue *q; 1234 struct device *dev = dqm->dev->adev->dev; 1235 struct kfd_process_device *pdd; 1236 uint64_t eviction_duration; 1237 int retval = 0; 1238 1239 pdd = qpd_to_pdd(qpd); 1240 1241 dqm_lock(dqm); 1242 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1243 goto out; 1244 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1245 qpd->evicted--; 1246 goto out; 1247 } 1248 1249 /* The debugger creates processes that temporarily have not acquired 1250 * all VMs for all devices and has no VMs itself. 1251 * Skip queue restore on process restore. 1252 */ 1253 if (!pdd->drm_priv) 1254 goto vm_not_acquired; 1255 1256 pr_debug_ratelimited("Restoring PASID 0x%x queues\n", 1257 pdd->process->pasid); 1258 1259 /* Update PD Base in QPD */ 1260 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1261 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); 1262 1263 /* activate all active queues on the qpd */ 1264 list_for_each_entry(q, &qpd->queues_list, list) { 1265 q->properties.is_evicted = false; 1266 if (!QUEUE_IS_ACTIVE(q->properties)) 1267 continue; 1268 1269 q->properties.is_active = true; 1270 increment_queue_count(dqm, &pdd->qpd, q); 1271 1272 if (dqm->dev->kfd->shared_resources.enable_mes) { 1273 retval = add_queue_mes(dqm, q, qpd); 1274 if (retval) { 1275 dev_err(dev, "Failed to restore queue %d\n", 1276 q->properties.queue_id); 1277 goto out; 1278 } 1279 } 1280 } 1281 if (!dqm->dev->kfd->shared_resources.enable_mes) 1282 retval = execute_queues_cpsch(dqm, 1283 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1284 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1285 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1286 vm_not_acquired: 1287 qpd->evicted = 0; 1288 out: 1289 dqm_unlock(dqm); 1290 return retval; 1291 } 1292 1293 static int register_process(struct device_queue_manager *dqm, 1294 struct qcm_process_device *qpd) 1295 { 1296 struct device_process_node *n; 1297 struct kfd_process_device *pdd; 1298 uint64_t pd_base; 1299 int retval; 1300 1301 n = kzalloc(sizeof(*n), GFP_KERNEL); 1302 if (!n) 1303 return -ENOMEM; 1304 1305 n->qpd = qpd; 1306 1307 pdd = qpd_to_pdd(qpd); 1308 /* Retrieve PD base */ 1309 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1310 1311 dqm_lock(dqm); 1312 list_add(&n->list, &dqm->queues); 1313 1314 /* Update PD Base in QPD */ 1315 qpd->page_table_base = pd_base; 1316 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1317 1318 retval = dqm->asic_ops.update_qpd(dqm, qpd); 1319 1320 dqm->processes_count++; 1321 1322 dqm_unlock(dqm); 1323 1324 /* Outside the DQM lock because under the DQM lock we can't do 1325 * reclaim or take other locks that others hold while reclaiming. 1326 */ 1327 kfd_inc_compute_active(dqm->dev); 1328 1329 return retval; 1330 } 1331 1332 static int unregister_process(struct device_queue_manager *dqm, 1333 struct qcm_process_device *qpd) 1334 { 1335 int retval; 1336 struct device_process_node *cur, *next; 1337 1338 pr_debug("qpd->queues_list is %s\n", 1339 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 1340 1341 retval = 0; 1342 dqm_lock(dqm); 1343 1344 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 1345 if (qpd == cur->qpd) { 1346 list_del(&cur->list); 1347 kfree(cur); 1348 dqm->processes_count--; 1349 goto out; 1350 } 1351 } 1352 /* qpd not found in dqm list */ 1353 retval = 1; 1354 out: 1355 dqm_unlock(dqm); 1356 1357 /* Outside the DQM lock because under the DQM lock we can't do 1358 * reclaim or take other locks that others hold while reclaiming. 1359 */ 1360 if (!retval) 1361 kfd_dec_compute_active(dqm->dev); 1362 1363 return retval; 1364 } 1365 1366 static int 1367 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, 1368 unsigned int vmid) 1369 { 1370 uint32_t xcc_mask = dqm->dev->xcc_mask; 1371 int xcc_id, ret; 1372 1373 for_each_inst(xcc_id, xcc_mask) { 1374 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 1375 dqm->dev->adev, pasid, vmid, xcc_id); 1376 if (ret) 1377 break; 1378 } 1379 1380 return ret; 1381 } 1382 1383 static void init_interrupts(struct device_queue_manager *dqm) 1384 { 1385 uint32_t xcc_mask = dqm->dev->xcc_mask; 1386 unsigned int i, xcc_id; 1387 1388 for_each_inst(xcc_id, xcc_mask) { 1389 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { 1390 if (is_pipe_enabled(dqm, 0, i)) { 1391 dqm->dev->kfd2kgd->init_interrupts( 1392 dqm->dev->adev, i, xcc_id); 1393 } 1394 } 1395 } 1396 } 1397 1398 static int initialize_nocpsch(struct device_queue_manager *dqm) 1399 { 1400 int pipe, queue; 1401 1402 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1403 1404 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), 1405 sizeof(unsigned int), GFP_KERNEL); 1406 if (!dqm->allocated_queues) 1407 return -ENOMEM; 1408 1409 mutex_init(&dqm->lock_hidden); 1410 INIT_LIST_HEAD(&dqm->queues); 1411 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; 1412 dqm->active_cp_queue_count = 0; 1413 dqm->gws_queue_count = 0; 1414 1415 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 1416 int pipe_offset = pipe * get_queues_per_pipe(dqm); 1417 1418 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) 1419 if (test_bit(pipe_offset + queue, 1420 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1421 dqm->allocated_queues[pipe] |= 1 << queue; 1422 } 1423 1424 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); 1425 1426 init_sdma_bitmaps(dqm); 1427 1428 return 0; 1429 } 1430 1431 static void uninitialize(struct device_queue_manager *dqm) 1432 { 1433 int i; 1434 1435 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); 1436 1437 kfree(dqm->allocated_queues); 1438 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 1439 kfree(dqm->mqd_mgrs[i]); 1440 mutex_destroy(&dqm->lock_hidden); 1441 } 1442 1443 static int start_nocpsch(struct device_queue_manager *dqm) 1444 { 1445 int r = 0; 1446 1447 pr_info("SW scheduler is used"); 1448 init_interrupts(dqm); 1449 1450 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1451 r = pm_init(&dqm->packet_mgr, dqm); 1452 if (!r) 1453 dqm->sched_running = true; 1454 1455 return r; 1456 } 1457 1458 static int stop_nocpsch(struct device_queue_manager *dqm) 1459 { 1460 dqm_lock(dqm); 1461 if (!dqm->sched_running) { 1462 dqm_unlock(dqm); 1463 return 0; 1464 } 1465 1466 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1467 pm_uninit(&dqm->packet_mgr); 1468 dqm->sched_running = false; 1469 dqm_unlock(dqm); 1470 1471 return 0; 1472 } 1473 1474 static int allocate_sdma_queue(struct device_queue_manager *dqm, 1475 struct queue *q, const uint32_t *restore_sdma_id) 1476 { 1477 struct device *dev = dqm->dev->adev->dev; 1478 int bit; 1479 1480 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1481 if (bitmap_empty(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES)) { 1482 dev_err(dev, "No more SDMA queue to allocate\n"); 1483 return -ENOMEM; 1484 } 1485 1486 if (restore_sdma_id) { 1487 /* Re-use existing sdma_id */ 1488 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { 1489 dev_err(dev, "SDMA queue already in use\n"); 1490 return -EBUSY; 1491 } 1492 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); 1493 q->sdma_id = *restore_sdma_id; 1494 } else { 1495 /* Find first available sdma_id */ 1496 bit = find_first_bit(dqm->sdma_bitmap, 1497 get_num_sdma_queues(dqm)); 1498 clear_bit(bit, dqm->sdma_bitmap); 1499 q->sdma_id = bit; 1500 } 1501 1502 q->properties.sdma_engine_id = 1503 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); 1504 q->properties.sdma_queue_id = q->sdma_id / 1505 kfd_get_num_sdma_engines(dqm->dev); 1506 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1507 if (bitmap_empty(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES)) { 1508 dev_err(dev, "No more XGMI SDMA queue to allocate\n"); 1509 return -ENOMEM; 1510 } 1511 if (restore_sdma_id) { 1512 /* Re-use existing sdma_id */ 1513 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { 1514 dev_err(dev, "SDMA queue already in use\n"); 1515 return -EBUSY; 1516 } 1517 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); 1518 q->sdma_id = *restore_sdma_id; 1519 } else { 1520 bit = find_first_bit(dqm->xgmi_sdma_bitmap, 1521 get_num_xgmi_sdma_queues(dqm)); 1522 clear_bit(bit, dqm->xgmi_sdma_bitmap); 1523 q->sdma_id = bit; 1524 } 1525 /* sdma_engine_id is sdma id including 1526 * both PCIe-optimized SDMAs and XGMI- 1527 * optimized SDMAs. The calculation below 1528 * assumes the first N engines are always 1529 * PCIe-optimized ones 1530 */ 1531 q->properties.sdma_engine_id = 1532 kfd_get_num_sdma_engines(dqm->dev) + 1533 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); 1534 q->properties.sdma_queue_id = q->sdma_id / 1535 kfd_get_num_xgmi_sdma_engines(dqm->dev); 1536 } 1537 1538 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); 1539 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); 1540 1541 return 0; 1542 } 1543 1544 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 1545 struct queue *q) 1546 { 1547 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1548 if (q->sdma_id >= get_num_sdma_queues(dqm)) 1549 return; 1550 set_bit(q->sdma_id, dqm->sdma_bitmap); 1551 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1552 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1553 return; 1554 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); 1555 } 1556 } 1557 1558 /* 1559 * Device Queue Manager implementation for cp scheduler 1560 */ 1561 1562 static int set_sched_resources(struct device_queue_manager *dqm) 1563 { 1564 int i, mec; 1565 struct scheduling_resources res; 1566 struct device *dev = dqm->dev->adev->dev; 1567 1568 res.vmid_mask = dqm->dev->compute_vmid_bitmap; 1569 1570 res.queue_mask = 0; 1571 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 1572 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 1573 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 1574 1575 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1576 continue; 1577 1578 /* only acquire queues from the first MEC */ 1579 if (mec > 0) 1580 continue; 1581 1582 /* This situation may be hit in the future if a new HW 1583 * generation exposes more than 64 queues. If so, the 1584 * definition of res.queue_mask needs updating 1585 */ 1586 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { 1587 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i); 1588 break; 1589 } 1590 1591 res.queue_mask |= 1ull 1592 << amdgpu_queue_mask_bit_to_set_resource_bit( 1593 dqm->dev->adev, i); 1594 } 1595 res.gws_mask = ~0ull; 1596 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; 1597 1598 pr_debug("Scheduling resources:\n" 1599 "vmid mask: 0x%8X\n" 1600 "queue mask: 0x%8llX\n", 1601 res.vmid_mask, res.queue_mask); 1602 1603 return pm_send_set_resources(&dqm->packet_mgr, &res); 1604 } 1605 1606 static int initialize_cpsch(struct device_queue_manager *dqm) 1607 { 1608 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1609 1610 mutex_init(&dqm->lock_hidden); 1611 INIT_LIST_HEAD(&dqm->queues); 1612 dqm->active_queue_count = dqm->processes_count = 0; 1613 dqm->active_cp_queue_count = 0; 1614 dqm->gws_queue_count = 0; 1615 dqm->active_runlist = false; 1616 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); 1617 dqm->trap_debug_vmid = 0; 1618 1619 init_sdma_bitmaps(dqm); 1620 1621 if (dqm->dev->kfd2kgd->get_iq_wait_times) 1622 dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev, 1623 &dqm->wait_times, 1624 ffs(dqm->dev->xcc_mask) - 1); 1625 return 0; 1626 } 1627 1628 static int start_cpsch(struct device_queue_manager *dqm) 1629 { 1630 struct device *dev = dqm->dev->adev->dev; 1631 int retval; 1632 1633 retval = 0; 1634 1635 dqm_lock(dqm); 1636 1637 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1638 retval = pm_init(&dqm->packet_mgr, dqm); 1639 if (retval) 1640 goto fail_packet_manager_init; 1641 1642 retval = set_sched_resources(dqm); 1643 if (retval) 1644 goto fail_set_sched_resources; 1645 } 1646 pr_debug("Allocating fence memory\n"); 1647 1648 /* allocate fence memory on the gart */ 1649 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 1650 &dqm->fence_mem); 1651 1652 if (retval) 1653 goto fail_allocate_vidmem; 1654 1655 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; 1656 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 1657 1658 init_interrupts(dqm); 1659 1660 /* clear hang status when driver try to start the hw scheduler */ 1661 dqm->sched_running = true; 1662 1663 if (!dqm->dev->kfd->shared_resources.enable_mes) 1664 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1665 1666 /* Set CWSR grace period to 1x1000 cycle for GFX9.4.3 APU */ 1667 if (amdgpu_emu_mode == 0 && dqm->dev->adev->gmc.is_app_apu && 1668 (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 3))) { 1669 uint32_t reg_offset = 0; 1670 uint32_t grace_period = 1; 1671 1672 retval = pm_update_grace_period(&dqm->packet_mgr, 1673 grace_period); 1674 if (retval) 1675 dev_err(dev, "Setting grace timeout failed\n"); 1676 else if (dqm->dev->kfd2kgd->build_grace_period_packet_info) 1677 /* Update dqm->wait_times maintained in software */ 1678 dqm->dev->kfd2kgd->build_grace_period_packet_info( 1679 dqm->dev->adev, dqm->wait_times, 1680 grace_period, ®_offset, 1681 &dqm->wait_times); 1682 } 1683 1684 dqm_unlock(dqm); 1685 1686 return 0; 1687 fail_allocate_vidmem: 1688 fail_set_sched_resources: 1689 if (!dqm->dev->kfd->shared_resources.enable_mes) 1690 pm_uninit(&dqm->packet_mgr); 1691 fail_packet_manager_init: 1692 dqm_unlock(dqm); 1693 return retval; 1694 } 1695 1696 static int stop_cpsch(struct device_queue_manager *dqm) 1697 { 1698 dqm_lock(dqm); 1699 if (!dqm->sched_running) { 1700 dqm_unlock(dqm); 1701 return 0; 1702 } 1703 1704 if (!dqm->dev->kfd->shared_resources.enable_mes) 1705 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 1706 else 1707 remove_all_queues_mes(dqm); 1708 1709 dqm->sched_running = false; 1710 1711 if (!dqm->dev->kfd->shared_resources.enable_mes) 1712 pm_release_ib(&dqm->packet_mgr); 1713 1714 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1715 if (!dqm->dev->kfd->shared_resources.enable_mes) 1716 pm_uninit(&dqm->packet_mgr); 1717 dqm_unlock(dqm); 1718 1719 return 0; 1720 } 1721 1722 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 1723 struct kernel_queue *kq, 1724 struct qcm_process_device *qpd) 1725 { 1726 dqm_lock(dqm); 1727 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1728 pr_warn("Can't create new kernel queue because %d queues were already created\n", 1729 dqm->total_queue_count); 1730 dqm_unlock(dqm); 1731 return -EPERM; 1732 } 1733 1734 /* 1735 * Unconditionally increment this counter, regardless of the queue's 1736 * type or whether the queue is active. 1737 */ 1738 dqm->total_queue_count++; 1739 pr_debug("Total of %d queues are accountable so far\n", 1740 dqm->total_queue_count); 1741 1742 list_add(&kq->list, &qpd->priv_queue_list); 1743 increment_queue_count(dqm, qpd, kq->queue); 1744 qpd->is_debug = true; 1745 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1746 USE_DEFAULT_GRACE_PERIOD); 1747 dqm_unlock(dqm); 1748 1749 return 0; 1750 } 1751 1752 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 1753 struct kernel_queue *kq, 1754 struct qcm_process_device *qpd) 1755 { 1756 dqm_lock(dqm); 1757 list_del(&kq->list); 1758 decrement_queue_count(dqm, qpd, kq->queue); 1759 qpd->is_debug = false; 1760 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1761 USE_DEFAULT_GRACE_PERIOD); 1762 /* 1763 * Unconditionally decrement this counter, regardless of the queue's 1764 * type. 1765 */ 1766 dqm->total_queue_count--; 1767 pr_debug("Total of %d queues are accountable so far\n", 1768 dqm->total_queue_count); 1769 dqm_unlock(dqm); 1770 } 1771 1772 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 1773 struct qcm_process_device *qpd, 1774 const struct kfd_criu_queue_priv_data *qd, 1775 const void *restore_mqd, const void *restore_ctl_stack) 1776 { 1777 int retval; 1778 struct mqd_manager *mqd_mgr; 1779 1780 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1781 pr_warn("Can't create new usermode queue because %d queues were already created\n", 1782 dqm->total_queue_count); 1783 retval = -EPERM; 1784 goto out; 1785 } 1786 1787 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1788 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1789 dqm_lock(dqm); 1790 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 1791 dqm_unlock(dqm); 1792 if (retval) 1793 goto out; 1794 } 1795 1796 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 1797 if (retval) 1798 goto out_deallocate_sdma_queue; 1799 1800 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1801 q->properties.type)]; 1802 1803 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1804 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 1805 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 1806 q->properties.tba_addr = qpd->tba_addr; 1807 q->properties.tma_addr = qpd->tma_addr; 1808 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 1809 if (!q->mqd_mem_obj) { 1810 retval = -ENOMEM; 1811 goto out_deallocate_doorbell; 1812 } 1813 1814 dqm_lock(dqm); 1815 /* 1816 * Eviction state logic: mark all queues as evicted, even ones 1817 * not currently active. Restoring inactive queues later only 1818 * updates the is_evicted flag but is a no-op otherwise. 1819 */ 1820 q->properties.is_evicted = !!qpd->evicted; 1821 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && 1822 kfd_dbg_has_cwsr_workaround(q->device); 1823 1824 if (qd) 1825 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 1826 &q->properties, restore_mqd, restore_ctl_stack, 1827 qd->ctl_stack_size); 1828 else 1829 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 1830 &q->gart_mqd_addr, &q->properties); 1831 1832 list_add(&q->list, &qpd->queues_list); 1833 qpd->queue_count++; 1834 1835 if (q->properties.is_active) { 1836 increment_queue_count(dqm, qpd, q); 1837 1838 if (!dqm->dev->kfd->shared_resources.enable_mes) 1839 retval = execute_queues_cpsch(dqm, 1840 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1841 else 1842 retval = add_queue_mes(dqm, q, qpd); 1843 if (retval) 1844 goto cleanup_queue; 1845 } 1846 1847 /* 1848 * Unconditionally increment this counter, regardless of the queue's 1849 * type or whether the queue is active. 1850 */ 1851 dqm->total_queue_count++; 1852 1853 pr_debug("Total of %d queues are accountable so far\n", 1854 dqm->total_queue_count); 1855 1856 dqm_unlock(dqm); 1857 return retval; 1858 1859 cleanup_queue: 1860 qpd->queue_count--; 1861 list_del(&q->list); 1862 if (q->properties.is_active) 1863 decrement_queue_count(dqm, qpd, q); 1864 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 1865 dqm_unlock(dqm); 1866 out_deallocate_doorbell: 1867 deallocate_doorbell(qpd, q); 1868 out_deallocate_sdma_queue: 1869 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1870 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1871 dqm_lock(dqm); 1872 deallocate_sdma_queue(dqm, q); 1873 dqm_unlock(dqm); 1874 } 1875 out: 1876 return retval; 1877 } 1878 1879 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, 1880 uint64_t fence_value, 1881 unsigned int timeout_ms) 1882 { 1883 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies; 1884 struct device *dev = dqm->dev->adev->dev; 1885 uint64_t *fence_addr = dqm->fence_addr; 1886 1887 while (*fence_addr != fence_value) { 1888 /* Fatal err detected, this response won't come */ 1889 if (amdgpu_amdkfd_is_fed(dqm->dev->adev)) 1890 return -EIO; 1891 1892 if (time_after(jiffies, end_jiffies)) { 1893 dev_err(dev, "qcm fence wait loop timeout expired\n"); 1894 /* In HWS case, this is used to halt the driver thread 1895 * in order not to mess up CP states before doing 1896 * scandumps for FW debugging. 1897 */ 1898 while (halt_if_hws_hang) 1899 schedule(); 1900 1901 return -ETIME; 1902 } 1903 schedule(); 1904 } 1905 1906 return 0; 1907 } 1908 1909 /* dqm->lock mutex has to be locked before calling this function */ 1910 static int map_queues_cpsch(struct device_queue_manager *dqm) 1911 { 1912 struct device *dev = dqm->dev->adev->dev; 1913 int retval; 1914 1915 if (!dqm->sched_running) 1916 return 0; 1917 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) 1918 return 0; 1919 if (dqm->active_runlist) 1920 return 0; 1921 1922 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); 1923 pr_debug("%s sent runlist\n", __func__); 1924 if (retval) { 1925 dev_err(dev, "failed to execute runlist\n"); 1926 return retval; 1927 } 1928 dqm->active_runlist = true; 1929 1930 return retval; 1931 } 1932 1933 /* dqm->lock mutex has to be locked before calling this function */ 1934 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 1935 enum kfd_unmap_queues_filter filter, 1936 uint32_t filter_param, 1937 uint32_t grace_period, 1938 bool reset) 1939 { 1940 struct device *dev = dqm->dev->adev->dev; 1941 struct mqd_manager *mqd_mgr; 1942 int retval; 1943 1944 if (!dqm->sched_running) 1945 return 0; 1946 if (!dqm->active_runlist) 1947 return 0; 1948 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 1949 return -EIO; 1950 1951 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 1952 retval = pm_update_grace_period(&dqm->packet_mgr, grace_period); 1953 if (retval) 1954 goto out; 1955 } 1956 1957 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); 1958 if (retval) 1959 goto out; 1960 1961 *dqm->fence_addr = KFD_FENCE_INIT; 1962 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, 1963 KFD_FENCE_COMPLETED); 1964 /* should be timed out */ 1965 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, 1966 queue_preemption_timeout_ms); 1967 if (retval) { 1968 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n"); 1969 kfd_hws_hang(dqm); 1970 goto out; 1971 } 1972 1973 /* In the current MEC firmware implementation, if compute queue 1974 * doesn't response to the preemption request in time, HIQ will 1975 * abandon the unmap request without returning any timeout error 1976 * to driver. Instead, MEC firmware will log the doorbell of the 1977 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields. 1978 * To make sure the queue unmap was successful, driver need to 1979 * check those fields 1980 */ 1981 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; 1982 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd)) { 1983 while (halt_if_hws_hang) 1984 schedule(); 1985 kfd_hws_hang(dqm); 1986 retval = -ETIME; 1987 goto out; 1988 } 1989 1990 /* We need to reset the grace period value for this device */ 1991 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 1992 if (pm_update_grace_period(&dqm->packet_mgr, 1993 USE_DEFAULT_GRACE_PERIOD)) 1994 dev_err(dev, "Failed to reset grace period\n"); 1995 } 1996 1997 pm_release_ib(&dqm->packet_mgr); 1998 dqm->active_runlist = false; 1999 2000 out: 2001 up_read(&dqm->dev->adev->reset_domain->sem); 2002 return retval; 2003 } 2004 2005 /* only for compute queue */ 2006 static int reset_queues_cpsch(struct device_queue_manager *dqm, 2007 uint16_t pasid) 2008 { 2009 int retval; 2010 2011 dqm_lock(dqm); 2012 2013 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, 2014 pasid, USE_DEFAULT_GRACE_PERIOD, true); 2015 2016 dqm_unlock(dqm); 2017 return retval; 2018 } 2019 2020 /* dqm->lock mutex has to be locked before calling this function */ 2021 static int execute_queues_cpsch(struct device_queue_manager *dqm, 2022 enum kfd_unmap_queues_filter filter, 2023 uint32_t filter_param, 2024 uint32_t grace_period) 2025 { 2026 int retval; 2027 2028 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2029 return -EIO; 2030 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); 2031 if (!retval) 2032 retval = map_queues_cpsch(dqm); 2033 up_read(&dqm->dev->adev->reset_domain->sem); 2034 return retval; 2035 } 2036 2037 static int wait_on_destroy_queue(struct device_queue_manager *dqm, 2038 struct queue *q) 2039 { 2040 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device, 2041 q->process); 2042 int ret = 0; 2043 2044 if (pdd->qpd.is_debug) 2045 return ret; 2046 2047 q->properties.is_being_destroyed = true; 2048 2049 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { 2050 dqm_unlock(dqm); 2051 mutex_unlock(&q->process->mutex); 2052 ret = wait_event_interruptible(dqm->destroy_wait, 2053 !q->properties.is_suspended); 2054 2055 mutex_lock(&q->process->mutex); 2056 dqm_lock(dqm); 2057 } 2058 2059 return ret; 2060 } 2061 2062 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 2063 struct qcm_process_device *qpd, 2064 struct queue *q) 2065 { 2066 int retval; 2067 struct mqd_manager *mqd_mgr; 2068 uint64_t sdma_val = 0; 2069 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2070 struct device *dev = dqm->dev->adev->dev; 2071 2072 /* Get the SDMA queue stats */ 2073 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2074 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2075 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 2076 &sdma_val); 2077 if (retval) 2078 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 2079 q->properties.queue_id); 2080 } 2081 2082 /* remove queue from list to prevent rescheduling after preemption */ 2083 dqm_lock(dqm); 2084 2085 retval = wait_on_destroy_queue(dqm, q); 2086 2087 if (retval) { 2088 dqm_unlock(dqm); 2089 return retval; 2090 } 2091 2092 if (qpd->is_debug) { 2093 /* 2094 * error, currently we do not allow to destroy a queue 2095 * of a currently debugged process 2096 */ 2097 retval = -EBUSY; 2098 goto failed_try_destroy_debugged_queue; 2099 2100 } 2101 2102 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2103 q->properties.type)]; 2104 2105 deallocate_doorbell(qpd, q); 2106 2107 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2108 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2109 deallocate_sdma_queue(dqm, q); 2110 pdd->sdma_past_activity_counter += sdma_val; 2111 } 2112 2113 list_del(&q->list); 2114 qpd->queue_count--; 2115 if (q->properties.is_active) { 2116 decrement_queue_count(dqm, qpd, q); 2117 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2118 retval = execute_queues_cpsch(dqm, 2119 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2120 USE_DEFAULT_GRACE_PERIOD); 2121 if (retval == -ETIME) 2122 qpd->reset_wavefronts = true; 2123 } else { 2124 retval = remove_queue_mes(dqm, q, qpd); 2125 } 2126 } 2127 2128 /* 2129 * Unconditionally decrement this counter, regardless of the queue's 2130 * type 2131 */ 2132 dqm->total_queue_count--; 2133 pr_debug("Total of %d queues are accountable so far\n", 2134 dqm->total_queue_count); 2135 2136 dqm_unlock(dqm); 2137 2138 /* 2139 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid 2140 * circular locking 2141 */ 2142 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE), 2143 qpd->pqm->process, q->device, 2144 -1, false, NULL, 0); 2145 2146 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2147 2148 return retval; 2149 2150 failed_try_destroy_debugged_queue: 2151 2152 dqm_unlock(dqm); 2153 return retval; 2154 } 2155 2156 /* 2157 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 2158 * stay in user mode. 2159 */ 2160 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 2161 /* APE1 limit is inclusive and 64K aligned. */ 2162 #define APE1_LIMIT_ALIGNMENT 0xFFFF 2163 2164 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 2165 struct qcm_process_device *qpd, 2166 enum cache_policy default_policy, 2167 enum cache_policy alternate_policy, 2168 void __user *alternate_aperture_base, 2169 uint64_t alternate_aperture_size) 2170 { 2171 bool retval = true; 2172 2173 if (!dqm->asic_ops.set_cache_memory_policy) 2174 return retval; 2175 2176 dqm_lock(dqm); 2177 2178 if (alternate_aperture_size == 0) { 2179 /* base > limit disables APE1 */ 2180 qpd->sh_mem_ape1_base = 1; 2181 qpd->sh_mem_ape1_limit = 0; 2182 } else { 2183 /* 2184 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 2185 * SH_MEM_APE1_BASE[31:0], 0x0000 } 2186 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 2187 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 2188 * Verify that the base and size parameters can be 2189 * represented in this format and convert them. 2190 * Additionally restrict APE1 to user-mode addresses. 2191 */ 2192 2193 uint64_t base = (uintptr_t)alternate_aperture_base; 2194 uint64_t limit = base + alternate_aperture_size - 1; 2195 2196 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 || 2197 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) { 2198 retval = false; 2199 goto out; 2200 } 2201 2202 qpd->sh_mem_ape1_base = base >> 16; 2203 qpd->sh_mem_ape1_limit = limit >> 16; 2204 } 2205 2206 retval = dqm->asic_ops.set_cache_memory_policy( 2207 dqm, 2208 qpd, 2209 default_policy, 2210 alternate_policy, 2211 alternate_aperture_base, 2212 alternate_aperture_size); 2213 2214 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 2215 program_sh_mem_settings(dqm, qpd); 2216 2217 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 2218 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 2219 qpd->sh_mem_ape1_limit); 2220 2221 out: 2222 dqm_unlock(dqm); 2223 return retval; 2224 } 2225 2226 static int process_termination_nocpsch(struct device_queue_manager *dqm, 2227 struct qcm_process_device *qpd) 2228 { 2229 struct queue *q; 2230 struct device_process_node *cur, *next_dpn; 2231 int retval = 0; 2232 bool found = false; 2233 2234 dqm_lock(dqm); 2235 2236 /* Clear all user mode queues */ 2237 while (!list_empty(&qpd->queues_list)) { 2238 struct mqd_manager *mqd_mgr; 2239 int ret; 2240 2241 q = list_first_entry(&qpd->queues_list, struct queue, list); 2242 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2243 q->properties.type)]; 2244 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); 2245 if (ret) 2246 retval = ret; 2247 dqm_unlock(dqm); 2248 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2249 dqm_lock(dqm); 2250 } 2251 2252 /* Unregister process */ 2253 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2254 if (qpd == cur->qpd) { 2255 list_del(&cur->list); 2256 kfree(cur); 2257 dqm->processes_count--; 2258 found = true; 2259 break; 2260 } 2261 } 2262 2263 dqm_unlock(dqm); 2264 2265 /* Outside the DQM lock because under the DQM lock we can't do 2266 * reclaim or take other locks that others hold while reclaiming. 2267 */ 2268 if (found) 2269 kfd_dec_compute_active(dqm->dev); 2270 2271 return retval; 2272 } 2273 2274 static int get_wave_state(struct device_queue_manager *dqm, 2275 struct queue *q, 2276 void __user *ctl_stack, 2277 u32 *ctl_stack_used_size, 2278 u32 *save_area_used_size) 2279 { 2280 struct mqd_manager *mqd_mgr; 2281 2282 dqm_lock(dqm); 2283 2284 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 2285 2286 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE || 2287 q->properties.is_active || !q->device->kfd->cwsr_enabled || 2288 !mqd_mgr->get_wave_state) { 2289 dqm_unlock(dqm); 2290 return -EINVAL; 2291 } 2292 2293 dqm_unlock(dqm); 2294 2295 /* 2296 * get_wave_state is outside the dqm lock to prevent circular locking 2297 * and the queue should be protected against destruction by the process 2298 * lock. 2299 */ 2300 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties, 2301 ctl_stack, ctl_stack_used_size, save_area_used_size); 2302 } 2303 2304 static void get_queue_checkpoint_info(struct device_queue_manager *dqm, 2305 const struct queue *q, 2306 u32 *mqd_size, 2307 u32 *ctl_stack_size) 2308 { 2309 struct mqd_manager *mqd_mgr; 2310 enum KFD_MQD_TYPE mqd_type = 2311 get_mqd_type_from_queue_type(q->properties.type); 2312 2313 dqm_lock(dqm); 2314 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2315 *mqd_size = mqd_mgr->mqd_size; 2316 *ctl_stack_size = 0; 2317 2318 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) 2319 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size); 2320 2321 dqm_unlock(dqm); 2322 } 2323 2324 static int checkpoint_mqd(struct device_queue_manager *dqm, 2325 const struct queue *q, 2326 void *mqd, 2327 void *ctl_stack) 2328 { 2329 struct mqd_manager *mqd_mgr; 2330 int r = 0; 2331 enum KFD_MQD_TYPE mqd_type = 2332 get_mqd_type_from_queue_type(q->properties.type); 2333 2334 dqm_lock(dqm); 2335 2336 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) { 2337 r = -EINVAL; 2338 goto dqm_unlock; 2339 } 2340 2341 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2342 if (!mqd_mgr->checkpoint_mqd) { 2343 r = -EOPNOTSUPP; 2344 goto dqm_unlock; 2345 } 2346 2347 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack); 2348 2349 dqm_unlock: 2350 dqm_unlock(dqm); 2351 return r; 2352 } 2353 2354 static int process_termination_cpsch(struct device_queue_manager *dqm, 2355 struct qcm_process_device *qpd) 2356 { 2357 int retval; 2358 struct queue *q; 2359 struct device *dev = dqm->dev->adev->dev; 2360 struct kernel_queue *kq, *kq_next; 2361 struct mqd_manager *mqd_mgr; 2362 struct device_process_node *cur, *next_dpn; 2363 enum kfd_unmap_queues_filter filter = 2364 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES; 2365 bool found = false; 2366 2367 retval = 0; 2368 2369 dqm_lock(dqm); 2370 2371 /* Clean all kernel queues */ 2372 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { 2373 list_del(&kq->list); 2374 decrement_queue_count(dqm, qpd, kq->queue); 2375 qpd->is_debug = false; 2376 dqm->total_queue_count--; 2377 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES; 2378 } 2379 2380 /* Clear all user mode queues */ 2381 list_for_each_entry(q, &qpd->queues_list, list) { 2382 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 2383 deallocate_sdma_queue(dqm, q); 2384 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2385 deallocate_sdma_queue(dqm, q); 2386 2387 if (q->properties.is_active) { 2388 decrement_queue_count(dqm, qpd, q); 2389 2390 if (dqm->dev->kfd->shared_resources.enable_mes) { 2391 retval = remove_queue_mes(dqm, q, qpd); 2392 if (retval) 2393 dev_err(dev, "Failed to remove queue %d\n", 2394 q->properties.queue_id); 2395 } 2396 } 2397 2398 dqm->total_queue_count--; 2399 } 2400 2401 /* Unregister process */ 2402 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2403 if (qpd == cur->qpd) { 2404 list_del(&cur->list); 2405 kfree(cur); 2406 dqm->processes_count--; 2407 found = true; 2408 break; 2409 } 2410 } 2411 2412 if (!dqm->dev->kfd->shared_resources.enable_mes) 2413 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); 2414 2415 if ((retval || qpd->reset_wavefronts) && 2416 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 2417 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); 2418 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); 2419 qpd->reset_wavefronts = false; 2420 up_read(&dqm->dev->adev->reset_domain->sem); 2421 } 2422 2423 /* Lastly, free mqd resources. 2424 * Do free_mqd() after dqm_unlock to avoid circular locking. 2425 */ 2426 while (!list_empty(&qpd->queues_list)) { 2427 q = list_first_entry(&qpd->queues_list, struct queue, list); 2428 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2429 q->properties.type)]; 2430 list_del(&q->list); 2431 qpd->queue_count--; 2432 dqm_unlock(dqm); 2433 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2434 dqm_lock(dqm); 2435 } 2436 dqm_unlock(dqm); 2437 2438 /* Outside the DQM lock because under the DQM lock we can't do 2439 * reclaim or take other locks that others hold while reclaiming. 2440 */ 2441 if (found) 2442 kfd_dec_compute_active(dqm->dev); 2443 2444 return retval; 2445 } 2446 2447 static int init_mqd_managers(struct device_queue_manager *dqm) 2448 { 2449 int i, j; 2450 struct device *dev = dqm->dev->adev->dev; 2451 struct mqd_manager *mqd_mgr; 2452 2453 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) { 2454 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); 2455 if (!mqd_mgr) { 2456 dev_err(dev, "mqd manager [%d] initialization failed\n", i); 2457 goto out_free; 2458 } 2459 dqm->mqd_mgrs[i] = mqd_mgr; 2460 } 2461 2462 return 0; 2463 2464 out_free: 2465 for (j = 0; j < i; j++) { 2466 kfree(dqm->mqd_mgrs[j]); 2467 dqm->mqd_mgrs[j] = NULL; 2468 } 2469 2470 return -ENOMEM; 2471 } 2472 2473 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/ 2474 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) 2475 { 2476 int retval; 2477 struct kfd_node *dev = dqm->dev; 2478 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; 2479 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * 2480 get_num_all_sdma_engines(dqm) * 2481 dev->kfd->device_info.num_sdma_queues_per_engine + 2482 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * 2483 NUM_XCC(dqm->dev->xcc_mask)); 2484 2485 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size, 2486 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr), 2487 (void *)&(mem_obj->cpu_ptr), false); 2488 2489 return retval; 2490 } 2491 2492 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) 2493 { 2494 struct device_queue_manager *dqm; 2495 2496 pr_debug("Loading device queue manager\n"); 2497 2498 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL); 2499 if (!dqm) 2500 return NULL; 2501 2502 switch (dev->adev->asic_type) { 2503 /* HWS is not available on Hawaii. */ 2504 case CHIP_HAWAII: 2505 /* HWS depends on CWSR for timely dequeue. CWSR is not 2506 * available on Tonga. 2507 * 2508 * FIXME: This argument also applies to Kaveri. 2509 */ 2510 case CHIP_TONGA: 2511 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; 2512 break; 2513 default: 2514 dqm->sched_policy = sched_policy; 2515 break; 2516 } 2517 2518 dqm->dev = dev; 2519 switch (dqm->sched_policy) { 2520 case KFD_SCHED_POLICY_HWS: 2521 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 2522 /* initialize dqm for cp scheduling */ 2523 dqm->ops.create_queue = create_queue_cpsch; 2524 dqm->ops.initialize = initialize_cpsch; 2525 dqm->ops.start = start_cpsch; 2526 dqm->ops.stop = stop_cpsch; 2527 dqm->ops.destroy_queue = destroy_queue_cpsch; 2528 dqm->ops.update_queue = update_queue; 2529 dqm->ops.register_process = register_process; 2530 dqm->ops.unregister_process = unregister_process; 2531 dqm->ops.uninitialize = uninitialize; 2532 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 2533 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 2534 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2535 dqm->ops.process_termination = process_termination_cpsch; 2536 dqm->ops.evict_process_queues = evict_process_queues_cpsch; 2537 dqm->ops.restore_process_queues = restore_process_queues_cpsch; 2538 dqm->ops.get_wave_state = get_wave_state; 2539 dqm->ops.reset_queues = reset_queues_cpsch; 2540 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2541 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2542 break; 2543 case KFD_SCHED_POLICY_NO_HWS: 2544 /* initialize dqm for no cp scheduling */ 2545 dqm->ops.start = start_nocpsch; 2546 dqm->ops.stop = stop_nocpsch; 2547 dqm->ops.create_queue = create_queue_nocpsch; 2548 dqm->ops.destroy_queue = destroy_queue_nocpsch; 2549 dqm->ops.update_queue = update_queue; 2550 dqm->ops.register_process = register_process; 2551 dqm->ops.unregister_process = unregister_process; 2552 dqm->ops.initialize = initialize_nocpsch; 2553 dqm->ops.uninitialize = uninitialize; 2554 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2555 dqm->ops.process_termination = process_termination_nocpsch; 2556 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; 2557 dqm->ops.restore_process_queues = 2558 restore_process_queues_nocpsch; 2559 dqm->ops.get_wave_state = get_wave_state; 2560 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2561 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2562 break; 2563 default: 2564 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); 2565 goto out_free; 2566 } 2567 2568 switch (dev->adev->asic_type) { 2569 case CHIP_KAVERI: 2570 case CHIP_HAWAII: 2571 device_queue_manager_init_cik(&dqm->asic_ops); 2572 break; 2573 2574 case CHIP_CARRIZO: 2575 case CHIP_TONGA: 2576 case CHIP_FIJI: 2577 case CHIP_POLARIS10: 2578 case CHIP_POLARIS11: 2579 case CHIP_POLARIS12: 2580 case CHIP_VEGAM: 2581 device_queue_manager_init_vi(&dqm->asic_ops); 2582 break; 2583 2584 default: 2585 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0)) 2586 device_queue_manager_init_v12(&dqm->asic_ops); 2587 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0)) 2588 device_queue_manager_init_v11(&dqm->asic_ops); 2589 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1)) 2590 device_queue_manager_init_v10(&dqm->asic_ops); 2591 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) 2592 device_queue_manager_init_v9(&dqm->asic_ops); 2593 else { 2594 WARN(1, "Unexpected ASIC family %u", 2595 dev->adev->asic_type); 2596 goto out_free; 2597 } 2598 } 2599 2600 if (init_mqd_managers(dqm)) 2601 goto out_free; 2602 2603 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { 2604 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n"); 2605 goto out_free; 2606 } 2607 2608 if (!dqm->ops.initialize(dqm)) { 2609 init_waitqueue_head(&dqm->destroy_wait); 2610 return dqm; 2611 } 2612 2613 out_free: 2614 kfree(dqm); 2615 return NULL; 2616 } 2617 2618 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, 2619 struct kfd_mem_obj *mqd) 2620 { 2621 WARN(!mqd, "No hiq sdma mqd trunk to free"); 2622 2623 amdgpu_amdkfd_free_gtt_mem(dev->adev, mqd->gtt_mem); 2624 } 2625 2626 void device_queue_manager_uninit(struct device_queue_manager *dqm) 2627 { 2628 dqm->ops.stop(dqm); 2629 dqm->ops.uninitialize(dqm); 2630 if (!dqm->dev->kfd->shared_resources.enable_mes) 2631 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); 2632 kfree(dqm); 2633 } 2634 2635 int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid) 2636 { 2637 struct kfd_process_device *pdd; 2638 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 2639 int ret = 0; 2640 2641 if (!p) 2642 return -EINVAL; 2643 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 2644 pdd = kfd_get_process_device_data(dqm->dev, p); 2645 if (pdd) 2646 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); 2647 kfd_unref_process(p); 2648 2649 return ret; 2650 } 2651 2652 static void kfd_process_hw_exception(struct work_struct *work) 2653 { 2654 struct device_queue_manager *dqm = container_of(work, 2655 struct device_queue_manager, hw_exception_work); 2656 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); 2657 } 2658 2659 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 2660 struct qcm_process_device *qpd) 2661 { 2662 int r; 2663 struct device *dev = dqm->dev->adev->dev; 2664 int updated_vmid_mask; 2665 2666 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 2667 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 2668 return -EINVAL; 2669 } 2670 2671 dqm_lock(dqm); 2672 2673 if (dqm->trap_debug_vmid != 0) { 2674 dev_err(dev, "Trap debug id already reserved\n"); 2675 r = -EBUSY; 2676 goto out_unlock; 2677 } 2678 2679 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 2680 USE_DEFAULT_GRACE_PERIOD, false); 2681 if (r) 2682 goto out_unlock; 2683 2684 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 2685 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); 2686 2687 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 2688 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; 2689 r = set_sched_resources(dqm); 2690 if (r) 2691 goto out_unlock; 2692 2693 r = map_queues_cpsch(dqm); 2694 if (r) 2695 goto out_unlock; 2696 2697 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); 2698 2699 out_unlock: 2700 dqm_unlock(dqm); 2701 return r; 2702 } 2703 2704 /* 2705 * Releases vmid for the trap debugger 2706 */ 2707 int release_debug_trap_vmid(struct device_queue_manager *dqm, 2708 struct qcm_process_device *qpd) 2709 { 2710 struct device *dev = dqm->dev->adev->dev; 2711 int r; 2712 int updated_vmid_mask; 2713 uint32_t trap_debug_vmid; 2714 2715 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 2716 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 2717 return -EINVAL; 2718 } 2719 2720 dqm_lock(dqm); 2721 trap_debug_vmid = dqm->trap_debug_vmid; 2722 if (dqm->trap_debug_vmid == 0) { 2723 dev_err(dev, "Trap debug id is not reserved\n"); 2724 r = -EINVAL; 2725 goto out_unlock; 2726 } 2727 2728 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 2729 USE_DEFAULT_GRACE_PERIOD, false); 2730 if (r) 2731 goto out_unlock; 2732 2733 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 2734 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); 2735 2736 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 2737 dqm->trap_debug_vmid = 0; 2738 r = set_sched_resources(dqm); 2739 if (r) 2740 goto out_unlock; 2741 2742 r = map_queues_cpsch(dqm); 2743 if (r) 2744 goto out_unlock; 2745 2746 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid); 2747 2748 out_unlock: 2749 dqm_unlock(dqm); 2750 return r; 2751 } 2752 2753 #define QUEUE_NOT_FOUND -1 2754 /* invalidate queue operation in array */ 2755 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids) 2756 { 2757 int i; 2758 2759 for (i = 0; i < num_queues; i++) 2760 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK; 2761 } 2762 2763 /* find queue index in array */ 2764 static int q_array_get_index(unsigned int queue_id, 2765 uint32_t num_queues, 2766 uint32_t *queue_ids) 2767 { 2768 int i; 2769 2770 for (i = 0; i < num_queues; i++) 2771 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK)) 2772 return i; 2773 2774 return QUEUE_NOT_FOUND; 2775 } 2776 2777 struct copy_context_work_handler_workarea { 2778 struct work_struct copy_context_work; 2779 struct kfd_process *p; 2780 }; 2781 2782 static void copy_context_work_handler (struct work_struct *work) 2783 { 2784 struct copy_context_work_handler_workarea *workarea; 2785 struct mqd_manager *mqd_mgr; 2786 struct queue *q; 2787 struct mm_struct *mm; 2788 struct kfd_process *p; 2789 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size; 2790 int i; 2791 2792 workarea = container_of(work, 2793 struct copy_context_work_handler_workarea, 2794 copy_context_work); 2795 2796 p = workarea->p; 2797 mm = get_task_mm(p->lead_thread); 2798 2799 if (!mm) 2800 return; 2801 2802 kthread_use_mm(mm); 2803 for (i = 0; i < p->n_pdds; i++) { 2804 struct kfd_process_device *pdd = p->pdds[i]; 2805 struct device_queue_manager *dqm = pdd->dev->dqm; 2806 struct qcm_process_device *qpd = &pdd->qpd; 2807 2808 list_for_each_entry(q, &qpd->queues_list, list) { 2809 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 2810 2811 /* We ignore the return value from get_wave_state 2812 * because 2813 * i) right now, it always returns 0, and 2814 * ii) if we hit an error, we would continue to the 2815 * next queue anyway. 2816 */ 2817 mqd_mgr->get_wave_state(mqd_mgr, 2818 q->mqd, 2819 &q->properties, 2820 (void __user *) q->properties.ctx_save_restore_area_address, 2821 &tmp_ctl_stack_used_size, 2822 &tmp_save_area_used_size); 2823 } 2824 } 2825 kthread_unuse_mm(mm); 2826 mmput(mm); 2827 } 2828 2829 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array) 2830 { 2831 size_t array_size = num_queues * sizeof(uint32_t); 2832 2833 if (!usr_queue_id_array) 2834 return NULL; 2835 2836 return memdup_user(usr_queue_id_array, array_size); 2837 } 2838 2839 int resume_queues(struct kfd_process *p, 2840 uint32_t num_queues, 2841 uint32_t *usr_queue_id_array) 2842 { 2843 uint32_t *queue_ids = NULL; 2844 int total_resumed = 0; 2845 int i; 2846 2847 if (usr_queue_id_array) { 2848 queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 2849 2850 if (IS_ERR(queue_ids)) 2851 return PTR_ERR(queue_ids); 2852 2853 /* mask all queues as invalid. unmask per successful request */ 2854 q_array_invalidate(num_queues, queue_ids); 2855 } 2856 2857 for (i = 0; i < p->n_pdds; i++) { 2858 struct kfd_process_device *pdd = p->pdds[i]; 2859 struct device_queue_manager *dqm = pdd->dev->dqm; 2860 struct device *dev = dqm->dev->adev->dev; 2861 struct qcm_process_device *qpd = &pdd->qpd; 2862 struct queue *q; 2863 int r, per_device_resumed = 0; 2864 2865 dqm_lock(dqm); 2866 2867 /* unmask queues that resume or already resumed as valid */ 2868 list_for_each_entry(q, &qpd->queues_list, list) { 2869 int q_idx = QUEUE_NOT_FOUND; 2870 2871 if (queue_ids) 2872 q_idx = q_array_get_index( 2873 q->properties.queue_id, 2874 num_queues, 2875 queue_ids); 2876 2877 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) { 2878 int err = resume_single_queue(dqm, &pdd->qpd, q); 2879 2880 if (queue_ids) { 2881 if (!err) { 2882 queue_ids[q_idx] &= 2883 ~KFD_DBG_QUEUE_INVALID_MASK; 2884 } else { 2885 queue_ids[q_idx] |= 2886 KFD_DBG_QUEUE_ERROR_MASK; 2887 break; 2888 } 2889 } 2890 2891 if (dqm->dev->kfd->shared_resources.enable_mes) { 2892 wake_up_all(&dqm->destroy_wait); 2893 if (!err) 2894 total_resumed++; 2895 } else { 2896 per_device_resumed++; 2897 } 2898 } 2899 } 2900 2901 if (!per_device_resumed) { 2902 dqm_unlock(dqm); 2903 continue; 2904 } 2905 2906 r = execute_queues_cpsch(dqm, 2907 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 2908 0, 2909 USE_DEFAULT_GRACE_PERIOD); 2910 if (r) { 2911 dev_err(dev, "Failed to resume process queues\n"); 2912 if (queue_ids) { 2913 list_for_each_entry(q, &qpd->queues_list, list) { 2914 int q_idx = q_array_get_index( 2915 q->properties.queue_id, 2916 num_queues, 2917 queue_ids); 2918 2919 /* mask queue as error on resume fail */ 2920 if (q_idx != QUEUE_NOT_FOUND) 2921 queue_ids[q_idx] |= 2922 KFD_DBG_QUEUE_ERROR_MASK; 2923 } 2924 } 2925 } else { 2926 wake_up_all(&dqm->destroy_wait); 2927 total_resumed += per_device_resumed; 2928 } 2929 2930 dqm_unlock(dqm); 2931 } 2932 2933 if (queue_ids) { 2934 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 2935 num_queues * sizeof(uint32_t))) 2936 pr_err("copy_to_user failed on queue resume\n"); 2937 2938 kfree(queue_ids); 2939 } 2940 2941 return total_resumed; 2942 } 2943 2944 int suspend_queues(struct kfd_process *p, 2945 uint32_t num_queues, 2946 uint32_t grace_period, 2947 uint64_t exception_clear_mask, 2948 uint32_t *usr_queue_id_array) 2949 { 2950 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 2951 int total_suspended = 0; 2952 int i; 2953 2954 if (IS_ERR(queue_ids)) 2955 return PTR_ERR(queue_ids); 2956 2957 /* mask all queues as invalid. umask on successful request */ 2958 q_array_invalidate(num_queues, queue_ids); 2959 2960 for (i = 0; i < p->n_pdds; i++) { 2961 struct kfd_process_device *pdd = p->pdds[i]; 2962 struct device_queue_manager *dqm = pdd->dev->dqm; 2963 struct device *dev = dqm->dev->adev->dev; 2964 struct qcm_process_device *qpd = &pdd->qpd; 2965 struct queue *q; 2966 int r, per_device_suspended = 0; 2967 2968 mutex_lock(&p->event_mutex); 2969 dqm_lock(dqm); 2970 2971 /* unmask queues that suspend or already suspended */ 2972 list_for_each_entry(q, &qpd->queues_list, list) { 2973 int q_idx = q_array_get_index(q->properties.queue_id, 2974 num_queues, 2975 queue_ids); 2976 2977 if (q_idx != QUEUE_NOT_FOUND) { 2978 int err = suspend_single_queue(dqm, pdd, q); 2979 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; 2980 2981 if (!err) { 2982 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK; 2983 if (exception_clear_mask && is_mes) 2984 q->properties.exception_status &= 2985 ~exception_clear_mask; 2986 2987 if (is_mes) 2988 total_suspended++; 2989 else 2990 per_device_suspended++; 2991 } else if (err != -EBUSY) { 2992 r = err; 2993 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 2994 break; 2995 } 2996 } 2997 } 2998 2999 if (!per_device_suspended) { 3000 dqm_unlock(dqm); 3001 mutex_unlock(&p->event_mutex); 3002 if (total_suspended) 3003 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); 3004 continue; 3005 } 3006 3007 r = execute_queues_cpsch(dqm, 3008 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3009 grace_period); 3010 3011 if (r) 3012 dev_err(dev, "Failed to suspend process queues.\n"); 3013 else 3014 total_suspended += per_device_suspended; 3015 3016 list_for_each_entry(q, &qpd->queues_list, list) { 3017 int q_idx = q_array_get_index(q->properties.queue_id, 3018 num_queues, queue_ids); 3019 3020 if (q_idx == QUEUE_NOT_FOUND) 3021 continue; 3022 3023 /* mask queue as error on suspend fail */ 3024 if (r) 3025 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3026 else if (exception_clear_mask) 3027 q->properties.exception_status &= 3028 ~exception_clear_mask; 3029 } 3030 3031 dqm_unlock(dqm); 3032 mutex_unlock(&p->event_mutex); 3033 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); 3034 } 3035 3036 if (total_suspended) { 3037 struct copy_context_work_handler_workarea copy_context_worker; 3038 3039 INIT_WORK_ONSTACK( 3040 ©_context_worker.copy_context_work, 3041 copy_context_work_handler); 3042 3043 copy_context_worker.p = p; 3044 3045 schedule_work(©_context_worker.copy_context_work); 3046 3047 3048 flush_work(©_context_worker.copy_context_work); 3049 destroy_work_on_stack(©_context_worker.copy_context_work); 3050 } 3051 3052 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3053 num_queues * sizeof(uint32_t))) 3054 pr_err("copy_to_user failed on queue suspend\n"); 3055 3056 kfree(queue_ids); 3057 3058 return total_suspended; 3059 } 3060 3061 static uint32_t set_queue_type_for_user(struct queue_properties *q_props) 3062 { 3063 switch (q_props->type) { 3064 case KFD_QUEUE_TYPE_COMPUTE: 3065 return q_props->format == KFD_QUEUE_FORMAT_PM4 3066 ? KFD_IOC_QUEUE_TYPE_COMPUTE 3067 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; 3068 case KFD_QUEUE_TYPE_SDMA: 3069 return KFD_IOC_QUEUE_TYPE_SDMA; 3070 case KFD_QUEUE_TYPE_SDMA_XGMI: 3071 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI; 3072 default: 3073 WARN_ONCE(true, "queue type not recognized!"); 3074 return 0xffffffff; 3075 }; 3076 } 3077 3078 void set_queue_snapshot_entry(struct queue *q, 3079 uint64_t exception_clear_mask, 3080 struct kfd_queue_snapshot_entry *qss_entry) 3081 { 3082 qss_entry->ring_base_address = q->properties.queue_address; 3083 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr; 3084 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr; 3085 qss_entry->ctx_save_restore_address = 3086 q->properties.ctx_save_restore_area_address; 3087 qss_entry->ctx_save_restore_area_size = 3088 q->properties.ctx_save_restore_area_size; 3089 qss_entry->exception_status = q->properties.exception_status; 3090 qss_entry->queue_id = q->properties.queue_id; 3091 qss_entry->gpu_id = q->device->id; 3092 qss_entry->ring_size = (uint32_t)q->properties.queue_size; 3093 qss_entry->queue_type = set_queue_type_for_user(&q->properties); 3094 q->properties.exception_status &= ~exception_clear_mask; 3095 } 3096 3097 int debug_lock_and_unmap(struct device_queue_manager *dqm) 3098 { 3099 struct device *dev = dqm->dev->adev->dev; 3100 int r; 3101 3102 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3103 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3104 return -EINVAL; 3105 } 3106 3107 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3108 return 0; 3109 3110 dqm_lock(dqm); 3111 3112 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); 3113 if (r) 3114 dqm_unlock(dqm); 3115 3116 return r; 3117 } 3118 3119 int debug_map_and_unlock(struct device_queue_manager *dqm) 3120 { 3121 struct device *dev = dqm->dev->adev->dev; 3122 int r; 3123 3124 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3125 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3126 return -EINVAL; 3127 } 3128 3129 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3130 return 0; 3131 3132 r = map_queues_cpsch(dqm); 3133 3134 dqm_unlock(dqm); 3135 3136 return r; 3137 } 3138 3139 int debug_refresh_runlist(struct device_queue_manager *dqm) 3140 { 3141 int r = debug_lock_and_unmap(dqm); 3142 3143 if (r) 3144 return r; 3145 3146 return debug_map_and_unlock(dqm); 3147 } 3148 3149 #if defined(CONFIG_DEBUG_FS) 3150 3151 static void seq_reg_dump(struct seq_file *m, 3152 uint32_t (*dump)[2], uint32_t n_regs) 3153 { 3154 uint32_t i, count; 3155 3156 for (i = 0, count = 0; i < n_regs; i++) { 3157 if (count == 0 || 3158 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) { 3159 seq_printf(m, "%s %08x: %08x", 3160 i ? "\n" : "", 3161 dump[i][0], dump[i][1]); 3162 count = 7; 3163 } else { 3164 seq_printf(m, " %08x", dump[i][1]); 3165 count--; 3166 } 3167 } 3168 3169 seq_puts(m, "\n"); 3170 } 3171 3172 int dqm_debugfs_hqds(struct seq_file *m, void *data) 3173 { 3174 struct device_queue_manager *dqm = data; 3175 uint32_t xcc_mask = dqm->dev->xcc_mask; 3176 uint32_t (*dump)[2], n_regs; 3177 int pipe, queue; 3178 int r = 0, xcc_id; 3179 uint32_t sdma_engine_start; 3180 3181 if (!dqm->sched_running) { 3182 seq_puts(m, " Device is stopped\n"); 3183 return 0; 3184 } 3185 3186 for_each_inst(xcc_id, xcc_mask) { 3187 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3188 KFD_CIK_HIQ_PIPE, 3189 KFD_CIK_HIQ_QUEUE, &dump, 3190 &n_regs, xcc_id); 3191 if (!r) { 3192 seq_printf( 3193 m, 3194 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n", 3195 xcc_id, 3196 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, 3197 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), 3198 KFD_CIK_HIQ_QUEUE); 3199 seq_reg_dump(m, dump, n_regs); 3200 3201 kfree(dump); 3202 } 3203 3204 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 3205 int pipe_offset = pipe * get_queues_per_pipe(dqm); 3206 3207 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { 3208 if (!test_bit(pipe_offset + queue, 3209 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 3210 continue; 3211 3212 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3213 pipe, queue, 3214 &dump, &n_regs, 3215 xcc_id); 3216 if (r) 3217 break; 3218 3219 seq_printf(m, 3220 " Inst %d, CP Pipe %d, Queue %d\n", 3221 xcc_id, pipe, queue); 3222 seq_reg_dump(m, dump, n_regs); 3223 3224 kfree(dump); 3225 } 3226 } 3227 } 3228 3229 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 3230 for (pipe = sdma_engine_start; 3231 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); 3232 pipe++) { 3233 for (queue = 0; 3234 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 3235 queue++) { 3236 r = dqm->dev->kfd2kgd->hqd_sdma_dump( 3237 dqm->dev->adev, pipe, queue, &dump, &n_regs); 3238 if (r) 3239 break; 3240 3241 seq_printf(m, " SDMA Engine %d, RLC %d\n", 3242 pipe, queue); 3243 seq_reg_dump(m, dump, n_regs); 3244 3245 kfree(dump); 3246 } 3247 } 3248 3249 return r; 3250 } 3251 3252 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) 3253 { 3254 int r = 0; 3255 3256 dqm_lock(dqm); 3257 r = pm_debugfs_hang_hws(&dqm->packet_mgr); 3258 if (r) { 3259 dqm_unlock(dqm); 3260 return r; 3261 } 3262 dqm->active_runlist = true; 3263 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 3264 0, USE_DEFAULT_GRACE_PERIOD); 3265 dqm_unlock(dqm); 3266 3267 return r; 3268 } 3269 3270 #endif 3271