Lines Matching refs:adev
46 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, in amdgpu_gfx_mec_queue_to_bit() argument
51 bit += mec * adev->gfx.mec.num_pipe_per_mec in amdgpu_gfx_mec_queue_to_bit()
52 * adev->gfx.mec.num_queue_per_pipe; in amdgpu_gfx_mec_queue_to_bit()
53 bit += pipe * adev->gfx.mec.num_queue_per_pipe; in amdgpu_gfx_mec_queue_to_bit()
59 void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit, in amdgpu_queue_mask_bit_to_mec_queue() argument
62 *queue = bit % adev->gfx.mec.num_queue_per_pipe; in amdgpu_queue_mask_bit_to_mec_queue()
63 *pipe = (bit / adev->gfx.mec.num_queue_per_pipe) in amdgpu_queue_mask_bit_to_mec_queue()
64 % adev->gfx.mec.num_pipe_per_mec; in amdgpu_queue_mask_bit_to_mec_queue()
65 *mec = (bit / adev->gfx.mec.num_queue_per_pipe) in amdgpu_queue_mask_bit_to_mec_queue()
66 / adev->gfx.mec.num_pipe_per_mec; in amdgpu_queue_mask_bit_to_mec_queue()
70 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, in amdgpu_gfx_is_mec_queue_enabled() argument
73 return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue), in amdgpu_gfx_is_mec_queue_enabled()
74 adev->gfx.mec_bitmap[xcc_id].queue_bitmap); in amdgpu_gfx_is_mec_queue_enabled()
77 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, in amdgpu_gfx_me_queue_to_bit() argument
82 bit += me * adev->gfx.me.num_pipe_per_me in amdgpu_gfx_me_queue_to_bit()
83 * adev->gfx.me.num_queue_per_pipe; in amdgpu_gfx_me_queue_to_bit()
84 bit += pipe * adev->gfx.me.num_queue_per_pipe; in amdgpu_gfx_me_queue_to_bit()
90 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, in amdgpu_gfx_is_me_queue_enabled() argument
93 return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue), in amdgpu_gfx_is_me_queue_enabled()
94 adev->gfx.me.queue_bitmap); in amdgpu_gfx_is_me_queue_enabled()
142 static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev) in amdgpu_gfx_is_graphics_multipipe_capable() argument
144 return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1; in amdgpu_gfx_is_graphics_multipipe_capable()
147 static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev) in amdgpu_gfx_is_compute_multipipe_capable() argument
155 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) in amdgpu_gfx_is_compute_multipipe_capable()
160 if (adev->asic_type == CHIP_POLARIS11) in amdgpu_gfx_is_compute_multipipe_capable()
163 return adev->gfx.mec.num_mec > 1; in amdgpu_gfx_is_compute_multipipe_capable()
166 bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev, in amdgpu_gfx_is_high_priority_graphics_queue() argument
175 if (amdgpu_gfx_is_graphics_multipipe_capable(adev) && in amdgpu_gfx_is_high_priority_graphics_queue()
176 adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) { in amdgpu_gfx_is_high_priority_graphics_queue()
180 bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue); in amdgpu_gfx_is_high_priority_graphics_queue()
181 if (ring == &adev->gfx.gfx_ring[bit]) in amdgpu_gfx_is_high_priority_graphics_queue()
188 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, in amdgpu_gfx_is_high_priority_compute_queue() argument
194 if (adev->gfx.num_compute_rings > 1 && in amdgpu_gfx_is_high_priority_compute_queue()
195 ring == &adev->gfx.compute_ring[0]) in amdgpu_gfx_is_high_priority_compute_queue()
201 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) in amdgpu_gfx_compute_queue_acquire() argument
204 bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev); in amdgpu_gfx_compute_queue_acquire()
205 int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec * in amdgpu_gfx_compute_queue_acquire()
206 adev->gfx.mec.num_queue_per_pipe, in amdgpu_gfx_compute_queue_acquire()
207 adev->gfx.num_compute_rings); in amdgpu_gfx_compute_queue_acquire()
208 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; in amdgpu_gfx_compute_queue_acquire()
215 pipe = i % adev->gfx.mec.num_pipe_per_mec; in amdgpu_gfx_compute_queue_acquire()
216 queue = (i / adev->gfx.mec.num_pipe_per_mec) % in amdgpu_gfx_compute_queue_acquire()
217 adev->gfx.mec.num_queue_per_pipe; in amdgpu_gfx_compute_queue_acquire()
219 set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue, in amdgpu_gfx_compute_queue_acquire()
220 adev->gfx.mec_bitmap[j].queue_bitmap); in amdgpu_gfx_compute_queue_acquire()
227 set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap); in amdgpu_gfx_compute_queue_acquire()
232 dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", in amdgpu_gfx_compute_queue_acquire()
233 bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); in amdgpu_gfx_compute_queue_acquire()
237 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) in amdgpu_gfx_graphics_queue_acquire() argument
240 bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev); in amdgpu_gfx_graphics_queue_acquire()
241 int max_queues_per_me = adev->gfx.me.num_pipe_per_me * in amdgpu_gfx_graphics_queue_acquire()
242 adev->gfx.me.num_queue_per_pipe; in amdgpu_gfx_graphics_queue_acquire()
248 pipe = i % adev->gfx.me.num_pipe_per_me; in amdgpu_gfx_graphics_queue_acquire()
249 queue = (i / adev->gfx.me.num_pipe_per_me) % in amdgpu_gfx_graphics_queue_acquire()
250 adev->gfx.me.num_queue_per_pipe; in amdgpu_gfx_graphics_queue_acquire()
252 set_bit(pipe * adev->gfx.me.num_queue_per_pipe + queue, in amdgpu_gfx_graphics_queue_acquire()
253 adev->gfx.me.queue_bitmap); in amdgpu_gfx_graphics_queue_acquire()
257 set_bit(i, adev->gfx.me.queue_bitmap); in amdgpu_gfx_graphics_queue_acquire()
261 adev->gfx.num_gfx_rings = in amdgpu_gfx_graphics_queue_acquire()
262 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); in amdgpu_gfx_graphics_queue_acquire()
265 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, in amdgpu_gfx_kiq_acquire() argument
271 queue_bit = adev->gfx.mec.num_mec in amdgpu_gfx_kiq_acquire()
272 * adev->gfx.mec.num_pipe_per_mec in amdgpu_gfx_kiq_acquire()
273 * adev->gfx.mec.num_queue_per_pipe; in amdgpu_gfx_kiq_acquire()
276 if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap)) in amdgpu_gfx_kiq_acquire()
279 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); in amdgpu_gfx_kiq_acquire()
296 dev_err(adev->dev, "Failed to find a queue for KIQ\n"); in amdgpu_gfx_kiq_acquire()
300 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_kiq_init_ring() argument
302 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_kiq_init_ring()
309 ring->adev = NULL; in amdgpu_gfx_kiq_init_ring()
315 (adev->doorbell_index.kiq + in amdgpu_gfx_kiq_init_ring()
316 xcc_id * adev->doorbell_index.xcc_doorbell_range) in amdgpu_gfx_kiq_init_ring()
319 r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id); in amdgpu_gfx_kiq_init_ring()
328 r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0, in amdgpu_gfx_kiq_init_ring()
331 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r); in amdgpu_gfx_kiq_init_ring()
341 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_kiq_fini() argument
343 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_kiq_fini()
348 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, in amdgpu_gfx_kiq_init() argument
353 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_kiq_init()
355 r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE, in amdgpu_gfx_kiq_init()
359 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r); in amdgpu_gfx_kiq_init()
367 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); in amdgpu_gfx_kiq_init()
375 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, in amdgpu_gfx_mqd_sw_init() argument
379 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_mqd_sw_init()
385 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0)) in amdgpu_gfx_mqd_sw_init()
390 if (!adev->enable_mes_kiq && !ring->mqd_obj) { in amdgpu_gfx_mqd_sw_init()
396 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, in amdgpu_gfx_mqd_sw_init()
403 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r); in amdgpu_gfx_mqd_sw_init()
410 dev_warn(adev->dev, in amdgpu_gfx_mqd_sw_init()
416 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { in amdgpu_gfx_mqd_sw_init()
418 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in amdgpu_gfx_mqd_sw_init()
419 ring = &adev->gfx.gfx_ring[i]; in amdgpu_gfx_mqd_sw_init()
421 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, in amdgpu_gfx_mqd_sw_init()
425 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); in amdgpu_gfx_mqd_sw_init()
431 adev->gfx.me.mqd_backup[i] = kzalloc(mqd_size, GFP_KERNEL); in amdgpu_gfx_mqd_sw_init()
432 if (!adev->gfx.me.mqd_backup[i]) { in amdgpu_gfx_mqd_sw_init()
433 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); in amdgpu_gfx_mqd_sw_init()
441 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_mqd_sw_init()
442 j = i + xcc_id * adev->gfx.num_compute_rings; in amdgpu_gfx_mqd_sw_init()
443 ring = &adev->gfx.compute_ring[j]; in amdgpu_gfx_mqd_sw_init()
445 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, in amdgpu_gfx_mqd_sw_init()
449 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); in amdgpu_gfx_mqd_sw_init()
455 adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL); in amdgpu_gfx_mqd_sw_init()
456 if (!adev->gfx.mec.mqd_backup[j]) { in amdgpu_gfx_mqd_sw_init()
457 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); in amdgpu_gfx_mqd_sw_init()
466 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_mqd_sw_fini() argument
470 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_mqd_sw_fini()
472 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { in amdgpu_gfx_mqd_sw_fini()
473 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in amdgpu_gfx_mqd_sw_fini()
474 ring = &adev->gfx.gfx_ring[i]; in amdgpu_gfx_mqd_sw_fini()
475 kfree(adev->gfx.me.mqd_backup[i]); in amdgpu_gfx_mqd_sw_fini()
482 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_mqd_sw_fini()
483 j = i + xcc_id * adev->gfx.num_compute_rings; in amdgpu_gfx_mqd_sw_fini()
484 ring = &adev->gfx.compute_ring[j]; in amdgpu_gfx_mqd_sw_fini()
485 kfree(adev->gfx.mec.mqd_backup[j]); in amdgpu_gfx_mqd_sw_fini()
498 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_disable_kcq() argument
500 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_disable_kcq()
505 if (adev->enable_mes) { in amdgpu_gfx_disable_kcq()
506 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_disable_kcq()
507 j = i + xcc_id * adev->gfx.num_compute_rings; in amdgpu_gfx_disable_kcq()
508 amdgpu_mes_unmap_legacy_queue(adev, in amdgpu_gfx_disable_kcq()
509 &adev->gfx.compute_ring[j], in amdgpu_gfx_disable_kcq()
518 if (!kiq_ring->sched.ready || amdgpu_in_reset(adev)) in amdgpu_gfx_disable_kcq()
523 adev->gfx.num_compute_rings)) { in amdgpu_gfx_disable_kcq()
528 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_disable_kcq()
529 j = i + xcc_id * adev->gfx.num_compute_rings; in amdgpu_gfx_disable_kcq()
531 &adev->gfx.compute_ring[j], in amdgpu_gfx_disable_kcq()
548 int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_disable_kgq() argument
550 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_disable_kgq()
555 if (adev->enable_mes) { in amdgpu_gfx_disable_kgq()
556 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { in amdgpu_gfx_disable_kgq()
557 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in amdgpu_gfx_disable_kgq()
558 j = i + xcc_id * adev->gfx.num_gfx_rings; in amdgpu_gfx_disable_kgq()
559 amdgpu_mes_unmap_legacy_queue(adev, in amdgpu_gfx_disable_kgq()
560 &adev->gfx.gfx_ring[j], in amdgpu_gfx_disable_kgq()
570 if (!adev->gfx.kiq[0].ring.sched.ready || amdgpu_in_reset(adev)) in amdgpu_gfx_disable_kgq()
573 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { in amdgpu_gfx_disable_kgq()
576 adev->gfx.num_gfx_rings)) { in amdgpu_gfx_disable_kgq()
581 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in amdgpu_gfx_disable_kgq()
582 j = i + xcc_id * adev->gfx.num_gfx_rings; in amdgpu_gfx_disable_kgq()
584 &adev->gfx.gfx_ring[j], in amdgpu_gfx_disable_kgq()
602 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, in amdgpu_queue_mask_bit_to_set_resource_bit() argument
608 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); in amdgpu_queue_mask_bit_to_set_resource_bit()
615 static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_mes_enable_kcq() argument
617 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_mes_enable_kcq()
622 amdgpu_device_flush_hdp(adev, NULL); in amdgpu_gfx_mes_enable_kcq()
624 if (!adev->enable_uni_mes) { in amdgpu_gfx_mes_enable_kcq()
628 dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r); in amdgpu_gfx_mes_enable_kcq()
637 dev_err(adev->dev, "KIQ failed to set resources\n"); in amdgpu_gfx_mes_enable_kcq()
640 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_mes_enable_kcq()
641 j = i + xcc_id * adev->gfx.num_compute_rings; in amdgpu_gfx_mes_enable_kcq()
642 r = amdgpu_mes_map_legacy_queue(adev, in amdgpu_gfx_mes_enable_kcq()
643 &adev->gfx.compute_ring[j]); in amdgpu_gfx_mes_enable_kcq()
645 dev_err(adev->dev, "failed to map compute queue\n"); in amdgpu_gfx_mes_enable_kcq()
653 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_enable_kcq() argument
655 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_enable_kcq()
660 if (adev->mes.enable_legacy_queue_map) in amdgpu_gfx_enable_kcq()
661 return amdgpu_gfx_mes_enable_kcq(adev, xcc_id); in amdgpu_gfx_enable_kcq()
667 if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap)) in amdgpu_gfx_enable_kcq()
678 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i)); in amdgpu_gfx_enable_kcq()
681 amdgpu_device_flush_hdp(adev, NULL); in amdgpu_gfx_enable_kcq()
688 adev->gfx.num_compute_rings + in amdgpu_gfx_enable_kcq()
697 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_enable_kcq()
698 j = i + xcc_id * adev->gfx.num_compute_rings; in amdgpu_gfx_enable_kcq()
700 &adev->gfx.compute_ring[j]); in amdgpu_gfx_enable_kcq()
717 int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_enable_kgq() argument
719 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_gfx_enable_kgq()
726 amdgpu_device_flush_hdp(adev, NULL); in amdgpu_gfx_enable_kgq()
728 if (adev->mes.enable_legacy_queue_map) { in amdgpu_gfx_enable_kgq()
729 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in amdgpu_gfx_enable_kgq()
730 j = i + xcc_id * adev->gfx.num_gfx_rings; in amdgpu_gfx_enable_kgq()
731 r = amdgpu_mes_map_legacy_queue(adev, in amdgpu_gfx_enable_kgq()
732 &adev->gfx.gfx_ring[j]); in amdgpu_gfx_enable_kgq()
744 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { in amdgpu_gfx_enable_kgq()
746 adev->gfx.num_gfx_rings); in amdgpu_gfx_enable_kgq()
753 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in amdgpu_gfx_enable_kgq()
754 j = i + xcc_id * adev->gfx.num_gfx_rings; in amdgpu_gfx_enable_kgq()
756 &adev->gfx.gfx_ring[j]); in amdgpu_gfx_enable_kgq()
774 static void amdgpu_gfx_do_off_ctrl(struct amdgpu_device *adev, bool enable, in amdgpu_gfx_do_off_ctrl() argument
779 if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) in amdgpu_gfx_do_off_ctrl()
782 mutex_lock(&adev->gfx.gfx_off_mutex); in amdgpu_gfx_do_off_ctrl()
789 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0)) in amdgpu_gfx_do_off_ctrl()
792 adev->gfx.gfx_off_req_count--; in amdgpu_gfx_do_off_ctrl()
794 if (adev->gfx.gfx_off_req_count == 0 && in amdgpu_gfx_do_off_ctrl()
795 !adev->gfx.gfx_off_state) { in amdgpu_gfx_do_off_ctrl()
798 if (!amdgpu_dpm_set_powergating_by_smu(adev, in amdgpu_gfx_do_off_ctrl()
800 adev->gfx.gfx_off_state = true; in amdgpu_gfx_do_off_ctrl()
802 schedule_delayed_work(&adev->gfx.gfx_off_delay_work, in amdgpu_gfx_do_off_ctrl()
807 if (adev->gfx.gfx_off_req_count == 0) { in amdgpu_gfx_do_off_ctrl()
808 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); in amdgpu_gfx_do_off_ctrl()
810 if (adev->gfx.gfx_off_state && in amdgpu_gfx_do_off_ctrl()
811 !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false, 0)) { in amdgpu_gfx_do_off_ctrl()
812 adev->gfx.gfx_off_state = false; in amdgpu_gfx_do_off_ctrl()
814 if (adev->gfx.funcs->init_spm_golden) { in amdgpu_gfx_do_off_ctrl()
815 dev_dbg(adev->dev, in amdgpu_gfx_do_off_ctrl()
817 amdgpu_gfx_init_spm_golden(adev); in amdgpu_gfx_do_off_ctrl()
822 adev->gfx.gfx_off_req_count++; in amdgpu_gfx_do_off_ctrl()
826 mutex_unlock(&adev->gfx.gfx_off_mutex); in amdgpu_gfx_do_off_ctrl()
841 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) in amdgpu_gfx_off_ctrl() argument
844 bool no_delay = adev->in_s0ix ? true : false; in amdgpu_gfx_off_ctrl()
846 amdgpu_gfx_do_off_ctrl(adev, enable, no_delay); in amdgpu_gfx_off_ctrl()
861 void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable) in amdgpu_gfx_off_ctrl_immediate() argument
863 amdgpu_gfx_do_off_ctrl(adev, enable, true); in amdgpu_gfx_off_ctrl_immediate()
866 int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value) in amdgpu_set_gfx_off_residency() argument
870 mutex_lock(&adev->gfx.gfx_off_mutex); in amdgpu_set_gfx_off_residency()
872 r = amdgpu_dpm_set_residency_gfxoff(adev, value); in amdgpu_set_gfx_off_residency()
874 mutex_unlock(&adev->gfx.gfx_off_mutex); in amdgpu_set_gfx_off_residency()
879 int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value) in amdgpu_get_gfx_off_residency() argument
883 mutex_lock(&adev->gfx.gfx_off_mutex); in amdgpu_get_gfx_off_residency()
885 r = amdgpu_dpm_get_residency_gfxoff(adev, value); in amdgpu_get_gfx_off_residency()
887 mutex_unlock(&adev->gfx.gfx_off_mutex); in amdgpu_get_gfx_off_residency()
892 int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value) in amdgpu_get_gfx_off_entrycount() argument
896 mutex_lock(&adev->gfx.gfx_off_mutex); in amdgpu_get_gfx_off_entrycount()
898 r = amdgpu_dpm_get_entrycount_gfxoff(adev, value); in amdgpu_get_gfx_off_entrycount()
900 mutex_unlock(&adev->gfx.gfx_off_mutex); in amdgpu_get_gfx_off_entrycount()
905 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value) in amdgpu_get_gfx_off_status() argument
910 mutex_lock(&adev->gfx.gfx_off_mutex); in amdgpu_get_gfx_off_status()
912 r = amdgpu_dpm_get_status_gfxoff(adev, value); in amdgpu_get_gfx_off_status()
914 mutex_unlock(&adev->gfx.gfx_off_mutex); in amdgpu_get_gfx_off_status()
919 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) in amdgpu_gfx_ras_late_init() argument
923 if (amdgpu_ras_is_supported(adev, ras_block->block)) { in amdgpu_gfx_ras_late_init()
924 if (!amdgpu_persistent_edc_harvesting_supported(adev)) { in amdgpu_gfx_ras_late_init()
925 r = amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX); in amdgpu_gfx_ras_late_init()
930 r = amdgpu_ras_block_late_init(adev, ras_block); in amdgpu_gfx_ras_late_init()
934 if (amdgpu_sriov_vf(adev)) in amdgpu_gfx_ras_late_init()
937 if (adev->gfx.cp_ecc_error_irq.funcs) { in amdgpu_gfx_ras_late_init()
938 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); in amdgpu_gfx_ras_late_init()
943 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); in amdgpu_gfx_ras_late_init()
948 amdgpu_ras_block_late_fini(adev, ras_block); in amdgpu_gfx_ras_late_init()
952 int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) in amdgpu_gfx_ras_sw_init() argument
960 if (!adev->gfx.ras) in amdgpu_gfx_ras_sw_init()
963 ras = adev->gfx.ras; in amdgpu_gfx_ras_sw_init()
965 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block); in amdgpu_gfx_ras_sw_init()
967 dev_err(adev->dev, "Failed to register gfx ras block!\n"); in amdgpu_gfx_ras_sw_init()
974 adev->gfx.ras_if = &ras->ras_block.ras_comm; in amdgpu_gfx_ras_sw_init()
987 int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev, in amdgpu_gfx_poison_consumption_handler() argument
990 if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler) in amdgpu_gfx_poison_consumption_handler()
991 return adev->gfx.ras->poison_consumption_handler(adev, entry); in amdgpu_gfx_poison_consumption_handler()
996 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev, in amdgpu_gfx_process_ras_data_cb() argument
1006 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) { in amdgpu_gfx_process_ras_data_cb()
1007 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); in amdgpu_gfx_process_ras_data_cb()
1008 if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops && in amdgpu_gfx_process_ras_data_cb()
1009 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count) in amdgpu_gfx_process_ras_data_cb()
1010 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); in amdgpu_gfx_process_ras_data_cb()
1011 amdgpu_ras_reset_gpu(adev); in amdgpu_gfx_process_ras_data_cb()
1016 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, in amdgpu_gfx_cp_ecc_error_irq() argument
1020 struct ras_common_if *ras_if = adev->gfx.ras_if; in amdgpu_gfx_cp_ecc_error_irq()
1031 amdgpu_ras_interrupt_dispatch(adev, &ih_data); in amdgpu_gfx_cp_ecc_error_irq()
1035 void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev, in amdgpu_gfx_ras_error_func() argument
1037 void (*func)(struct amdgpu_device *adev, void *ras_error_status, in amdgpu_gfx_ras_error_func() argument
1041 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; in amdgpu_gfx_ras_error_func()
1051 func(adev, ras_error_status, i); in amdgpu_gfx_ras_error_func()
1054 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id) in amdgpu_kiq_rreg() argument
1059 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_kiq_rreg()
1062 if (amdgpu_device_skip_hw_access(adev)) in amdgpu_kiq_rreg()
1065 if (adev->mes.ring[0].sched.ready) in amdgpu_kiq_rreg()
1066 return amdgpu_mes_rreg(adev, reg); in amdgpu_kiq_rreg()
1071 if (amdgpu_device_wb_get(adev, ®_val_offs)) { in amdgpu_kiq_rreg()
1097 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) in amdgpu_kiq_rreg()
1110 value = adev->wb.wb[reg_val_offs]; in amdgpu_kiq_rreg()
1111 amdgpu_device_wb_free(adev, reg_val_offs); in amdgpu_kiq_rreg()
1120 amdgpu_device_wb_free(adev, reg_val_offs); in amdgpu_kiq_rreg()
1121 dev_err(adev->dev, "failed to read reg:%x\n", reg); in amdgpu_kiq_rreg()
1125 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id) in amdgpu_kiq_wreg() argument
1130 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; in amdgpu_kiq_wreg()
1135 if (amdgpu_device_skip_hw_access(adev)) in amdgpu_kiq_wreg()
1138 if (adev->mes.ring[0].sched.ready) { in amdgpu_kiq_wreg()
1139 amdgpu_mes_wreg(adev, reg, v); in amdgpu_kiq_wreg()
1166 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) in amdgpu_kiq_wreg()
1186 dev_err(adev->dev, "failed to write reg:%x\n", reg); in amdgpu_kiq_wreg()
1189 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev) in amdgpu_gfx_get_num_kcq() argument
1194 …dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by use… in amdgpu_gfx_get_num_kcq()
1200 void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, in amdgpu_gfx_cp_init_microcode() argument
1212 adev->gfx.pfp_fw->data; in amdgpu_gfx_cp_init_microcode()
1213 adev->gfx.pfp_fw_version = in amdgpu_gfx_cp_init_microcode()
1215 adev->gfx.pfp_feature_version = in amdgpu_gfx_cp_init_microcode()
1217 ucode_fw = adev->gfx.pfp_fw; in amdgpu_gfx_cp_init_microcode()
1222 adev->gfx.pfp_fw->data; in amdgpu_gfx_cp_init_microcode()
1223 adev->gfx.pfp_fw_version = in amdgpu_gfx_cp_init_microcode()
1225 adev->gfx.pfp_feature_version = in amdgpu_gfx_cp_init_microcode()
1227 ucode_fw = adev->gfx.pfp_fw; in amdgpu_gfx_cp_init_microcode()
1233 adev->gfx.pfp_fw->data; in amdgpu_gfx_cp_init_microcode()
1234 ucode_fw = adev->gfx.pfp_fw; in amdgpu_gfx_cp_init_microcode()
1239 adev->gfx.me_fw->data; in amdgpu_gfx_cp_init_microcode()
1240 adev->gfx.me_fw_version = in amdgpu_gfx_cp_init_microcode()
1242 adev->gfx.me_feature_version = in amdgpu_gfx_cp_init_microcode()
1244 ucode_fw = adev->gfx.me_fw; in amdgpu_gfx_cp_init_microcode()
1249 adev->gfx.me_fw->data; in amdgpu_gfx_cp_init_microcode()
1250 adev->gfx.me_fw_version = in amdgpu_gfx_cp_init_microcode()
1252 adev->gfx.me_feature_version = in amdgpu_gfx_cp_init_microcode()
1254 ucode_fw = adev->gfx.me_fw; in amdgpu_gfx_cp_init_microcode()
1260 adev->gfx.me_fw->data; in amdgpu_gfx_cp_init_microcode()
1261 ucode_fw = adev->gfx.me_fw; in amdgpu_gfx_cp_init_microcode()
1266 adev->gfx.ce_fw->data; in amdgpu_gfx_cp_init_microcode()
1267 adev->gfx.ce_fw_version = in amdgpu_gfx_cp_init_microcode()
1269 adev->gfx.ce_feature_version = in amdgpu_gfx_cp_init_microcode()
1271 ucode_fw = adev->gfx.ce_fw; in amdgpu_gfx_cp_init_microcode()
1276 adev->gfx.mec_fw->data; in amdgpu_gfx_cp_init_microcode()
1277 adev->gfx.mec_fw_version = in amdgpu_gfx_cp_init_microcode()
1279 adev->gfx.mec_feature_version = in amdgpu_gfx_cp_init_microcode()
1281 ucode_fw = adev->gfx.mec_fw; in amdgpu_gfx_cp_init_microcode()
1287 adev->gfx.mec_fw->data; in amdgpu_gfx_cp_init_microcode()
1288 ucode_fw = adev->gfx.mec_fw; in amdgpu_gfx_cp_init_microcode()
1293 adev->gfx.mec2_fw->data; in amdgpu_gfx_cp_init_microcode()
1294 adev->gfx.mec2_fw_version = in amdgpu_gfx_cp_init_microcode()
1296 adev->gfx.mec2_feature_version = in amdgpu_gfx_cp_init_microcode()
1298 ucode_fw = adev->gfx.mec2_fw; in amdgpu_gfx_cp_init_microcode()
1304 adev->gfx.mec2_fw->data; in amdgpu_gfx_cp_init_microcode()
1305 ucode_fw = adev->gfx.mec2_fw; in amdgpu_gfx_cp_init_microcode()
1310 adev->gfx.mec_fw->data; in amdgpu_gfx_cp_init_microcode()
1311 adev->gfx.mec_fw_version = in amdgpu_gfx_cp_init_microcode()
1313 adev->gfx.mec_feature_version = in amdgpu_gfx_cp_init_microcode()
1315 ucode_fw = adev->gfx.mec_fw; in amdgpu_gfx_cp_init_microcode()
1323 adev->gfx.mec_fw->data; in amdgpu_gfx_cp_init_microcode()
1324 ucode_fw = adev->gfx.mec_fw; in amdgpu_gfx_cp_init_microcode()
1328 dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id); in amdgpu_gfx_cp_init_microcode()
1332 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { in amdgpu_gfx_cp_init_microcode()
1333 info = &adev->firmware.ucode[ucode_id]; in amdgpu_gfx_cp_init_microcode()
1336 adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE); in amdgpu_gfx_cp_init_microcode()
1340 bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id) in amdgpu_gfx_is_master_xcc() argument
1342 return !(xcc_id % (adev->gfx.num_xcc_per_xcp ? in amdgpu_gfx_is_master_xcc()
1343 adev->gfx.num_xcc_per_xcp : 1)); in amdgpu_gfx_is_master_xcc()
1351 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_get_current_compute_partition() local
1354 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr, in amdgpu_gfx_get_current_compute_partition()
1365 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_set_compute_partition() local
1369 num_xcc = NUM_XCC(adev->gfx.xcc_mask); in amdgpu_gfx_set_compute_partition()
1397 ret = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, mode); in amdgpu_gfx_set_compute_partition()
1418 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_get_available_compute_partition() local
1419 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; in amdgpu_gfx_get_available_compute_partition()
1438 struct amdgpu_device *adev = ring->adev; in amdgpu_gfx_run_cleaner_shader_job() local
1452 dev_err(adev->dev, "Failed setting up GFX kernel entity.\n"); in amdgpu_gfx_run_cleaner_shader_job()
1463 r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner, in amdgpu_gfx_run_cleaner_shader_job()
1491 static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id) in amdgpu_gfx_run_cleaner_shader() argument
1493 int num_xcc = NUM_XCC(adev->gfx.xcc_mask); in amdgpu_gfx_run_cleaner_shader()
1498 if (adev->gfx.num_xcc_per_xcp) in amdgpu_gfx_run_cleaner_shader()
1499 num_xcc_to_clear = adev->gfx.num_xcc_per_xcp; in amdgpu_gfx_run_cleaner_shader()
1504 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in amdgpu_gfx_run_cleaner_shader()
1505 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings]; in amdgpu_gfx_run_cleaner_shader()
1546 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_set_run_cleaner_shader() local
1550 if (amdgpu_in_reset(adev)) in amdgpu_gfx_set_run_cleaner_shader()
1552 if (adev->in_suspend && !adev->in_runpm) in amdgpu_gfx_set_run_cleaner_shader()
1563 if (adev->xcp_mgr) { in amdgpu_gfx_set_run_cleaner_shader()
1564 if (value >= adev->xcp_mgr->num_xcps) in amdgpu_gfx_set_run_cleaner_shader()
1577 ret = amdgpu_gfx_run_cleaner_shader(adev, value); in amdgpu_gfx_set_run_cleaner_shader()
1606 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_get_enforce_isolation() local
1610 if (adev->xcp_mgr) { in amdgpu_gfx_get_enforce_isolation()
1611 for (i = 0; i < adev->xcp_mgr->num_xcps; i++) { in amdgpu_gfx_get_enforce_isolation()
1612 size += sysfs_emit_at(buf, size, "%u", adev->enforce_isolation[i]); in amdgpu_gfx_get_enforce_isolation()
1613 if (i < (adev->xcp_mgr->num_xcps - 1)) in amdgpu_gfx_get_enforce_isolation()
1618 size = sysfs_emit_at(buf, 0, "%u\n", adev->enforce_isolation[0]); in amdgpu_gfx_get_enforce_isolation()
1643 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_set_enforce_isolation() local
1648 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { in amdgpu_gfx_set_enforce_isolation()
1664 if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps) in amdgpu_gfx_set_enforce_isolation()
1667 if (!adev->xcp_mgr && num_partitions != 1) in amdgpu_gfx_set_enforce_isolation()
1675 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_gfx_set_enforce_isolation()
1677 adev->enforce_isolation[i] = partition_values[i]; in amdgpu_gfx_set_enforce_isolation()
1678 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_gfx_set_enforce_isolation()
1680 amdgpu_mes_update_enforce_isolation(adev); in amdgpu_gfx_set_enforce_isolation()
1690 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_get_gfx_reset_mask() local
1692 if (!adev) in amdgpu_gfx_get_gfx_reset_mask()
1695 return amdgpu_show_reset_mask(buf, adev->gfx.gfx_supported_reset); in amdgpu_gfx_get_gfx_reset_mask()
1703 struct amdgpu_device *adev = drm_to_adev(ddev); in amdgpu_gfx_get_compute_reset_mask() local
1705 if (!adev) in amdgpu_gfx_get_compute_reset_mask()
1708 return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset); in amdgpu_gfx_get_compute_reset_mask()
1730 static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_xcp_init() argument
1732 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; in amdgpu_gfx_sysfs_xcp_init()
1746 r = device_create_file(adev->dev, &dev_attr_current_compute_partition); in amdgpu_gfx_sysfs_xcp_init()
1751 r = device_create_file(adev->dev, in amdgpu_gfx_sysfs_xcp_init()
1757 static void amdgpu_gfx_sysfs_xcp_fini(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_xcp_fini() argument
1759 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; in amdgpu_gfx_sysfs_xcp_fini()
1767 device_remove_file(adev->dev, &dev_attr_current_compute_partition); in amdgpu_gfx_sysfs_xcp_fini()
1770 device_remove_file(adev->dev, in amdgpu_gfx_sysfs_xcp_fini()
1774 static int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_isolation_shader_init() argument
1778 r = device_create_file(adev->dev, &dev_attr_enforce_isolation); in amdgpu_gfx_sysfs_isolation_shader_init()
1781 if (adev->gfx.enable_cleaner_shader) in amdgpu_gfx_sysfs_isolation_shader_init()
1782 r = device_create_file(adev->dev, &dev_attr_run_cleaner_shader); in amdgpu_gfx_sysfs_isolation_shader_init()
1787 static void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_isolation_shader_fini() argument
1789 device_remove_file(adev->dev, &dev_attr_enforce_isolation); in amdgpu_gfx_sysfs_isolation_shader_fini()
1790 if (adev->gfx.enable_cleaner_shader) in amdgpu_gfx_sysfs_isolation_shader_fini()
1791 device_remove_file(adev->dev, &dev_attr_run_cleaner_shader); in amdgpu_gfx_sysfs_isolation_shader_fini()
1794 static int amdgpu_gfx_sysfs_reset_mask_init(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_reset_mask_init() argument
1801 if (adev->gfx.num_gfx_rings) { in amdgpu_gfx_sysfs_reset_mask_init()
1802 r = device_create_file(adev->dev, &dev_attr_gfx_reset_mask); in amdgpu_gfx_sysfs_reset_mask_init()
1807 if (adev->gfx.num_compute_rings) { in amdgpu_gfx_sysfs_reset_mask_init()
1808 r = device_create_file(adev->dev, &dev_attr_compute_reset_mask); in amdgpu_gfx_sysfs_reset_mask_init()
1816 static void amdgpu_gfx_sysfs_reset_mask_fini(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_reset_mask_fini() argument
1821 if (adev->gfx.num_gfx_rings) in amdgpu_gfx_sysfs_reset_mask_fini()
1822 device_remove_file(adev->dev, &dev_attr_gfx_reset_mask); in amdgpu_gfx_sysfs_reset_mask_fini()
1824 if (adev->gfx.num_compute_rings) in amdgpu_gfx_sysfs_reset_mask_fini()
1825 device_remove_file(adev->dev, &dev_attr_compute_reset_mask); in amdgpu_gfx_sysfs_reset_mask_fini()
1828 int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_init() argument
1832 r = amdgpu_gfx_sysfs_xcp_init(adev); in amdgpu_gfx_sysfs_init()
1834 dev_err(adev->dev, "failed to create xcp sysfs files"); in amdgpu_gfx_sysfs_init()
1838 r = amdgpu_gfx_sysfs_isolation_shader_init(adev); in amdgpu_gfx_sysfs_init()
1840 dev_err(adev->dev, "failed to create isolation sysfs files"); in amdgpu_gfx_sysfs_init()
1842 r = amdgpu_gfx_sysfs_reset_mask_init(adev); in amdgpu_gfx_sysfs_init()
1844 dev_err(adev->dev, "failed to create reset mask sysfs files"); in amdgpu_gfx_sysfs_init()
1849 void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev) in amdgpu_gfx_sysfs_fini() argument
1851 if (adev->dev->kobj.sd) { in amdgpu_gfx_sysfs_fini()
1852 amdgpu_gfx_sysfs_xcp_fini(adev); in amdgpu_gfx_sysfs_fini()
1853 amdgpu_gfx_sysfs_isolation_shader_fini(adev); in amdgpu_gfx_sysfs_fini()
1854 amdgpu_gfx_sysfs_reset_mask_fini(adev); in amdgpu_gfx_sysfs_fini()
1858 int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev, in amdgpu_gfx_cleaner_shader_sw_init() argument
1861 if (!adev->gfx.enable_cleaner_shader) in amdgpu_gfx_cleaner_shader_sw_init()
1864 return amdgpu_bo_create_kernel(adev, cleaner_shader_size, PAGE_SIZE, in amdgpu_gfx_cleaner_shader_sw_init()
1866 &adev->gfx.cleaner_shader_obj, in amdgpu_gfx_cleaner_shader_sw_init()
1867 &adev->gfx.cleaner_shader_gpu_addr, in amdgpu_gfx_cleaner_shader_sw_init()
1868 (void **)&adev->gfx.cleaner_shader_cpu_ptr); in amdgpu_gfx_cleaner_shader_sw_init()
1871 void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev) in amdgpu_gfx_cleaner_shader_sw_fini() argument
1873 if (!adev->gfx.enable_cleaner_shader) in amdgpu_gfx_cleaner_shader_sw_fini()
1876 amdgpu_bo_free_kernel(&adev->gfx.cleaner_shader_obj, in amdgpu_gfx_cleaner_shader_sw_fini()
1877 &adev->gfx.cleaner_shader_gpu_addr, in amdgpu_gfx_cleaner_shader_sw_fini()
1878 (void **)&adev->gfx.cleaner_shader_cpu_ptr); in amdgpu_gfx_cleaner_shader_sw_fini()
1881 void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev, in amdgpu_gfx_cleaner_shader_init() argument
1885 if (!adev->gfx.enable_cleaner_shader) in amdgpu_gfx_cleaner_shader_init()
1888 if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr) in amdgpu_gfx_cleaner_shader_init()
1889 memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr, in amdgpu_gfx_cleaner_shader_init()
1923 static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx, in amdgpu_gfx_kfd_sch_ctrl() argument
1926 mutex_lock(&adev->gfx.kfd_sch_mutex); in amdgpu_gfx_kfd_sch_ctrl()
1933 if (WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx] == 0)) { in amdgpu_gfx_kfd_sch_ctrl()
1934 dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n"); in amdgpu_gfx_kfd_sch_ctrl()
1938 adev->gfx.kfd_sch_req_count[idx]--; in amdgpu_gfx_kfd_sch_ctrl()
1940 if (adev->gfx.kfd_sch_req_count[idx] == 0 && in amdgpu_gfx_kfd_sch_ctrl()
1941 adev->gfx.kfd_sch_inactive[idx]) { in amdgpu_gfx_kfd_sch_ctrl()
1942 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work, in amdgpu_gfx_kfd_sch_ctrl()
1943 msecs_to_jiffies(adev->gfx.enforce_isolation_time[idx])); in amdgpu_gfx_kfd_sch_ctrl()
1946 if (adev->gfx.kfd_sch_req_count[idx] == 0) { in amdgpu_gfx_kfd_sch_ctrl()
1947 cancel_delayed_work_sync(&adev->gfx.enforce_isolation[idx].work); in amdgpu_gfx_kfd_sch_ctrl()
1948 if (!adev->gfx.kfd_sch_inactive[idx]) { in amdgpu_gfx_kfd_sch_ctrl()
1949 amdgpu_amdkfd_stop_sched(adev, idx); in amdgpu_gfx_kfd_sch_ctrl()
1950 adev->gfx.kfd_sch_inactive[idx] = true; in amdgpu_gfx_kfd_sch_ctrl()
1954 adev->gfx.kfd_sch_req_count[idx]++; in amdgpu_gfx_kfd_sch_ctrl()
1958 mutex_unlock(&adev->gfx.kfd_sch_mutex); in amdgpu_gfx_kfd_sch_ctrl()
1977 struct amdgpu_device *adev = isolation_work->adev; in amdgpu_gfx_enforce_isolation_handler() local
1988 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_handler()
1990 if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id) in amdgpu_gfx_enforce_isolation_handler()
1991 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]); in amdgpu_gfx_enforce_isolation_handler()
1994 if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id) in amdgpu_gfx_enforce_isolation_handler()
1995 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]); in amdgpu_gfx_enforce_isolation_handler()
1999 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work, in amdgpu_gfx_enforce_isolation_handler()
2003 if (adev->kfd.init_complete) { in amdgpu_gfx_enforce_isolation_handler()
2004 WARN_ON_ONCE(!adev->gfx.kfd_sch_inactive[idx]); in amdgpu_gfx_enforce_isolation_handler()
2005 WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx]); in amdgpu_gfx_enforce_isolation_handler()
2006 amdgpu_amdkfd_start_sched(adev, idx); in amdgpu_gfx_enforce_isolation_handler()
2007 adev->gfx.kfd_sch_inactive[idx] = false; in amdgpu_gfx_enforce_isolation_handler()
2010 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_handler()
2025 amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev, in amdgpu_gfx_enforce_isolation_wait_for_kfd() argument
2031 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2032 if (adev->enforce_isolation[idx]) { in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2034 if (!adev->gfx.enforce_isolation_jiffies[idx]) { in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2035 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2036 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2039 if (amdgpu_amdkfd_compute_active(adev, idx)) { in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2041 if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) { in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2042 cjiffies -= adev->gfx.enforce_isolation_jiffies[idx]; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2047 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2050 adev->gfx.enforce_isolation_time[idx] = in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2055 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2059 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2060 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2063 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_wait_for_kfd()
2080 struct amdgpu_device *adev = ring->adev; in amdgpu_gfx_enforce_isolation_ring_begin_use() local
2084 if (!adev->gfx.enable_cleaner_shader) in amdgpu_gfx_enforce_isolation_ring_begin_use()
2096 amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx); in amdgpu_gfx_enforce_isolation_ring_begin_use()
2098 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_ring_begin_use()
2099 if (adev->enforce_isolation[idx]) { in amdgpu_gfx_enforce_isolation_ring_begin_use()
2100 if (adev->kfd.init_complete) in amdgpu_gfx_enforce_isolation_ring_begin_use()
2103 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_ring_begin_use()
2106 amdgpu_gfx_kfd_sch_ctrl(adev, idx, false); in amdgpu_gfx_enforce_isolation_ring_begin_use()
2120 struct amdgpu_device *adev = ring->adev; in amdgpu_gfx_enforce_isolation_ring_end_use() local
2124 if (!adev->gfx.enable_cleaner_shader) in amdgpu_gfx_enforce_isolation_ring_end_use()
2135 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_ring_end_use()
2136 if (adev->enforce_isolation[idx]) { in amdgpu_gfx_enforce_isolation_ring_end_use()
2137 if (adev->kfd.init_complete) in amdgpu_gfx_enforce_isolation_ring_end_use()
2140 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_gfx_enforce_isolation_ring_end_use()
2143 amdgpu_gfx_kfd_sch_ctrl(adev, idx, true); in amdgpu_gfx_enforce_isolation_ring_end_use()
2148 struct amdgpu_device *adev = in amdgpu_gfx_profile_idle_work_handler() local
2154 if (adev->gfx.num_gfx_rings) in amdgpu_gfx_profile_idle_work_handler()
2160 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]); in amdgpu_gfx_profile_idle_work_handler()
2162 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]); in amdgpu_gfx_profile_idle_work_handler()
2163 if (!fences && !atomic_read(&adev->gfx.total_submission_cnt)) { in amdgpu_gfx_profile_idle_work_handler()
2164 mutex_lock(&adev->gfx.workload_profile_mutex); in amdgpu_gfx_profile_idle_work_handler()
2165 if (adev->gfx.workload_profile_active) { in amdgpu_gfx_profile_idle_work_handler()
2166 r = amdgpu_dpm_switch_power_profile(adev, profile, false); in amdgpu_gfx_profile_idle_work_handler()
2168 dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r, in amdgpu_gfx_profile_idle_work_handler()
2171 adev->gfx.workload_profile_active = false; in amdgpu_gfx_profile_idle_work_handler()
2173 mutex_unlock(&adev->gfx.workload_profile_mutex); in amdgpu_gfx_profile_idle_work_handler()
2175 schedule_delayed_work(&adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT); in amdgpu_gfx_profile_idle_work_handler()
2181 struct amdgpu_device *adev = ring->adev; in amdgpu_gfx_profile_ring_begin_use() local
2185 if (adev->gfx.num_gfx_rings) in amdgpu_gfx_profile_ring_begin_use()
2190 atomic_inc(&adev->gfx.total_submission_cnt); in amdgpu_gfx_profile_ring_begin_use()
2192 cancel_delayed_work_sync(&adev->gfx.idle_work); in amdgpu_gfx_profile_ring_begin_use()
2198 if (adev->gfx.workload_profile_active) in amdgpu_gfx_profile_ring_begin_use()
2201 mutex_lock(&adev->gfx.workload_profile_mutex); in amdgpu_gfx_profile_ring_begin_use()
2202 if (!adev->gfx.workload_profile_active) { in amdgpu_gfx_profile_ring_begin_use()
2203 r = amdgpu_dpm_switch_power_profile(adev, profile, true); in amdgpu_gfx_profile_ring_begin_use()
2205 dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r, in amdgpu_gfx_profile_ring_begin_use()
2208 adev->gfx.workload_profile_active = true; in amdgpu_gfx_profile_ring_begin_use()
2210 mutex_unlock(&adev->gfx.workload_profile_mutex); in amdgpu_gfx_profile_ring_begin_use()
2215 atomic_dec(&ring->adev->gfx.total_submission_cnt); in amdgpu_gfx_profile_ring_end_use()
2217 schedule_delayed_work(&ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT); in amdgpu_gfx_profile_ring_end_use()
2226 struct amdgpu_device *adev = (struct amdgpu_device *)data; in amdgpu_debugfs_gfx_sched_mask_set() local
2231 if (!adev) in amdgpu_debugfs_gfx_sched_mask_set()
2234 mask = (1ULL << adev->gfx.num_gfx_rings) - 1; in amdgpu_debugfs_gfx_sched_mask_set()
2238 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) { in amdgpu_debugfs_gfx_sched_mask_set()
2239 ring = &adev->gfx.gfx_ring[i]; in amdgpu_debugfs_gfx_sched_mask_set()
2252 struct amdgpu_device *adev = (struct amdgpu_device *)data; in amdgpu_debugfs_gfx_sched_mask_get() local
2257 if (!adev) in amdgpu_debugfs_gfx_sched_mask_get()
2259 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) { in amdgpu_debugfs_gfx_sched_mask_get()
2260 ring = &adev->gfx.gfx_ring[i]; in amdgpu_debugfs_gfx_sched_mask_get()
2275 void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev) in amdgpu_debugfs_gfx_sched_mask_init() argument
2278 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_debugfs_gfx_sched_mask_init()
2282 if (!(adev->gfx.num_gfx_rings > 1)) in amdgpu_debugfs_gfx_sched_mask_init()
2285 debugfs_create_file(name, 0600, root, adev, in amdgpu_debugfs_gfx_sched_mask_init()
2296 struct amdgpu_device *adev = (struct amdgpu_device *)data; in amdgpu_debugfs_compute_sched_mask_set() local
2301 if (!adev) in amdgpu_debugfs_compute_sched_mask_set()
2304 mask = (1ULL << adev->gfx.num_compute_rings) - 1; in amdgpu_debugfs_compute_sched_mask_set()
2308 for (i = 0; i < adev->gfx.num_compute_rings; ++i) { in amdgpu_debugfs_compute_sched_mask_set()
2309 ring = &adev->gfx.compute_ring[i]; in amdgpu_debugfs_compute_sched_mask_set()
2323 struct amdgpu_device *adev = (struct amdgpu_device *)data; in amdgpu_debugfs_compute_sched_mask_get() local
2328 if (!adev) in amdgpu_debugfs_compute_sched_mask_get()
2330 for (i = 0; i < adev->gfx.num_compute_rings; ++i) { in amdgpu_debugfs_compute_sched_mask_get()
2331 ring = &adev->gfx.compute_ring[i]; in amdgpu_debugfs_compute_sched_mask_get()
2346 void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev) in amdgpu_debugfs_compute_sched_mask_init() argument
2349 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_debugfs_compute_sched_mask_init()
2353 if (!(adev->gfx.num_compute_rings > 1)) in amdgpu_debugfs_compute_sched_mask_init()
2356 debugfs_create_file(name, 0600, root, adev, in amdgpu_debugfs_compute_sched_mask_init()