277 	struct xe_hw_engine *hwe = arg;  in hw_engine_fini()  local
279 	if (hwe->exl_port)  in hw_engine_fini()
280 		xe_execlist_port_destroy(hwe->exl_port);  in hw_engine_fini()
282 	hwe->gt = NULL;  in hw_engine_fini()
295 void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe,  in xe_hw_engine_mmio_write32()  argument
298 	xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base));  in xe_hw_engine_mmio_write32()
299 	xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain);  in xe_hw_engine_mmio_write32()
301 	reg.addr += hwe->mmio_base;  in xe_hw_engine_mmio_write32()
303 	xe_mmio_write32(&hwe->gt->mmio, reg, val);  in xe_hw_engine_mmio_write32()
316 u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg)  in xe_hw_engine_mmio_read32()  argument
318 	xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base));  in xe_hw_engine_mmio_read32()
319 	xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain);  in xe_hw_engine_mmio_read32()
321 	reg.addr += hwe->mmio_base;  in xe_hw_engine_mmio_read32()
323 	return xe_mmio_read32(&hwe->gt->mmio, reg);  in xe_hw_engine_mmio_read32()
326 void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe)  in xe_hw_engine_enable_ring()  argument
329 		xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE);  in xe_hw_engine_enable_ring()
332 	if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask)  in xe_hw_engine_enable_ring()
333 		xe_mmio_write32(&hwe->gt->mmio, RCU_MODE,  in xe_hw_engine_enable_ring()
336 	xe_hw_engine_mmio_write32(hwe, RING_HWSTAM(0), ~0x0);  in xe_hw_engine_enable_ring()
337 	xe_hw_engine_mmio_write32(hwe, RING_HWS_PGA(0),  in xe_hw_engine_enable_ring()
338 				  xe_bo_ggtt_addr(hwe->hwsp));  in xe_hw_engine_enable_ring()
340 	if (xe_device_has_msix(gt_to_xe(hwe->gt)))  in xe_hw_engine_enable_ring()
342 	xe_hw_engine_mmio_write32(hwe, RING_MODE(0), ring_mode);  in xe_hw_engine_enable_ring()
343 	xe_hw_engine_mmio_write32(hwe, RING_MI_MODE(0),  in xe_hw_engine_enable_ring()
345 	xe_hw_engine_mmio_read32(hwe, RING_MI_MODE(0));  in xe_hw_engine_enable_ring()
349 						 const struct xe_hw_engine *hwe)  in xe_hw_engine_match_fixed_cslice_mode()  argument
352 	       xe_rtp_match_first_render_or_compute(gt, hwe);  in xe_hw_engine_match_fixed_cslice_mode()
356 				      const struct xe_hw_engine *hwe)  in xe_rtp_cfeg_wmtp_disabled()  argument
361 	if (hwe->class != XE_ENGINE_CLASS_COMPUTE &&  in xe_rtp_cfeg_wmtp_disabled()
362 	    hwe->class != XE_ENGINE_CLASS_RENDER)  in xe_rtp_cfeg_wmtp_disabled()
365 	return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE;  in xe_rtp_cfeg_wmtp_disabled()
369 xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)  in xe_hw_engine_setup_default_lrc_state()  argument
371 	struct xe_gt *gt = hwe->gt;  in xe_hw_engine_setup_default_lrc_state()
376 	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);  in xe_hw_engine_setup_default_lrc_state()
402 	xe_rtp_process_to_sr(&ctx, lrc_setup, ARRAY_SIZE(lrc_setup), &hwe->reg_lrc);  in xe_hw_engine_setup_default_lrc_state()
406 hw_engine_setup_default_state(struct xe_hw_engine *hwe)  in hw_engine_setup_default_state()  argument
408 	struct xe_gt *gt = hwe->gt;  in hw_engine_setup_default_state()
421 	const u8 mocs_read_idx = hwe->class == XE_ENGINE_CLASS_COMPUTE && IS_DGFX(xe) &&  in hw_engine_setup_default_state()
426 	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);  in hw_engine_setup_default_state()
466 	xe_rtp_process_to_sr(&ctx, engine_entries, ARRAY_SIZE(engine_entries), &hwe->reg_sr);  in hw_engine_setup_default_state()
493 static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe,  in hw_engine_init_early()  argument
506 	xe_gt_assert(gt, !hwe->gt);  in hw_engine_init_early()
508 	hwe->gt = gt;  in hw_engine_init_early()
509 	hwe->class = info->class;  in hw_engine_init_early()
510 	hwe->instance = info->instance;  in hw_engine_init_early()
511 	hwe->mmio_base = info->mmio_base;  in hw_engine_init_early()
512 	hwe->irq_offset = xe_device_has_msix(gt_to_xe(gt)) ?  in hw_engine_init_early()
515 	hwe->domain = info->domain;  in hw_engine_init_early()
516 	hwe->name = info->name;  in hw_engine_init_early()
517 	hwe->fence_irq = &gt->fence_irq[info->class];  in hw_engine_init_early()
518 	hwe->engine_id = id;  in hw_engine_init_early()
520 	hwe->eclass = &gt->eclass[hwe->class];  in hw_engine_init_early()
521 	if (!hwe->eclass->sched_props.job_timeout_ms) {  in hw_engine_init_early()
522 		hwe->eclass->sched_props.job_timeout_ms = 5 * 1000;  in hw_engine_init_early()
523 		hwe->eclass->sched_props.job_timeout_min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;  in hw_engine_init_early()
524 		hwe->eclass->sched_props.job_timeout_max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;  in hw_engine_init_early()
525 		hwe->eclass->sched_props.timeslice_us = 1 * 1000;  in hw_engine_init_early()
526 		hwe->eclass->sched_props.timeslice_min = XE_HW_ENGINE_TIMESLICE_MIN;  in hw_engine_init_early()
527 		hwe->eclass->sched_props.timeslice_max = XE_HW_ENGINE_TIMESLICE_MAX;  in hw_engine_init_early()
528 		hwe->eclass->sched_props.preempt_timeout_us = XE_HW_ENGINE_PREEMPT_TIMEOUT;  in hw_engine_init_early()
529 		hwe->eclass->sched_props.preempt_timeout_min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;  in hw_engine_init_early()
530 		hwe->eclass->sched_props.preempt_timeout_max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;  in hw_engine_init_early()
549 		if (hwe->class == XE_ENGINE_CLASS_OTHER) {  in hw_engine_init_early()
551 			if (hwe->eclass->sched_props.preempt_timeout_us < min_preempt_timeout) {  in hw_engine_init_early()
552 				hwe->eclass->sched_props.preempt_timeout_us = min_preempt_timeout;  in hw_engine_init_early()
558 		hwe->eclass->defaults = hwe->eclass->sched_props;  in hw_engine_init_early()
561 	xe_reg_sr_init(&hwe->reg_sr, hwe->name, gt_to_xe(gt));  in hw_engine_init_early()
562 	xe_tuning_process_engine(hwe);  in hw_engine_init_early()
563 	xe_wa_process_engine(hwe);  in hw_engine_init_early()
564 	hw_engine_setup_default_state(hwe);  in hw_engine_init_early()
566 	xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt));  in hw_engine_init_early()
567 	xe_reg_whitelist_process_engine(hwe);  in hw_engine_init_early()
570 static void adjust_idledly(struct xe_hw_engine *hwe)  in adjust_idledly()  argument
572 	struct xe_gt *gt = hwe->gt;  in adjust_idledly()
578 	if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_WA(gt, 16023105232)) {  in adjust_idledly()
579 		idledly = xe_mmio_read32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base));  in adjust_idledly()
580 		maxcnt = xe_mmio_read32(&gt->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base));  in adjust_idledly()
592 			xe_mmio_write32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base), idledly);  in adjust_idledly()
597 static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,  in hw_engine_init()  argument
607 	xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);  in hw_engine_init()
609 	hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K,  in hw_engine_init()
613 	if (IS_ERR(hwe->hwsp)) {  in hw_engine_init()
614 		err = PTR_ERR(hwe->hwsp);  in hw_engine_init()
619 		hwe->exl_port = xe_execlist_port_create(xe, hwe);  in hw_engine_init()
620 		if (IS_ERR(hwe->exl_port)) {  in hw_engine_init()
621 			err = PTR_ERR(hwe->exl_port);  in hw_engine_init()
626 		if (hwe->class == XE_ENGINE_CLASS_OTHER)  in hw_engine_init()
627 			hwe->irq_handler = xe_gsc_hwe_irq_handler;  in hw_engine_init()
630 			xe_hw_engine_enable_ring(hwe);  in hw_engine_init()
634 	if (xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY)  in hw_engine_init()
635 		gt->usm.reserved_bcs_instance = hwe->instance;  in hw_engine_init()
638 	adjust_idledly(hwe);  in hw_engine_init()
640 	return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe);  in hw_engine_init()
643 	xe_bo_unpin_map_no_vm(hwe->hwsp);  in hw_engine_init()
645 	hwe->name = NULL;  in hw_engine_init()
656 		struct xe_hw_engine *hwe;  in hw_engine_setup_logical_mapping()  local
660 		for_each_hw_engine(hwe, gt, id)  in hw_engine_setup_logical_mapping()
661 			if (hwe->class == class)  in hw_engine_setup_logical_mapping()
662 				hwe->logical_instance = logical_instance++;  in hw_engine_setup_logical_mapping()
834 	struct xe_hw_engine *hwe;  in xe_hw_engines_init()  local
837 	for_each_hw_engine(hwe, gt, id) {  in xe_hw_engines_init()
838 		err = hw_engine_init(gt, hwe, id);  in xe_hw_engines_init()
851 void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec)  in xe_hw_engine_handle_irq()  argument
853 	wake_up_all(&gt_to_xe(hwe->gt)->ufence_wq);  in xe_hw_engine_handle_irq()
855 	if (hwe->irq_handler)  in xe_hw_engine_handle_irq()
856 		hwe->irq_handler(hwe, intr_vec);  in xe_hw_engine_handle_irq()
859 		xe_hw_fence_irq_run(hwe->fence_irq);  in xe_hw_engine_handle_irq()
874 xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q)  in xe_hw_engine_snapshot_capture()  argument
879 	if (!xe_hw_engine_is_valid(hwe))  in xe_hw_engine_snapshot_capture()
887 	snapshot->name = kstrdup(hwe->name, GFP_ATOMIC);  in xe_hw_engine_snapshot_capture()
888 	snapshot->hwe = hwe;  in xe_hw_engine_snapshot_capture()
889 	snapshot->logical_instance = hwe->logical_instance;  in xe_hw_engine_snapshot_capture()
890 	snapshot->forcewake.domain = hwe->domain;  in xe_hw_engine_snapshot_capture()
891 	snapshot->forcewake.ref = xe_force_wake_ref(gt_to_fw(hwe->gt),  in xe_hw_engine_snapshot_capture()
892 						    hwe->domain);  in xe_hw_engine_snapshot_capture()
893 	snapshot->mmio_base = hwe->mmio_base;  in xe_hw_engine_snapshot_capture()
894 	snapshot->kernel_reserved = xe_hw_engine_is_reserved(hwe);  in xe_hw_engine_snapshot_capture()
897 	if (IS_SRIOV_VF(gt_to_xe(hwe->gt)))  in xe_hw_engine_snapshot_capture()
904 			struct xe_device *xe = gt_to_xe(hwe->gt);  in xe_hw_engine_snapshot_capture()
908 			xe_gt_dbg(hwe->gt, "Found and locked GuC-err-capture node");  in xe_hw_engine_snapshot_capture()
914 	xe_engine_manual_capture(hwe, snapshot);  in xe_hw_engine_snapshot_capture()
915 	xe_gt_dbg(hwe->gt, "Proceeding with manual engine snapshot");  in xe_hw_engine_snapshot_capture()
933 	gt = snapshot->hwe->gt;  in xe_hw_engine_snapshot_free()
952 void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p)  in xe_hw_engine_print()  argument
956 	snapshot = xe_hw_engine_snapshot_capture(hwe, NULL);  in xe_hw_engine_print()
975 bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe)  in xe_hw_engine_is_reserved()  argument
977 	struct xe_gt *gt = hwe->gt;  in xe_hw_engine_is_reserved()
980 	if (hwe->class == XE_ENGINE_CLASS_OTHER)  in xe_hw_engine_is_reserved()
985 	    hwe->class == XE_ENGINE_CLASS_COMPUTE &&  in xe_hw_engine_is_reserved()
986 	    hwe->logical_instance >= gt->ccs_mode)  in xe_hw_engine_is_reserved()
989 	return xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY &&  in xe_hw_engine_is_reserved()
990 		hwe->instance == gt->usm.reserved_bcs_instance;  in xe_hw_engine_is_reserved()
1015 u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe)  in xe_hw_engine_read_timestamp()  argument
1017 	return xe_mmio_read64_2x32(&hwe->gt->mmio, RING_TIMESTAMP(hwe->mmio_base));  in xe_hw_engine_read_timestamp()
1020 enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe)  in xe_hw_engine_to_fw_domain()  argument
1022 	return engine_infos[hwe->engine_id].domain;  in xe_hw_engine_to_fw_domain()