180 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,  in amdgpu_ip_member_of_hwini()  argument
183 	return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0;  in amdgpu_ip_member_of_hwini()
186 void amdgpu_set_init_level(struct amdgpu_device *adev,  in amdgpu_set_init_level()  argument
191 		adev->init_lvl = &amdgpu_init_minimal_xgmi;  in amdgpu_set_init_level()
194 		adev->init_lvl = &amdgpu_init_recovery;  in amdgpu_set_init_level()
199 		adev->init_lvl = &amdgpu_init_default;  in amdgpu_set_init_level()
204 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
221 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_device_get_pcie_replay_count()  local
222 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);  in amdgpu_device_get_pcie_replay_count()
230 static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev)  in amdgpu_device_attr_sysfs_init()  argument
234 	if (!amdgpu_sriov_vf(adev))  in amdgpu_device_attr_sysfs_init()
235 		ret = sysfs_create_file(&adev->dev->kobj,  in amdgpu_device_attr_sysfs_init()
241 static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev)  in amdgpu_device_attr_sysfs_fini()  argument
243 	if (!amdgpu_sriov_vf(adev))  in amdgpu_device_attr_sysfs_fini()
244 		sysfs_remove_file(&adev->dev->kobj,  in amdgpu_device_attr_sysfs_fini()
254 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_sysfs_reg_state_get()  local
260 			adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);  in amdgpu_sysfs_reg_state_get()
264 			adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);  in amdgpu_sysfs_reg_state_get()
268 			adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);  in amdgpu_sysfs_reg_state_get()
272 			adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);  in amdgpu_sysfs_reg_state_get()
276 			adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);  in amdgpu_sysfs_reg_state_get()
288 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)  in amdgpu_reg_state_sysfs_init()  argument
292 	if (!amdgpu_asic_get_reg_state_supported(adev))  in amdgpu_reg_state_sysfs_init()
295 	ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);  in amdgpu_reg_state_sysfs_init()
300 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)  in amdgpu_reg_state_sysfs_fini()  argument
302 	if (!amdgpu_asic_get_reg_state_supported(adev))  in amdgpu_reg_state_sysfs_fini()
304 	sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);  in amdgpu_reg_state_sysfs_fini()
314 			dev_err(ip_block->adev->dev,  in amdgpu_ip_block_suspend()
332 			dev_err(ip_block->adev->dev,  in amdgpu_ip_block_resume()
364 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_device_get_board_info()  local
368 	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)  in amdgpu_device_get_board_info()
369 		pkg_type = adev->smuio.funcs->get_pkg_type(adev);  in amdgpu_device_get_board_info()
398 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_board_attrs_is_visible()  local
400 	if (adev->flags & AMD_IS_APU)  in amdgpu_board_attrs_is_visible()
411 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
424 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_supports_px()  local
426 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())  in amdgpu_device_supports_px()
441 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_supports_boco()  local
446 	if (adev->has_pr3 ||  in amdgpu_device_supports_boco()
447 	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))  in amdgpu_device_supports_boco()
464 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_supports_baco()  local
466 	return amdgpu_asic_supports_baco(adev);  in amdgpu_device_supports_baco()
469 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)  in amdgpu_device_detect_runtime_pm_mode()  argument
474 	dev = adev_to_drm(adev);  in amdgpu_device_detect_runtime_pm_mode()
476 	adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;  in amdgpu_device_detect_runtime_pm_mode()
482 			adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;  in amdgpu_device_detect_runtime_pm_mode()
483 			dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");  in amdgpu_device_detect_runtime_pm_mode()
485 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;  in amdgpu_device_detect_runtime_pm_mode()
486 			dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");  in amdgpu_device_detect_runtime_pm_mode()
491 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;  in amdgpu_device_detect_runtime_pm_mode()
492 			dev_info(adev->dev, "Forcing BACO for runtime pm\n");  in amdgpu_device_detect_runtime_pm_mode()
498 			adev->pm.rpm_mode = AMDGPU_RUNPM_PX;  in amdgpu_device_detect_runtime_pm_mode()
499 			dev_info(adev->dev, "Using ATPX for runtime pm\n");  in amdgpu_device_detect_runtime_pm_mode()
501 			adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;  in amdgpu_device_detect_runtime_pm_mode()
502 			dev_info(adev->dev, "Using BOCO for runtime pm\n");  in amdgpu_device_detect_runtime_pm_mode()
507 			switch (adev->asic_type) {  in amdgpu_device_detect_runtime_pm_mode()
514 				if (!adev->gmc.noretry)  in amdgpu_device_detect_runtime_pm_mode()
515 					adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;  in amdgpu_device_detect_runtime_pm_mode()
519 				adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;  in amdgpu_device_detect_runtime_pm_mode()
523 			if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {  in amdgpu_device_detect_runtime_pm_mode()
525 					adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;  in amdgpu_device_detect_runtime_pm_mode()
526 					dev_info(adev->dev, "Using BAMACO for runtime pm\n");  in amdgpu_device_detect_runtime_pm_mode()
528 					dev_info(adev->dev, "Using BACO for runtime pm\n");  in amdgpu_device_detect_runtime_pm_mode()
534 		dev_info(adev->dev, "runtime pm is manually disabled\n");  in amdgpu_device_detect_runtime_pm_mode()
541 	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)  in amdgpu_device_detect_runtime_pm_mode()
542 		dev_info(adev->dev, "Runtime PM not available\n");  in amdgpu_device_detect_runtime_pm_mode()
572 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,  in amdgpu_device_mm_access()  argument
581 	if (!drm_dev_enter(adev_to_drm(adev), &idx))  in amdgpu_device_mm_access()
586 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);  in amdgpu_device_mm_access()
601 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);  in amdgpu_device_mm_access()
616 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,  in amdgpu_device_aper_access()  argument
624 	if (!adev->mman.aper_base_kaddr)  in amdgpu_device_aper_access()
627 	last = min(pos + size, adev->gmc.visible_vram_size);  in amdgpu_device_aper_access()
629 		addr = adev->mman.aper_base_kaddr + pos;  in amdgpu_device_aper_access()
638 			amdgpu_device_flush_hdp(adev, NULL);  in amdgpu_device_aper_access()
640 			amdgpu_device_invalidate_hdp(adev, NULL);  in amdgpu_device_aper_access()
665 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,  in amdgpu_device_vram_access()  argument
671 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);  in amdgpu_device_vram_access()
677 		amdgpu_device_mm_access(adev, pos, buf, size, write);  in amdgpu_device_vram_access()
686 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)  in amdgpu_device_skip_hw_access()  argument
688 	if (adev->no_hw_access)  in amdgpu_device_skip_hw_access()
704 		if (down_read_trylock(&adev->reset_domain->sem))  in amdgpu_device_skip_hw_access()
705 			up_read(&adev->reset_domain->sem);  in amdgpu_device_skip_hw_access()
707 			lockdep_assert_held(&adev->reset_domain->sem);  in amdgpu_device_skip_hw_access()
722 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,  in amdgpu_device_rreg()  argument
727 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_device_rreg()
730 	if ((reg * 4) < adev->rmmio_size) {  in amdgpu_device_rreg()
732 		    amdgpu_sriov_runtime(adev) &&  in amdgpu_device_rreg()
733 		    down_read_trylock(&adev->reset_domain->sem)) {  in amdgpu_device_rreg()
734 			ret = amdgpu_kiq_rreg(adev, reg, 0);  in amdgpu_device_rreg()
735 			up_read(&adev->reset_domain->sem);  in amdgpu_device_rreg()
737 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_device_rreg()
740 		ret = adev->pcie_rreg(adev, reg * 4);  in amdgpu_device_rreg()
743 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);  in amdgpu_device_rreg()
761 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)  in amdgpu_mm_rreg8()  argument
763 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_mm_rreg8()
766 	if (offset < adev->rmmio_size)  in amdgpu_mm_rreg8()
767 		return (readb(adev->rmmio + offset));  in amdgpu_mm_rreg8()
782 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,  in amdgpu_device_xcc_rreg()  argument
788 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_device_xcc_rreg()
791 	if ((reg * 4) < adev->rmmio_size) {  in amdgpu_device_xcc_rreg()
792 		if (amdgpu_sriov_vf(adev) &&  in amdgpu_device_xcc_rreg()
793 		    !amdgpu_sriov_runtime(adev) &&  in amdgpu_device_xcc_rreg()
794 		    adev->gfx.rlc.rlcg_reg_access_supported &&  in amdgpu_device_xcc_rreg()
795 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,  in amdgpu_device_xcc_rreg()
798 			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));  in amdgpu_device_xcc_rreg()
800 		    amdgpu_sriov_runtime(adev) &&  in amdgpu_device_xcc_rreg()
801 		    down_read_trylock(&adev->reset_domain->sem)) {  in amdgpu_device_xcc_rreg()
802 			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);  in amdgpu_device_xcc_rreg()
803 			up_read(&adev->reset_domain->sem);  in amdgpu_device_xcc_rreg()
805 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_device_xcc_rreg()
808 		ret = adev->pcie_rreg(adev, reg * 4);  in amdgpu_device_xcc_rreg()
829 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)  in amdgpu_mm_wreg8()  argument
831 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_mm_wreg8()
834 	if (offset < adev->rmmio_size)  in amdgpu_mm_wreg8()
835 		writeb(value, adev->rmmio + offset);  in amdgpu_mm_wreg8()
850 void amdgpu_device_wreg(struct amdgpu_device *adev,  in amdgpu_device_wreg()  argument
854 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_device_wreg()
857 	if ((reg * 4) < adev->rmmio_size) {  in amdgpu_device_wreg()
859 		    amdgpu_sriov_runtime(adev) &&  in amdgpu_device_wreg()
860 		    down_read_trylock(&adev->reset_domain->sem)) {  in amdgpu_device_wreg()
861 			amdgpu_kiq_wreg(adev, reg, v, 0);  in amdgpu_device_wreg()
862 			up_read(&adev->reset_domain->sem);  in amdgpu_device_wreg()
864 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_device_wreg()
867 		adev->pcie_wreg(adev, reg * 4, v);  in amdgpu_device_wreg()
870 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);  in amdgpu_device_wreg()
883 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,  in amdgpu_mm_wreg_mmio_rlc()  argument
887 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_mm_wreg_mmio_rlc()
890 	if (amdgpu_sriov_fullaccess(adev) &&  in amdgpu_mm_wreg_mmio_rlc()
891 	    adev->gfx.rlc.funcs &&  in amdgpu_mm_wreg_mmio_rlc()
892 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {  in amdgpu_mm_wreg_mmio_rlc()
893 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))  in amdgpu_mm_wreg_mmio_rlc()
894 			return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);  in amdgpu_mm_wreg_mmio_rlc()
895 	} else if ((reg * 4) >= adev->rmmio_size) {  in amdgpu_mm_wreg_mmio_rlc()
896 		adev->pcie_wreg(adev, reg * 4, v);  in amdgpu_mm_wreg_mmio_rlc()
898 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_mm_wreg_mmio_rlc()
913 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,  in amdgpu_device_xcc_wreg()  argument
919 	if (amdgpu_device_skip_hw_access(adev))  in amdgpu_device_xcc_wreg()
922 	if ((reg * 4) < adev->rmmio_size) {  in amdgpu_device_xcc_wreg()
923 		if (amdgpu_sriov_vf(adev) &&  in amdgpu_device_xcc_wreg()
924 		    !amdgpu_sriov_runtime(adev) &&  in amdgpu_device_xcc_wreg()
925 		    adev->gfx.rlc.rlcg_reg_access_supported &&  in amdgpu_device_xcc_wreg()
926 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,  in amdgpu_device_xcc_wreg()
929 			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));  in amdgpu_device_xcc_wreg()
931 		    amdgpu_sriov_runtime(adev) &&  in amdgpu_device_xcc_wreg()
932 		    down_read_trylock(&adev->reset_domain->sem)) {  in amdgpu_device_xcc_wreg()
933 			amdgpu_kiq_wreg(adev, reg, v, xcc_id);  in amdgpu_device_xcc_wreg()
934 			up_read(&adev->reset_domain->sem);  in amdgpu_device_xcc_wreg()
936 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_device_xcc_wreg()
939 		adev->pcie_wreg(adev, reg * 4, v);  in amdgpu_device_xcc_wreg()
951 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg()  argument
959 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_rreg()
960 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_rreg()
962 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg()
963 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_rreg()
964 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_rreg()
969 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg()
974 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg_ext()  argument
983 	if (unlikely(!adev->nbio.funcs)) {  in amdgpu_device_indirect_rreg_ext()
987 		pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_rreg_ext()
988 		pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_rreg_ext()
992 		if (unlikely(!adev->nbio.funcs))  in amdgpu_device_indirect_rreg_ext()
995 			pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);  in amdgpu_device_indirect_rreg_ext()
1000 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg_ext()
1001 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_rreg_ext()
1002 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_rreg_ext()
1004 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +  in amdgpu_device_indirect_rreg_ext()
1021 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg_ext()
1034 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg64()  argument
1042 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_rreg64()
1043 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_rreg64()
1045 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg64()
1046 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_rreg64()
1047 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_rreg64()
1057 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg64()
1062 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg64_ext()  argument
1072 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_rreg64_ext()
1073 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_rreg64_ext()
1074 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))  in amdgpu_device_indirect_rreg64_ext()
1075 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);  in amdgpu_device_indirect_rreg64_ext()
1077 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg64_ext()
1078 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_rreg64_ext()
1079 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_rreg64_ext()
1081 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +  in amdgpu_device_indirect_rreg64_ext()
1107 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg64_ext()
1120 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg()  argument
1127 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_wreg()
1128 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_wreg()
1130 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg()
1131 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_wreg()
1132 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_wreg()
1138 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg()
1141 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg_ext()  argument
1149 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_wreg_ext()
1150 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_wreg_ext()
1151 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))  in amdgpu_device_indirect_wreg_ext()
1152 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);  in amdgpu_device_indirect_wreg_ext()
1156 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg_ext()
1157 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_wreg_ext()
1158 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_wreg_ext()
1160 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +  in amdgpu_device_indirect_wreg_ext()
1178 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg_ext()
1189 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg64()  argument
1196 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_wreg64()
1197 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_wreg64()
1199 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg64()
1200 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_wreg64()
1201 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_wreg64()
1213 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg64()
1216 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg64_ext()  argument
1225 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);  in amdgpu_device_indirect_wreg64_ext()
1226 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);  in amdgpu_device_indirect_wreg64_ext()
1227 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))  in amdgpu_device_indirect_wreg64_ext()
1228 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);  in amdgpu_device_indirect_wreg64_ext()
1230 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg64_ext()
1231 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_wreg64_ext()
1232 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_wreg64_ext()
1234 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +  in amdgpu_device_indirect_wreg64_ext()
1262 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg64_ext()
1272 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)  in amdgpu_device_get_rev_id()  argument
1274 	return adev->nbio.funcs->get_rev_id(adev);  in amdgpu_device_get_rev_id()
1287 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)  in amdgpu_invalid_rreg()  argument
1294 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)  in amdgpu_invalid_rreg_ext()  argument
1311 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)  in amdgpu_invalid_wreg()  argument
1318 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)  in amdgpu_invalid_wreg_ext()  argument
1335 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)  in amdgpu_invalid_rreg64()  argument
1342 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)  in amdgpu_invalid_rreg64_ext()  argument
1359 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)  in amdgpu_invalid_wreg64()  argument
1366 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)  in amdgpu_invalid_wreg64_ext()  argument
1384 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,  in amdgpu_block_invalid_rreg()  argument
1404 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,  in amdgpu_block_invalid_wreg()  argument
1413 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)  in amdgpu_device_get_vbios_flags()  argument
1415 	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))  in amdgpu_device_get_vbios_flags()
1418 	if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev))  in amdgpu_device_get_vbios_flags()
1431 static int amdgpu_device_asic_init(struct amdgpu_device *adev)  in amdgpu_device_asic_init()  argument
1437 	amdgpu_asic_pre_asic_init(adev);  in amdgpu_device_asic_init()
1438 	flags = amdgpu_device_get_vbios_flags(adev);  in amdgpu_device_asic_init()
1441 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||  in amdgpu_device_asic_init()
1442 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||  in amdgpu_device_asic_init()
1443 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||  in amdgpu_device_asic_init()
1444 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {  in amdgpu_device_asic_init()
1445 		amdgpu_psp_wait_for_bootloader(adev);  in amdgpu_device_asic_init()
1446 		if (optional && !adev->bios)  in amdgpu_device_asic_init()
1449 		ret = amdgpu_atomfirmware_asic_init(adev, true);  in amdgpu_device_asic_init()
1452 		if (optional && !adev->bios)  in amdgpu_device_asic_init()
1455 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);  in amdgpu_device_asic_init()
1469 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)  in amdgpu_device_mem_scratch_init()  argument
1471 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,  in amdgpu_device_mem_scratch_init()
1474 				       &adev->mem_scratch.robj,  in amdgpu_device_mem_scratch_init()
1475 				       &adev->mem_scratch.gpu_addr,  in amdgpu_device_mem_scratch_init()
1476 				       (void **)&adev->mem_scratch.ptr);  in amdgpu_device_mem_scratch_init()
1486 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)  in amdgpu_device_mem_scratch_fini()  argument
1488 	amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);  in amdgpu_device_mem_scratch_fini()
1501 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,  in amdgpu_device_program_register_sequence()  argument
1521 			if (adev->family >= AMDGPU_FAMILY_AI)  in amdgpu_device_program_register_sequence()
1538 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)  in amdgpu_device_pci_config_reset()  argument
1540 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);  in amdgpu_device_pci_config_reset()
1550 int amdgpu_device_pci_reset(struct amdgpu_device *adev)  in amdgpu_device_pci_reset()  argument
1552 	return pci_reset_function(adev->pdev);  in amdgpu_device_pci_reset()
1569 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)  in amdgpu_device_wb_fini()  argument
1571 	if (adev->wb.wb_obj) {  in amdgpu_device_wb_fini()
1572 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,  in amdgpu_device_wb_fini()
1573 				      &adev->wb.gpu_addr,  in amdgpu_device_wb_fini()
1574 				      (void **)&adev->wb.wb);  in amdgpu_device_wb_fini()
1575 		adev->wb.wb_obj = NULL;  in amdgpu_device_wb_fini()
1588 static int amdgpu_device_wb_init(struct amdgpu_device *adev)  in amdgpu_device_wb_init()  argument
1592 	if (adev->wb.wb_obj == NULL) {  in amdgpu_device_wb_init()
1594 		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,  in amdgpu_device_wb_init()
1596 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,  in amdgpu_device_wb_init()
1597 					    (void **)&adev->wb.wb);  in amdgpu_device_wb_init()
1599 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);  in amdgpu_device_wb_init()
1603 		adev->wb.num_wb = AMDGPU_MAX_WB;  in amdgpu_device_wb_init()
1604 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));  in amdgpu_device_wb_init()
1607 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);  in amdgpu_device_wb_init()
1622 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)  in amdgpu_device_wb_get()  argument
1626 	spin_lock_irqsave(&adev->wb.lock, flags);  in amdgpu_device_wb_get()
1627 	offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);  in amdgpu_device_wb_get()
1628 	if (offset < adev->wb.num_wb) {  in amdgpu_device_wb_get()
1629 		__set_bit(offset, adev->wb.used);  in amdgpu_device_wb_get()
1630 		spin_unlock_irqrestore(&adev->wb.lock, flags);  in amdgpu_device_wb_get()
1634 		spin_unlock_irqrestore(&adev->wb.lock, flags);  in amdgpu_device_wb_get()
1647 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)  in amdgpu_device_wb_free()  argument
1652 	spin_lock_irqsave(&adev->wb.lock, flags);  in amdgpu_device_wb_free()
1653 	if (wb < adev->wb.num_wb)  in amdgpu_device_wb_free()
1654 		__clear_bit(wb, adev->wb.used);  in amdgpu_device_wb_free()
1655 	spin_unlock_irqrestore(&adev->wb.lock, flags);  in amdgpu_device_wb_free()
1667 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)  in amdgpu_device_resize_fb_bar()  argument
1669 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);  in amdgpu_device_resize_fb_bar()
1680 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_resize_fb_bar()
1685 	    adev->pdev->vendor == PCI_VENDOR_ID_ATI &&  in amdgpu_device_resize_fb_bar()
1686 	    adev->pdev->device == 0x731f &&  in amdgpu_device_resize_fb_bar()
1687 	    adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)  in amdgpu_device_resize_fb_bar()
1691 	if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))  in amdgpu_device_resize_fb_bar()
1695 	if (adev->gmc.real_vram_size &&  in amdgpu_device_resize_fb_bar()
1696 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))  in amdgpu_device_resize_fb_bar()
1700 	root = adev->pdev->bus;  in amdgpu_device_resize_fb_bar()
1715 	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,  in amdgpu_device_resize_fb_bar()
1719 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);  in amdgpu_device_resize_fb_bar()
1720 	pci_write_config_word(adev->pdev, PCI_COMMAND,  in amdgpu_device_resize_fb_bar()
1724 	amdgpu_doorbell_fini(adev);  in amdgpu_device_resize_fb_bar()
1725 	if (adev->asic_type >= CHIP_BONAIRE)  in amdgpu_device_resize_fb_bar()
1726 		pci_release_resource(adev->pdev, 2);  in amdgpu_device_resize_fb_bar()
1728 	pci_release_resource(adev->pdev, 0);  in amdgpu_device_resize_fb_bar()
1730 	r = pci_resize_resource(adev->pdev, 0, rbar_size);  in amdgpu_device_resize_fb_bar()
1736 	pci_assign_unassigned_bus_resources(adev->pdev->bus);  in amdgpu_device_resize_fb_bar()
1741 	r = amdgpu_doorbell_init(adev);  in amdgpu_device_resize_fb_bar()
1742 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))  in amdgpu_device_resize_fb_bar()
1745 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);  in amdgpu_device_resize_fb_bar()
1762 bool amdgpu_device_need_post(struct amdgpu_device *adev)  in amdgpu_device_need_post()  argument
1766 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_need_post()
1769 	flags = amdgpu_device_get_vbios_flags(adev);  in amdgpu_device_need_post()
1772 	if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios)  in amdgpu_device_need_post()
1775 	if (amdgpu_passthrough(adev)) {  in amdgpu_device_need_post()
1781 		if (adev->asic_type == CHIP_FIJI) {  in amdgpu_device_need_post()
1785 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);  in amdgpu_device_need_post()
1790 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);  in amdgpu_device_need_post()
1791 			release_firmware(adev->pm.fw);  in amdgpu_device_need_post()
1798 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)  in amdgpu_device_need_post()
1801 	if (adev->has_hw_reset) {  in amdgpu_device_need_post()
1802 		adev->has_hw_reset = false;  in amdgpu_device_need_post()
1807 	if (adev->asic_type >= CHIP_BONAIRE)  in amdgpu_device_need_post()
1808 		return amdgpu_atombios_scratch_need_asic_init(adev);  in amdgpu_device_need_post()
1811 	reg = amdgpu_asic_get_config_memsize(adev);  in amdgpu_device_need_post()
1826 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)  in amdgpu_device_seamless_boot_supported()  argument
1841 	if (!(adev->flags & AMD_IS_APU))  in amdgpu_device_seamless_boot_supported()
1844 	if (adev->mman.keep_stolen_vga_memory)  in amdgpu_device_seamless_boot_supported()
1847 	return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);  in amdgpu_device_seamless_boot_supported()
1858 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)  in amdgpu_device_pcie_dynamic_switching_supported()  argument
1864 	if (dev_is_removable(adev->dev))  in amdgpu_device_pcie_dynamic_switching_supported()
1883 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)  in amdgpu_device_should_use_aspm()  argument
1895 	if (adev->flags & AMD_IS_APU)  in amdgpu_device_should_use_aspm()
1897 	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))  in amdgpu_device_should_use_aspm()
1899 	return pcie_aspm_enabled(adev->pdev);  in amdgpu_device_should_use_aspm()
1915 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));  in amdgpu_device_vga_set_decode()  local
1917 	amdgpu_asic_set_vga_state(adev, state);  in amdgpu_device_vga_set_decode()
1935 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)  in amdgpu_device_check_block_size()  argument
1945 		dev_warn(adev->dev, "VM page table size (%d) too small\n",  in amdgpu_device_check_block_size()
1959 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)  in amdgpu_device_check_vm_size()  argument
1966 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",  in amdgpu_device_check_vm_size()
1972 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)  in amdgpu_device_check_smu_prv_buffer_size()  argument
2002 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;  in amdgpu_device_check_smu_prv_buffer_size()
2009 	adev->pm.smu_prv_buffer_size = 0;  in amdgpu_device_check_smu_prv_buffer_size()
2012 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)  in amdgpu_device_init_apu_flags()  argument
2014 	if (!(adev->flags & AMD_IS_APU) ||  in amdgpu_device_init_apu_flags()
2015 	    adev->asic_type < CHIP_RAVEN)  in amdgpu_device_init_apu_flags()
2018 	switch (adev->asic_type) {  in amdgpu_device_init_apu_flags()
2020 		if (adev->pdev->device == 0x15dd)  in amdgpu_device_init_apu_flags()
2021 			adev->apu_flags |= AMD_APU_IS_RAVEN;  in amdgpu_device_init_apu_flags()
2022 		if (adev->pdev->device == 0x15d8)  in amdgpu_device_init_apu_flags()
2023 			adev->apu_flags |= AMD_APU_IS_PICASSO;  in amdgpu_device_init_apu_flags()
2026 		if ((adev->pdev->device == 0x1636) ||  in amdgpu_device_init_apu_flags()
2027 		    (adev->pdev->device == 0x164c))  in amdgpu_device_init_apu_flags()
2028 			adev->apu_flags |= AMD_APU_IS_RENOIR;  in amdgpu_device_init_apu_flags()
2030 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;  in amdgpu_device_init_apu_flags()
2033 		adev->apu_flags |= AMD_APU_IS_VANGOGH;  in amdgpu_device_init_apu_flags()
2038 		if ((adev->pdev->device == 0x13FE) ||  in amdgpu_device_init_apu_flags()
2039 		    (adev->pdev->device == 0x143F))  in amdgpu_device_init_apu_flags()
2040 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;  in amdgpu_device_init_apu_flags()
2057 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)  in amdgpu_device_check_arguments()  argument
2062 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",  in amdgpu_device_check_arguments()
2066 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",  in amdgpu_device_check_arguments()
2073 		dev_warn(adev->dev, "gart size (%d) too small\n",  in amdgpu_device_check_arguments()
2080 		dev_warn(adev->dev, "gtt size (%d) too small\n",  in amdgpu_device_check_arguments()
2088 		dev_warn(adev->dev, "valid range is between 4 and 9\n");  in amdgpu_device_check_arguments()
2093 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",  in amdgpu_device_check_arguments()
2097 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",  in amdgpu_device_check_arguments()
2103 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");  in amdgpu_device_check_arguments()
2107 	amdgpu_device_check_smu_prv_buffer_size(adev);  in amdgpu_device_check_arguments()
2109 	amdgpu_device_check_vm_size(adev);  in amdgpu_device_check_arguments()
2111 	amdgpu_device_check_block_size(adev);  in amdgpu_device_check_arguments()
2113 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);  in amdgpu_device_check_arguments()
2116 		adev->enforce_isolation[i] = !!enforce_isolation;  in amdgpu_device_check_arguments()
2207 	struct amdgpu_device *adev = dev;  in amdgpu_device_ip_set_clockgating_state()  local
2210 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_set_clockgating_state()
2211 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_set_clockgating_state()
2213 		if (adev->ip_blocks[i].version->type != block_type)  in amdgpu_device_ip_set_clockgating_state()
2215 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)  in amdgpu_device_ip_set_clockgating_state()
2217 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(  in amdgpu_device_ip_set_clockgating_state()
2218 			&adev->ip_blocks[i], state);  in amdgpu_device_ip_set_clockgating_state()
2221 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_set_clockgating_state()
2241 	struct amdgpu_device *adev = dev;  in amdgpu_device_ip_set_powergating_state()  local
2244 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_set_powergating_state()
2245 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_set_powergating_state()
2247 		if (adev->ip_blocks[i].version->type != block_type)  in amdgpu_device_ip_set_powergating_state()
2249 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)  in amdgpu_device_ip_set_powergating_state()
2251 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(  in amdgpu_device_ip_set_powergating_state()
2252 			&adev->ip_blocks[i], state);  in amdgpu_device_ip_set_powergating_state()
2255 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_set_powergating_state()
2271 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,  in amdgpu_device_ip_get_clockgating_state()  argument
2276 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_get_clockgating_state()
2277 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_get_clockgating_state()
2279 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)  in amdgpu_device_ip_get_clockgating_state()
2280 			adev->ip_blocks[i].version->funcs->get_clockgating_state(  in amdgpu_device_ip_get_clockgating_state()
2281 				&adev->ip_blocks[i], flags);  in amdgpu_device_ip_get_clockgating_state()
2294 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,  in amdgpu_device_ip_wait_for_idle()  argument
2299 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_wait_for_idle()
2300 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_wait_for_idle()
2302 		if (adev->ip_blocks[i].version->type == block_type) {  in amdgpu_device_ip_wait_for_idle()
2303 			if (adev->ip_blocks[i].version->funcs->wait_for_idle) {  in amdgpu_device_ip_wait_for_idle()
2304 				r = adev->ip_blocks[i].version->funcs->wait_for_idle(  in amdgpu_device_ip_wait_for_idle()
2305 								&adev->ip_blocks[i]);  in amdgpu_device_ip_wait_for_idle()
2325 bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,  in amdgpu_device_ip_is_valid()  argument
2330 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_is_valid()
2331 		if (adev->ip_blocks[i].version->type == block_type)  in amdgpu_device_ip_is_valid()
2332 			return adev->ip_blocks[i].status.valid;  in amdgpu_device_ip_is_valid()
2348 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,  in amdgpu_device_ip_get_ip_block()  argument
2353 	for (i = 0; i < adev->num_ip_blocks; i++)  in amdgpu_device_ip_get_ip_block()
2354 		if (adev->ip_blocks[i].version->type == type)  in amdgpu_device_ip_get_ip_block()
2355 			return &adev->ip_blocks[i];  in amdgpu_device_ip_get_ip_block()
2371 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,  in amdgpu_device_ip_block_version_cmp()  argument
2375 	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);  in amdgpu_device_ip_block_version_cmp()
2394 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,  in amdgpu_device_ip_block_add()  argument
2402 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)  in amdgpu_device_ip_block_add()
2406 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)  in amdgpu_device_ip_block_add()
2413 	dev_info(adev->dev, "detected ip block number %d <%s>\n",  in amdgpu_device_ip_block_add()
2414 		 adev->num_ip_blocks, ip_block_version->funcs->name);  in amdgpu_device_ip_block_add()
2416 	adev->ip_blocks[adev->num_ip_blocks].adev = adev;  in amdgpu_device_ip_block_add()
2418 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;  in amdgpu_device_ip_block_add()
2435 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)  in amdgpu_device_enable_virtual_display()  argument
2437 	adev->enable_virtual_display = false;  in amdgpu_device_enable_virtual_display()
2440 		const char *pci_address_name = pci_name(adev->pdev);  in amdgpu_device_enable_virtual_display()
2452 				adev->enable_virtual_display = true;  in amdgpu_device_enable_virtual_display()
2463 					adev->mode_info.num_crtc = num_crtc;  in amdgpu_device_enable_virtual_display()
2465 					adev->mode_info.num_crtc = 1;  in amdgpu_device_enable_virtual_display()
2473 			 adev->enable_virtual_display, adev->mode_info.num_crtc);  in amdgpu_device_enable_virtual_display()
2479 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)  in amdgpu_device_set_sriov_virtual_display()  argument
2481 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {  in amdgpu_device_set_sriov_virtual_display()
2482 		adev->mode_info.num_crtc = 1;  in amdgpu_device_set_sriov_virtual_display()
2483 		adev->enable_virtual_display = true;  in amdgpu_device_set_sriov_virtual_display()
2485 			 adev->enable_virtual_display, adev->mode_info.num_crtc);  in amdgpu_device_set_sriov_virtual_display()
2499 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)  in amdgpu_device_parse_gpu_info_fw()  argument
2505 	adev->firmware.gpu_info_fw = NULL;  in amdgpu_device_parse_gpu_info_fw()
2507 	if (adev->mman.discovery_bin)  in amdgpu_device_parse_gpu_info_fw()
2510 	switch (adev->asic_type) {  in amdgpu_device_parse_gpu_info_fw()
2520 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)  in amdgpu_device_parse_gpu_info_fw()
2522 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)  in amdgpu_device_parse_gpu_info_fw()
2535 	err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw,  in amdgpu_device_parse_gpu_info_fw()
2539 		dev_err(adev->dev,  in amdgpu_device_parse_gpu_info_fw()
2545 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;  in amdgpu_device_parse_gpu_info_fw()
2552 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +  in amdgpu_device_parse_gpu_info_fw()
2558 		if (adev->asic_type == CHIP_NAVI12)  in amdgpu_device_parse_gpu_info_fw()
2561 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);  in amdgpu_device_parse_gpu_info_fw()
2562 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);  in amdgpu_device_parse_gpu_info_fw()
2563 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);  in amdgpu_device_parse_gpu_info_fw()
2564 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);  in amdgpu_device_parse_gpu_info_fw()
2565 		adev->gfx.config.max_texture_channel_caches =  in amdgpu_device_parse_gpu_info_fw()
2567 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);  in amdgpu_device_parse_gpu_info_fw()
2568 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);  in amdgpu_device_parse_gpu_info_fw()
2569 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);  in amdgpu_device_parse_gpu_info_fw()
2570 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);  in amdgpu_device_parse_gpu_info_fw()
2571 		adev->gfx.config.double_offchip_lds_buf =  in amdgpu_device_parse_gpu_info_fw()
2573 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);  in amdgpu_device_parse_gpu_info_fw()
2574 		adev->gfx.cu_info.max_waves_per_simd =  in amdgpu_device_parse_gpu_info_fw()
2576 		adev->gfx.cu_info.max_scratch_slots_per_cu =  in amdgpu_device_parse_gpu_info_fw()
2578 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);  in amdgpu_device_parse_gpu_info_fw()
2581 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +  in amdgpu_device_parse_gpu_info_fw()
2583 			adev->gfx.config.num_sc_per_sh =  in amdgpu_device_parse_gpu_info_fw()
2585 			adev->gfx.config.num_packer_per_sc =  in amdgpu_device_parse_gpu_info_fw()
2596 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +  in amdgpu_device_parse_gpu_info_fw()
2598 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;  in amdgpu_device_parse_gpu_info_fw()
2603 		dev_err(adev->dev,  in amdgpu_device_parse_gpu_info_fw()
2622 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)  in amdgpu_device_ip_early_init()  argument
2630 	amdgpu_device_enable_virtual_display(adev);  in amdgpu_device_ip_early_init()
2632 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_ip_early_init()
2633 		r = amdgpu_virt_request_full_gpu(adev, true);  in amdgpu_device_ip_early_init()
2638 	switch (adev->asic_type) {  in amdgpu_device_ip_early_init()
2645 		adev->family = AMDGPU_FAMILY_SI;  in amdgpu_device_ip_early_init()
2646 		r = si_set_ip_blocks(adev);  in amdgpu_device_ip_early_init()
2657 		if (adev->flags & AMD_IS_APU)  in amdgpu_device_ip_early_init()
2658 			adev->family = AMDGPU_FAMILY_KV;  in amdgpu_device_ip_early_init()
2660 			adev->family = AMDGPU_FAMILY_CI;  in amdgpu_device_ip_early_init()
2662 		r = cik_set_ip_blocks(adev);  in amdgpu_device_ip_early_init()
2676 		if (adev->flags & AMD_IS_APU)  in amdgpu_device_ip_early_init()
2677 			adev->family = AMDGPU_FAMILY_CZ;  in amdgpu_device_ip_early_init()
2679 			adev->family = AMDGPU_FAMILY_VI;  in amdgpu_device_ip_early_init()
2681 		r = vi_set_ip_blocks(adev);  in amdgpu_device_ip_early_init()
2686 		r = amdgpu_discovery_set_ip_blocks(adev);  in amdgpu_device_ip_early_init()
2695 	    ((adev->flags & AMD_IS_APU) == 0) &&  in amdgpu_device_ip_early_init()
2696 	    !dev_is_removable(&adev->pdev->dev))  in amdgpu_device_ip_early_init()
2697 		adev->flags |= AMD_IS_PX;  in amdgpu_device_ip_early_init()
2699 	if (!(adev->flags & AMD_IS_APU)) {  in amdgpu_device_ip_early_init()
2700 		parent = pcie_find_root_port(adev->pdev);  in amdgpu_device_ip_early_init()
2701 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;  in amdgpu_device_ip_early_init()
2705 	adev->pm.pp_feature = amdgpu_pp_feature_mask;  in amdgpu_device_ip_early_init()
2706 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)  in amdgpu_device_ip_early_init()
2707 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;  in amdgpu_device_ip_early_init()
2708 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)  in amdgpu_device_ip_early_init()
2709 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;  in amdgpu_device_ip_early_init()
2710 	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))  in amdgpu_device_ip_early_init()
2711 		adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;  in amdgpu_device_ip_early_init()
2714 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_early_init()
2715 		ip_block = &adev->ip_blocks[i];  in amdgpu_device_ip_early_init()
2719 				  i, adev->ip_blocks[i].version->funcs->name);  in amdgpu_device_ip_early_init()
2720 			adev->ip_blocks[i].status.valid = false;  in amdgpu_device_ip_early_init()
2724 				adev->ip_blocks[i].status.valid = false;  in amdgpu_device_ip_early_init()
2727 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_early_init()
2730 				adev->ip_blocks[i].status.valid = true;  in amdgpu_device_ip_early_init()
2733 			adev->ip_blocks[i].status.valid = true;  in amdgpu_device_ip_early_init()
2736 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {  in amdgpu_device_ip_early_init()
2737 			r = amdgpu_device_parse_gpu_info_fw(adev);  in amdgpu_device_ip_early_init()
2741 			bios_flags = amdgpu_device_get_vbios_flags(adev);  in amdgpu_device_ip_early_init()
2747 				if (!amdgpu_get_bios(adev) && !optional)  in amdgpu_device_ip_early_init()
2750 				if (optional && !adev->bios)  in amdgpu_device_ip_early_init()
2752 						adev->dev,  in amdgpu_device_ip_early_init()
2755 				if (adev->bios) {  in amdgpu_device_ip_early_init()
2756 					r = amdgpu_atombios_init(adev);  in amdgpu_device_ip_early_init()
2758 						dev_err(adev->dev,  in amdgpu_device_ip_early_init()
2761 							adev,  in amdgpu_device_ip_early_init()
2770 			if (amdgpu_sriov_vf(adev))  in amdgpu_device_ip_early_init()
2771 				amdgpu_virt_init_data_exchange(adev);  in amdgpu_device_ip_early_init()
2778 	if (adev->gmc.xgmi.supported)  in amdgpu_device_ip_early_init()
2779 		amdgpu_xgmi_early_init(adev);  in amdgpu_device_ip_early_init()
2781 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);  in amdgpu_device_ip_early_init()
2783 		amdgpu_amdkfd_device_probe(adev);  in amdgpu_device_ip_early_init()
2785 	adev->cg_flags &= amdgpu_cg_mask;  in amdgpu_device_ip_early_init()
2786 	adev->pg_flags &= amdgpu_pg_mask;  in amdgpu_device_ip_early_init()
2791 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)  in amdgpu_device_ip_hw_init_phase1()  argument
2795 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_hw_init_phase1()
2796 		if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_ip_hw_init_phase1()
2798 		if (adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_hw_init_phase1()
2801 			    adev, adev->ip_blocks[i].version->type))  in amdgpu_device_ip_hw_init_phase1()
2803 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||  in amdgpu_device_ip_hw_init_phase1()
2804 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||  in amdgpu_device_ip_hw_init_phase1()
2805 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {  in amdgpu_device_ip_hw_init_phase1()
2806 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_hw_init_phase1()
2809 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_hw_init_phase1()
2812 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_hw_init_phase1()
2819 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)  in amdgpu_device_ip_hw_init_phase2()  argument
2823 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_hw_init_phase2()
2824 		if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_ip_hw_init_phase2()
2826 		if (adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_hw_init_phase2()
2829 			    adev, adev->ip_blocks[i].version->type))  in amdgpu_device_ip_hw_init_phase2()
2831 		r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_hw_init_phase2()
2834 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_hw_init_phase2()
2837 		adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_hw_init_phase2()
2843 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)  in amdgpu_device_fw_loading()  argument
2849 	if (adev->asic_type >= CHIP_VEGA10) {  in amdgpu_device_fw_loading()
2850 		for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_fw_loading()
2851 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)  in amdgpu_device_fw_loading()
2854 			if (!amdgpu_ip_member_of_hwini(adev,  in amdgpu_device_fw_loading()
2858 			if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_fw_loading()
2862 			if (adev->ip_blocks[i].status.hw == true)  in amdgpu_device_fw_loading()
2865 			if (amdgpu_in_reset(adev) || adev->in_suspend) {  in amdgpu_device_fw_loading()
2866 				r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);  in amdgpu_device_fw_loading()
2870 				r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);  in amdgpu_device_fw_loading()
2873 							  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_fw_loading()
2876 				adev->ip_blocks[i].status.hw = true;  in amdgpu_device_fw_loading()
2882 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)  in amdgpu_device_fw_loading()
2883 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);  in amdgpu_device_fw_loading()
2888 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)  in amdgpu_device_init_schedulers()  argument
2893 		.timeout_wq = adev->reset_domain->wq,  in amdgpu_device_init_schedulers()
2894 		.dev = adev->dev,  in amdgpu_device_init_schedulers()
2900 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_device_init_schedulers()
2908 			timeout = adev->gfx_timeout;  in amdgpu_device_init_schedulers()
2911 			timeout = adev->compute_timeout;  in amdgpu_device_init_schedulers()
2914 			timeout = adev->sdma_timeout;  in amdgpu_device_init_schedulers()
2917 			timeout = adev->video_timeout;  in amdgpu_device_init_schedulers()
2932 		r = amdgpu_uvd_entity_init(adev, ring);  in amdgpu_device_init_schedulers()
2938 		r = amdgpu_vce_entity_init(adev, ring);  in amdgpu_device_init_schedulers()
2946 	amdgpu_xcp_update_partition_sched_list(adev);  in amdgpu_device_init_schedulers()
2963 static int amdgpu_device_ip_init(struct amdgpu_device *adev)  in amdgpu_device_ip_init()  argument
2968 	r = amdgpu_ras_init(adev);  in amdgpu_device_ip_init()
2972 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_init()
2973 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_init()
2975 		if (adev->ip_blocks[i].version->funcs->sw_init) {  in amdgpu_device_ip_init()
2976 			r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_init()
2979 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_init()
2983 		adev->ip_blocks[i].status.sw = true;  in amdgpu_device_ip_init()
2986 			    adev, adev->ip_blocks[i].version->type))  in amdgpu_device_ip_init()
2989 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {  in amdgpu_device_ip_init()
2991 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_init()
2996 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_init()
2997 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {  in amdgpu_device_ip_init()
3000 			if (amdgpu_sriov_vf(adev))  in amdgpu_device_ip_init()
3001 				amdgpu_virt_exchange_data(adev);  in amdgpu_device_ip_init()
3003 			r = amdgpu_device_mem_scratch_init(adev);  in amdgpu_device_ip_init()
3008 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_init()
3013 			r = amdgpu_device_wb_init(adev);  in amdgpu_device_ip_init()
3018 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_init()
3021 			if (adev->gfx.mcbp) {  in amdgpu_device_ip_init()
3022 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,  in amdgpu_device_ip_init()
3032 			r = amdgpu_seq64_init(adev);  in amdgpu_device_ip_init()
3040 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_ip_init()
3041 		amdgpu_virt_init_data_exchange(adev);  in amdgpu_device_ip_init()
3043 	r = amdgpu_ib_pool_init(adev);  in amdgpu_device_ip_init()
3045 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);  in amdgpu_device_ip_init()
3046 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);  in amdgpu_device_ip_init()
3050 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/  in amdgpu_device_ip_init()
3054 	r = amdgpu_device_ip_hw_init_phase1(adev);  in amdgpu_device_ip_init()
3058 	r = amdgpu_device_fw_loading(adev);  in amdgpu_device_ip_init()
3062 	r = amdgpu_device_ip_hw_init_phase2(adev);  in amdgpu_device_ip_init()
3081 	init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI);  in amdgpu_device_ip_init()
3082 	r = amdgpu_ras_recovery_init(adev, init_badpage);  in amdgpu_device_ip_init()
3089 	if (adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_device_ip_init()
3090 		if (amdgpu_xgmi_add_device(adev) == 0) {  in amdgpu_device_ip_init()
3091 			if (!amdgpu_sriov_vf(adev)) {  in amdgpu_device_ip_init()
3092 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);  in amdgpu_device_ip_init()
3107 				amdgpu_reset_put_reset_domain(adev->reset_domain);  in amdgpu_device_ip_init()
3108 				adev->reset_domain = hive->reset_domain;  in amdgpu_device_ip_init()
3114 	r = amdgpu_device_init_schedulers(adev);  in amdgpu_device_ip_init()
3118 	if (adev->mman.buffer_funcs_ring->sched.ready)  in amdgpu_device_ip_init()
3119 		amdgpu_ttm_set_buffer_funcs_status(adev, true);  in amdgpu_device_ip_init()
3122 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {  in amdgpu_device_ip_init()
3123 		kgd2kfd_init_zone_device(adev);  in amdgpu_device_ip_init()
3124 		amdgpu_amdkfd_device_init(adev);  in amdgpu_device_ip_init()
3127 	amdgpu_fru_get_product_info(adev);  in amdgpu_device_ip_init()
3129 	if (!amdgpu_sriov_vf(adev) || amdgpu_sriov_ras_cper_en(adev))  in amdgpu_device_ip_init()
3130 		r = amdgpu_cper_init(adev);  in amdgpu_device_ip_init()
3146 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)  in amdgpu_device_fill_reset_magic()  argument
3148 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);  in amdgpu_device_fill_reset_magic()
3161 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)  in amdgpu_device_check_vram_lost()  argument
3163 	if (memcmp(adev->gart.ptr, adev->reset_magic,  in amdgpu_device_check_vram_lost()
3167 	if (!amdgpu_in_reset(adev))  in amdgpu_device_check_vram_lost()
3174 	switch (amdgpu_asic_reset_method(adev)) {  in amdgpu_device_check_vram_lost()
3196 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,  in amdgpu_device_set_cg_state()  argument
3204 	for (j = 0; j < adev->num_ip_blocks; j++) {  in amdgpu_device_set_cg_state()
3205 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;  in amdgpu_device_set_cg_state()
3206 		if (!adev->ip_blocks[i].status.late_initialized)  in amdgpu_device_set_cg_state()
3209 		if (adev->in_s0ix &&  in amdgpu_device_set_cg_state()
3210 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||  in amdgpu_device_set_cg_state()
3211 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))  in amdgpu_device_set_cg_state()
3214 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&  in amdgpu_device_set_cg_state()
3215 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&  in amdgpu_device_set_cg_state()
3216 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&  in amdgpu_device_set_cg_state()
3217 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&  in amdgpu_device_set_cg_state()
3218 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {  in amdgpu_device_set_cg_state()
3220 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i],  in amdgpu_device_set_cg_state()
3224 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_set_cg_state()
3233 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,  in amdgpu_device_set_pg_state()  argument
3241 	for (j = 0; j < adev->num_ip_blocks; j++) {  in amdgpu_device_set_pg_state()
3242 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;  in amdgpu_device_set_pg_state()
3243 		if (!adev->ip_blocks[i].status.late_initialized)  in amdgpu_device_set_pg_state()
3246 		if (adev->in_s0ix &&  in amdgpu_device_set_pg_state()
3247 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||  in amdgpu_device_set_pg_state()
3248 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))  in amdgpu_device_set_pg_state()
3251 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&  in amdgpu_device_set_pg_state()
3252 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&  in amdgpu_device_set_pg_state()
3253 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&  in amdgpu_device_set_pg_state()
3254 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&  in amdgpu_device_set_pg_state()
3255 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {  in amdgpu_device_set_pg_state()
3257 			r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i],  in amdgpu_device_set_pg_state()
3261 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_set_pg_state()
3272 	struct amdgpu_device *adev;  in amdgpu_device_enable_mgpu_fan_boost()  local
3287 		adev = gpu_ins->adev;  in amdgpu_device_enable_mgpu_fan_boost()
3288 		if (!(adev->flags & AMD_IS_APU) &&  in amdgpu_device_enable_mgpu_fan_boost()
3290 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);  in amdgpu_device_enable_mgpu_fan_boost()
3316 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)  in amdgpu_device_ip_late_init()  argument
3321 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_late_init()
3322 		if (!adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_late_init()
3324 		if (adev->ip_blocks[i].version->funcs->late_init) {  in amdgpu_device_ip_late_init()
3325 			r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_late_init()
3328 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_late_init()
3332 		adev->ip_blocks[i].status.late_initialized = true;  in amdgpu_device_ip_late_init()
3335 	r = amdgpu_ras_late_init(adev);  in amdgpu_device_ip_late_init()
3341 	if (!amdgpu_reset_in_recovery(adev))  in amdgpu_device_ip_late_init()
3342 		amdgpu_ras_set_error_query_ready(adev, true);  in amdgpu_device_ip_late_init()
3344 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);  in amdgpu_device_ip_late_init()
3345 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);  in amdgpu_device_ip_late_init()
3347 	amdgpu_device_fill_reset_magic(adev);  in amdgpu_device_ip_late_init()
3354 	if (amdgpu_passthrough(adev) &&  in amdgpu_device_ip_late_init()
3355 	    ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||  in amdgpu_device_ip_late_init()
3356 	     adev->asic_type == CHIP_ALDEBARAN))  in amdgpu_device_ip_late_init()
3357 		amdgpu_dpm_handle_passthrough_sbr(adev, true);  in amdgpu_device_ip_late_init()
3359 	if (adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_device_ip_late_init()
3375 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {  in amdgpu_device_ip_late_init()
3378 				if (gpu_instance->adev->flags & AMD_IS_APU)  in amdgpu_device_ip_late_init()
3381 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,  in amdgpu_device_ip_late_init()
3422 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)  in amdgpu_device_smu_fini_early()  argument
3426 	if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))  in amdgpu_device_smu_fini_early()
3429 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_smu_fini_early()
3430 		if (!adev->ip_blocks[i].status.hw)  in amdgpu_device_smu_fini_early()
3432 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {  in amdgpu_device_smu_fini_early()
3433 			amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);  in amdgpu_device_smu_fini_early()
3439 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)  in amdgpu_device_ip_fini_early()  argument
3443 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_fini_early()
3444 		if (!adev->ip_blocks[i].version->funcs->early_fini)  in amdgpu_device_ip_fini_early()
3447 		r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]);  in amdgpu_device_ip_fini_early()
3450 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_fini_early()
3454 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);  in amdgpu_device_ip_fini_early()
3455 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);  in amdgpu_device_ip_fini_early()
3457 	amdgpu_amdkfd_suspend(adev, false);  in amdgpu_device_ip_fini_early()
3460 	amdgpu_device_smu_fini_early(adev);  in amdgpu_device_ip_fini_early()
3462 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_fini_early()
3463 		if (!adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_fini_early()
3466 		amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);  in amdgpu_device_ip_fini_early()
3469 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_ip_fini_early()
3470 		if (amdgpu_virt_release_full_gpu(adev, false))  in amdgpu_device_ip_fini_early()
3488 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)  in amdgpu_device_ip_fini()  argument
3492 	amdgpu_cper_fini(adev);  in amdgpu_device_ip_fini()
3494 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)  in amdgpu_device_ip_fini()
3495 		amdgpu_virt_release_ras_err_handler_data(adev);  in amdgpu_device_ip_fini()
3497 	if (adev->gmc.xgmi.num_physical_nodes > 1)  in amdgpu_device_ip_fini()
3498 		amdgpu_xgmi_remove_device(adev);  in amdgpu_device_ip_fini()
3500 	amdgpu_amdkfd_device_fini_sw(adev);  in amdgpu_device_ip_fini()
3502 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_fini()
3503 		if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_ip_fini()
3506 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {  in amdgpu_device_ip_fini()
3507 			amdgpu_ucode_free_bo(adev);  in amdgpu_device_ip_fini()
3508 			amdgpu_free_static_csa(&adev->virt.csa_obj);  in amdgpu_device_ip_fini()
3509 			amdgpu_device_wb_fini(adev);  in amdgpu_device_ip_fini()
3510 			amdgpu_device_mem_scratch_fini(adev);  in amdgpu_device_ip_fini()
3511 			amdgpu_ib_pool_fini(adev);  in amdgpu_device_ip_fini()
3512 			amdgpu_seq64_fini(adev);  in amdgpu_device_ip_fini()
3513 			amdgpu_doorbell_fini(adev);  in amdgpu_device_ip_fini()
3515 		if (adev->ip_blocks[i].version->funcs->sw_fini) {  in amdgpu_device_ip_fini()
3516 			r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]);  in amdgpu_device_ip_fini()
3520 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_fini()
3523 		adev->ip_blocks[i].status.sw = false;  in amdgpu_device_ip_fini()
3524 		adev->ip_blocks[i].status.valid = false;  in amdgpu_device_ip_fini()
3527 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_fini()
3528 		if (!adev->ip_blocks[i].status.late_initialized)  in amdgpu_device_ip_fini()
3530 		if (adev->ip_blocks[i].version->funcs->late_fini)  in amdgpu_device_ip_fini()
3531 			adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]);  in amdgpu_device_ip_fini()
3532 		adev->ip_blocks[i].status.late_initialized = false;  in amdgpu_device_ip_fini()
3535 	amdgpu_ras_fini(adev);  in amdgpu_device_ip_fini()
3547 	struct amdgpu_device *adev =  in amdgpu_device_delayed_init_work_handler()  local
3551 	r = amdgpu_ib_ring_tests(adev);  in amdgpu_device_delayed_init_work_handler()
3558 	struct amdgpu_device *adev =  in amdgpu_device_delay_enable_gfx_off()  local
3561 	WARN_ON_ONCE(adev->gfx.gfx_off_state);  in amdgpu_device_delay_enable_gfx_off()
3562 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);  in amdgpu_device_delay_enable_gfx_off()
3564 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))  in amdgpu_device_delay_enable_gfx_off()
3565 		adev->gfx.gfx_off_state = true;  in amdgpu_device_delay_enable_gfx_off()
3579 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)  in amdgpu_device_ip_suspend_phase1()  argument
3583 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);  in amdgpu_device_ip_suspend_phase1()
3584 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);  in amdgpu_device_ip_suspend_phase1()
3591 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))  in amdgpu_device_ip_suspend_phase1()
3592 		dev_warn(adev->dev, "Failed to disallow df cstate");  in amdgpu_device_ip_suspend_phase1()
3594 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_suspend_phase1()
3595 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_suspend_phase1()
3599 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)  in amdgpu_device_ip_suspend_phase1()
3603 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);  in amdgpu_device_ip_suspend_phase1()
3622 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)  in amdgpu_device_ip_suspend_phase2()  argument
3626 	if (adev->in_s0ix)  in amdgpu_device_ip_suspend_phase2()
3627 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);  in amdgpu_device_ip_suspend_phase2()
3629 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_suspend_phase2()
3630 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_suspend_phase2()
3633 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)  in amdgpu_device_ip_suspend_phase2()
3637 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {  in amdgpu_device_ip_suspend_phase2()
3638 			adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_suspend_phase2()
3644 			    adev, adev->ip_blocks[i].version->type))  in amdgpu_device_ip_suspend_phase2()
3650 		if (adev->in_s0ix &&  in amdgpu_device_ip_suspend_phase2()
3651 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX &&  in amdgpu_device_ip_suspend_phase2()
3652 		    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0))  in amdgpu_device_ip_suspend_phase2()
3653 			cancel_delayed_work_sync(&adev->gfx.idle_work);  in amdgpu_device_ip_suspend_phase2()
3659 		if (adev->in_s0ix &&  in amdgpu_device_ip_suspend_phase2()
3660 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||  in amdgpu_device_ip_suspend_phase2()
3661 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||  in amdgpu_device_ip_suspend_phase2()
3662 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))  in amdgpu_device_ip_suspend_phase2()
3666 		if (adev->in_s0ix &&  in amdgpu_device_ip_suspend_phase2()
3667 		    (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=  in amdgpu_device_ip_suspend_phase2()
3669 		    (adev->ip_blocks[i].version->type ==  in amdgpu_device_ip_suspend_phase2()
3680 		if (amdgpu_in_reset(adev) &&  in amdgpu_device_ip_suspend_phase2()
3681 		    (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&  in amdgpu_device_ip_suspend_phase2()
3682 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)  in amdgpu_device_ip_suspend_phase2()
3686 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);  in amdgpu_device_ip_suspend_phase2()
3687 		adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_suspend_phase2()
3690 		if (!amdgpu_sriov_vf(adev)) {  in amdgpu_device_ip_suspend_phase2()
3691 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {  in amdgpu_device_ip_suspend_phase2()
3692 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);  in amdgpu_device_ip_suspend_phase2()
3695 							adev->mp1_state, r);  in amdgpu_device_ip_suspend_phase2()
3716 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)  in amdgpu_device_ip_suspend()  argument
3720 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_ip_suspend()
3721 		amdgpu_virt_fini_data_exchange(adev);  in amdgpu_device_ip_suspend()
3722 		amdgpu_virt_request_full_gpu(adev, false);  in amdgpu_device_ip_suspend()
3725 	amdgpu_ttm_set_buffer_funcs_status(adev, false);  in amdgpu_device_ip_suspend()
3727 	r = amdgpu_device_ip_suspend_phase1(adev);  in amdgpu_device_ip_suspend()
3730 	r = amdgpu_device_ip_suspend_phase2(adev);  in amdgpu_device_ip_suspend()
3732 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_ip_suspend()
3733 		amdgpu_virt_release_full_gpu(adev, false);  in amdgpu_device_ip_suspend()
3738 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)  in amdgpu_device_ip_reinit_early_sriov()  argument
3749 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_reinit_early_sriov()
3753 		block = &adev->ip_blocks[i];  in amdgpu_device_ip_reinit_early_sriov()
3762 			r = block->version->funcs->hw_init(&adev->ip_blocks[i]);  in amdgpu_device_ip_reinit_early_sriov()
3764 				dev_err(adev->dev, "RE-INIT-early: %s failed\n",  in amdgpu_device_ip_reinit_early_sriov()
3775 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)  in amdgpu_device_ip_reinit_late_sriov()  argument
3793 		block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]);  in amdgpu_device_ip_reinit_late_sriov()
3806 				dev_err(adev->dev, "RE-INIT-late: %s failed\n",  in amdgpu_device_ip_reinit_late_sriov()
3829 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)  in amdgpu_device_ip_resume_phase1()  argument
3833 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_resume_phase1()
3834 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_resume_phase1()
3836 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||  in amdgpu_device_ip_resume_phase1()
3837 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||  in amdgpu_device_ip_resume_phase1()
3838 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||  in amdgpu_device_ip_resume_phase1()
3839 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {  in amdgpu_device_ip_resume_phase1()
3841 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);  in amdgpu_device_ip_resume_phase1()
3863 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)  in amdgpu_device_ip_resume_phase2()  argument
3867 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_resume_phase2()
3868 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_resume_phase2()
3870 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||  in amdgpu_device_ip_resume_phase2()
3871 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||  in amdgpu_device_ip_resume_phase2()
3872 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||  in amdgpu_device_ip_resume_phase2()
3873 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE ||  in amdgpu_device_ip_resume_phase2()
3874 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)  in amdgpu_device_ip_resume_phase2()
3876 		r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);  in amdgpu_device_ip_resume_phase2()
3897 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)  in amdgpu_device_ip_resume_phase3()  argument
3901 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_resume_phase3()
3902 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_resume_phase3()
3904 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {  in amdgpu_device_ip_resume_phase3()
3905 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);  in amdgpu_device_ip_resume_phase3()
3926 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)  in amdgpu_device_ip_resume()  argument
3930 	r = amdgpu_device_ip_resume_phase1(adev);  in amdgpu_device_ip_resume()
3934 	r = amdgpu_device_fw_loading(adev);  in amdgpu_device_ip_resume()
3938 	r = amdgpu_device_ip_resume_phase2(adev);  in amdgpu_device_ip_resume()
3940 	if (adev->mman.buffer_funcs_ring->sched.ready)  in amdgpu_device_ip_resume()
3941 		amdgpu_ttm_set_buffer_funcs_status(adev, true);  in amdgpu_device_ip_resume()
3946 	amdgpu_fence_driver_hw_init(adev);  in amdgpu_device_ip_resume()
3948 	r = amdgpu_device_ip_resume_phase3(adev);  in amdgpu_device_ip_resume()
3960 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)  in amdgpu_device_detect_sriov_bios()  argument
3962 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_detect_sriov_bios()
3963 		if (adev->is_atom_fw) {  in amdgpu_device_detect_sriov_bios()
3964 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))  in amdgpu_device_detect_sriov_bios()
3965 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;  in amdgpu_device_detect_sriov_bios()
3967 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))  in amdgpu_device_detect_sriov_bios()
3968 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;  in amdgpu_device_detect_sriov_bios()
3971 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))  in amdgpu_device_detect_sriov_bios()
3972 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);  in amdgpu_device_detect_sriov_bios()
4040 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)  in amdgpu_device_has_dc_support()  argument
4042 	if (adev->enable_virtual_display ||  in amdgpu_device_has_dc_support()
4043 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))  in amdgpu_device_has_dc_support()
4046 	return amdgpu_device_asic_has_dc_support(adev->asic_type);  in amdgpu_device_has_dc_support()
4051 	struct amdgpu_device *adev =  in amdgpu_device_xgmi_reset_func()  local
4053 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);  in amdgpu_device_xgmi_reset_func()
4065 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {  in amdgpu_device_xgmi_reset_func()
4068 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));  in amdgpu_device_xgmi_reset_func()
4070 		if (adev->asic_reset_res)  in amdgpu_device_xgmi_reset_func()
4074 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));  in amdgpu_device_xgmi_reset_func()
4076 		if (adev->asic_reset_res)  in amdgpu_device_xgmi_reset_func()
4079 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);  in amdgpu_device_xgmi_reset_func()
4083 		adev->asic_reset_res =  amdgpu_asic_reset(adev);  in amdgpu_device_xgmi_reset_func()
4087 	if (adev->asic_reset_res)  in amdgpu_device_xgmi_reset_func()
4089 			 adev->asic_reset_res, adev_to_drm(adev)->unique);  in amdgpu_device_xgmi_reset_func()
4093 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)  in amdgpu_device_get_job_timeout_settings()  argument
4107 	adev->gfx_timeout = msecs_to_jiffies(10000);  in amdgpu_device_get_job_timeout_settings()
4108 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;  in amdgpu_device_get_job_timeout_settings()
4109 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_get_job_timeout_settings()
4110 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?  in amdgpu_device_get_job_timeout_settings()
4113 		adev->compute_timeout =  msecs_to_jiffies(60000);  in amdgpu_device_get_job_timeout_settings()
4127 				dev_warn(adev->dev, "lockup timeout disabled");  in amdgpu_device_get_job_timeout_settings()
4135 				adev->gfx_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
4138 				adev->compute_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
4141 				adev->sdma_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
4144 				adev->video_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
4155 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;  in amdgpu_device_get_job_timeout_settings()
4156 			if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))  in amdgpu_device_get_job_timeout_settings()
4157 				adev->compute_timeout = adev->gfx_timeout;  in amdgpu_device_get_job_timeout_settings()
4171 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)  in amdgpu_device_check_iommu_direct_map()  argument
4175 	domain = iommu_get_domain_for_dev(adev->dev);  in amdgpu_device_check_iommu_direct_map()
4177 		adev->ram_is_direct_mapped = true;  in amdgpu_device_check_iommu_direct_map()
4188 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)  in amdgpu_device_check_iommu_remap()  argument
4192 	domain = iommu_get_domain_for_dev(adev->dev);  in amdgpu_device_check_iommu_remap()
4201 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)  in amdgpu_device_set_mcbp()  argument
4204 		adev->gfx.mcbp = true;  in amdgpu_device_set_mcbp()
4206 		adev->gfx.mcbp = false;  in amdgpu_device_set_mcbp()
4208 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_set_mcbp()
4209 		adev->gfx.mcbp = true;  in amdgpu_device_set_mcbp()
4211 	if (adev->gfx.mcbp)  in amdgpu_device_set_mcbp()
4225 int amdgpu_device_init(struct amdgpu_device *adev,  in amdgpu_device_init()  argument
4228 	struct drm_device *ddev = adev_to_drm(adev);  in amdgpu_device_init()
4229 	struct pci_dev *pdev = adev->pdev;  in amdgpu_device_init()
4235 	adev->shutdown = false;  in amdgpu_device_init()
4236 	adev->flags = flags;  in amdgpu_device_init()
4239 		adev->asic_type = amdgpu_force_asic_type;  in amdgpu_device_init()
4241 		adev->asic_type = flags & AMD_ASIC_MASK;  in amdgpu_device_init()
4243 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;  in amdgpu_device_init()
4245 		adev->usec_timeout *= 10;  in amdgpu_device_init()
4246 	adev->gmc.gart_size = 512 * 1024 * 1024;  in amdgpu_device_init()
4247 	adev->accel_working = false;  in amdgpu_device_init()
4248 	adev->num_rings = 0;  in amdgpu_device_init()
4249 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());  in amdgpu_device_init()
4250 	adev->mman.buffer_funcs = NULL;  in amdgpu_device_init()
4251 	adev->mman.buffer_funcs_ring = NULL;  in amdgpu_device_init()
4252 	adev->vm_manager.vm_pte_funcs = NULL;  in amdgpu_device_init()
4253 	adev->vm_manager.vm_pte_num_scheds = 0;  in amdgpu_device_init()
4254 	adev->gmc.gmc_funcs = NULL;  in amdgpu_device_init()
4255 	adev->harvest_ip_mask = 0x0;  in amdgpu_device_init()
4256 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);  in amdgpu_device_init()
4257 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);  in amdgpu_device_init()
4259 	adev->smc_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
4260 	adev->smc_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
4261 	adev->pcie_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
4262 	adev->pcie_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
4263 	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;  in amdgpu_device_init()
4264 	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;  in amdgpu_device_init()
4265 	adev->pciep_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
4266 	adev->pciep_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
4267 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;  in amdgpu_device_init()
4268 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;  in amdgpu_device_init()
4269 	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;  in amdgpu_device_init()
4270 	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;  in amdgpu_device_init()
4271 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
4272 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
4273 	adev->didt_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
4274 	adev->didt_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
4275 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
4276 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
4277 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;  in amdgpu_device_init()
4278 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;  in amdgpu_device_init()
4281 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,  in amdgpu_device_init()
4287 	mutex_init(&adev->firmware.mutex);  in amdgpu_device_init()
4288 	mutex_init(&adev->pm.mutex);  in amdgpu_device_init()
4289 	mutex_init(&adev->gfx.gpu_clock_mutex);  in amdgpu_device_init()
4290 	mutex_init(&adev->srbm_mutex);  in amdgpu_device_init()
4291 	mutex_init(&adev->gfx.pipe_reserve_mutex);  in amdgpu_device_init()
4292 	mutex_init(&adev->gfx.gfx_off_mutex);  in amdgpu_device_init()
4293 	mutex_init(&adev->gfx.partition_mutex);  in amdgpu_device_init()
4294 	mutex_init(&adev->grbm_idx_mutex);  in amdgpu_device_init()
4295 	mutex_init(&adev->mn_lock);  in amdgpu_device_init()
4296 	mutex_init(&adev->virt.vf_errors.lock);  in amdgpu_device_init()
4297 	hash_init(adev->mn_hash);  in amdgpu_device_init()
4298 	mutex_init(&adev->psp.mutex);  in amdgpu_device_init()
4299 	mutex_init(&adev->notifier_lock);  in amdgpu_device_init()
4300 	mutex_init(&adev->pm.stable_pstate_ctx_lock);  in amdgpu_device_init()
4301 	mutex_init(&adev->benchmark_mutex);  in amdgpu_device_init()
4302 	mutex_init(&adev->gfx.reset_sem_mutex);  in amdgpu_device_init()
4304 	mutex_init(&adev->enforce_isolation_mutex);  in amdgpu_device_init()
4306 		adev->isolation[i].spearhead = dma_fence_get_stub();  in amdgpu_device_init()
4307 		amdgpu_sync_create(&adev->isolation[i].active);  in amdgpu_device_init()
4308 		amdgpu_sync_create(&adev->isolation[i].prev);  in amdgpu_device_init()
4310 	mutex_init(&adev->gfx.kfd_sch_mutex);  in amdgpu_device_init()
4311 	mutex_init(&adev->gfx.workload_profile_mutex);  in amdgpu_device_init()
4312 	mutex_init(&adev->vcn.workload_profile_mutex);  in amdgpu_device_init()
4314 	amdgpu_device_init_apu_flags(adev);  in amdgpu_device_init()
4316 	r = amdgpu_device_check_arguments(adev);  in amdgpu_device_init()
4320 	spin_lock_init(&adev->mmio_idx_lock);  in amdgpu_device_init()
4321 	spin_lock_init(&adev->smc_idx_lock);  in amdgpu_device_init()
4322 	spin_lock_init(&adev->pcie_idx_lock);  in amdgpu_device_init()
4323 	spin_lock_init(&adev->uvd_ctx_idx_lock);  in amdgpu_device_init()
4324 	spin_lock_init(&adev->didt_idx_lock);  in amdgpu_device_init()
4325 	spin_lock_init(&adev->gc_cac_idx_lock);  in amdgpu_device_init()
4326 	spin_lock_init(&adev->se_cac_idx_lock);  in amdgpu_device_init()
4327 	spin_lock_init(&adev->audio_endpt_idx_lock);  in amdgpu_device_init()
4328 	spin_lock_init(&adev->mm_stats.lock);  in amdgpu_device_init()
4329 	spin_lock_init(&adev->virt.rlcg_reg_lock);  in amdgpu_device_init()
4330 	spin_lock_init(&adev->wb.lock);  in amdgpu_device_init()
4332 	INIT_LIST_HEAD(&adev->reset_list);  in amdgpu_device_init()
4334 	INIT_LIST_HEAD(&adev->ras_list);  in amdgpu_device_init()
4336 	INIT_LIST_HEAD(&adev->pm.od_kobj_list);  in amdgpu_device_init()
4338 	INIT_DELAYED_WORK(&adev->delayed_init_work,  in amdgpu_device_init()
4340 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,  in amdgpu_device_init()
4352 		INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work,  in amdgpu_device_init()
4354 		adev->gfx.enforce_isolation[i].adev = adev;  in amdgpu_device_init()
4355 		adev->gfx.enforce_isolation[i].xcp_id = i;  in amdgpu_device_init()
4358 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);  in amdgpu_device_init()
4360 	adev->gfx.gfx_off_req_count = 1;  in amdgpu_device_init()
4361 	adev->gfx.gfx_off_residency = 0;  in amdgpu_device_init()
4362 	adev->gfx.gfx_off_entrycount = 0;  in amdgpu_device_init()
4363 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;  in amdgpu_device_init()
4365 	atomic_set(&adev->throttling_logging_enabled, 1);  in amdgpu_device_init()
4373 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);  in amdgpu_device_init()
4375 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);  in amdgpu_device_init()
4379 	if (adev->asic_type >= CHIP_BONAIRE) {  in amdgpu_device_init()
4380 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);  in amdgpu_device_init()
4381 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);  in amdgpu_device_init()
4383 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);  in amdgpu_device_init()
4384 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);  in amdgpu_device_init()
4388 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);  in amdgpu_device_init()
4390 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);  in amdgpu_device_init()
4391 	if (!adev->rmmio)  in amdgpu_device_init()
4394 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);  in amdgpu_device_init()
4395 	DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);  in amdgpu_device_init()
4402 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");  in amdgpu_device_init()
4403 	if (!adev->reset_domain)  in amdgpu_device_init()
4407 	amdgpu_virt_init(adev);  in amdgpu_device_init()
4409 	amdgpu_device_get_pcie_info(adev);  in amdgpu_device_init()
4411 	r = amdgpu_device_get_job_timeout_settings(adev);  in amdgpu_device_init()
4413 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");  in amdgpu_device_init()
4417 	amdgpu_device_set_mcbp(adev);  in amdgpu_device_init()
4424 	amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);  in amdgpu_device_init()
4426 	r = amdgpu_device_ip_early_init(adev);  in amdgpu_device_init()
4437 		r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);  in amdgpu_device_init()
4443 	amdgpu_gmc_tmz_set(adev);  in amdgpu_device_init()
4445 	if (amdgpu_sriov_vf(adev) &&  in amdgpu_device_init()
4446 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))  in amdgpu_device_init()
4450 		adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;  in amdgpu_device_init()
4452 	amdgpu_gmc_noretry_set(adev);  in amdgpu_device_init()
4454 	if (adev->gmc.xgmi.supported) {  in amdgpu_device_init()
4455 		r = adev->gfxhub.funcs->get_xgmi_info(adev);  in amdgpu_device_init()
4461 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_init()
4462 		if (adev->virt.fw_reserve.p_pf2vf)  in amdgpu_device_init()
4463 			adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)  in amdgpu_device_init()
4464 						      adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==  in amdgpu_device_init()
4469 	} else if ((adev->flags & AMD_IS_APU) &&  in amdgpu_device_init()
4470 		   (amdgpu_ip_version(adev, GC_HWIP, 0) >  in amdgpu_device_init()
4472 		adev->have_atomics_support = true;  in amdgpu_device_init()
4474 		adev->have_atomics_support =  in amdgpu_device_init()
4475 			!pci_enable_atomic_ops_to_root(adev->pdev,  in amdgpu_device_init()
4480 	if (!adev->have_atomics_support)  in amdgpu_device_init()
4481 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");  in amdgpu_device_init()
4484 	amdgpu_doorbell_init(adev);  in amdgpu_device_init()
4488 		emu_soc_asic_init(adev);  in amdgpu_device_init()
4492 	amdgpu_reset_init(adev);  in amdgpu_device_init()
4495 	if (adev->bios)  in amdgpu_device_init()
4496 		amdgpu_device_detect_sriov_bios(adev);  in amdgpu_device_init()
4501 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {  in amdgpu_device_init()
4502 		if (adev->gmc.xgmi.num_physical_nodes) {  in amdgpu_device_init()
4503 			dev_info(adev->dev, "Pending hive reset.\n");  in amdgpu_device_init()
4504 			amdgpu_set_init_level(adev,  in amdgpu_device_init()
4506 		} else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) &&  in amdgpu_device_init()
4507 				   !amdgpu_device_has_display_hardware(adev)) {  in amdgpu_device_init()
4508 					r = psp_gpu_reset(adev);  in amdgpu_device_init()
4515 				r = amdgpu_asic_reset(adev);  in amdgpu_device_init()
4520 		  dev_err(adev->dev, "asic reset on init failed\n");  in amdgpu_device_init()
4526 	if (amdgpu_device_need_post(adev)) {  in amdgpu_device_init()
4527 		if (!adev->bios) {  in amdgpu_device_init()
4528 			dev_err(adev->dev, "no vBIOS found\n");  in amdgpu_device_init()
4533 		r = amdgpu_device_asic_init(adev);  in amdgpu_device_init()
4535 			dev_err(adev->dev, "gpu post error!\n");  in amdgpu_device_init()
4540 	if (adev->bios) {  in amdgpu_device_init()
4541 		if (adev->is_atom_fw) {  in amdgpu_device_init()
4543 			r = amdgpu_atomfirmware_get_clock_info(adev);  in amdgpu_device_init()
4545 				dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");  in amdgpu_device_init()
4546 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);  in amdgpu_device_init()
4551 			r = amdgpu_atombios_get_clock_info(adev);  in amdgpu_device_init()
4553 				dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");  in amdgpu_device_init()
4554 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);  in amdgpu_device_init()
4558 			amdgpu_i2c_init(adev);  in amdgpu_device_init()
4564 	r = amdgpu_fence_driver_sw_init(adev);  in amdgpu_device_init()
4566 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");  in amdgpu_device_init()
4567 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);  in amdgpu_device_init()
4572 	drm_mode_config_init(adev_to_drm(adev));  in amdgpu_device_init()
4574 	r = amdgpu_device_ip_init(adev);  in amdgpu_device_init()
4576 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");  in amdgpu_device_init()
4577 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);  in amdgpu_device_init()
4581 	amdgpu_fence_driver_hw_init(adev);  in amdgpu_device_init()
4583 	dev_info(adev->dev,  in amdgpu_device_init()
4585 			adev->gfx.config.max_shader_engines,  in amdgpu_device_init()
4586 			adev->gfx.config.max_sh_per_se,  in amdgpu_device_init()
4587 			adev->gfx.config.max_cu_per_sh,  in amdgpu_device_init()
4588 			adev->gfx.cu_info.number);  in amdgpu_device_init()
4590 	adev->accel_working = true;  in amdgpu_device_init()
4592 	amdgpu_vm_check_compute_bug(adev);  in amdgpu_device_init()
4600 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));  in amdgpu_device_init()
4607 	amdgpu_register_gpu_instance(adev);  in amdgpu_device_init()
4612 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {  in amdgpu_device_init()
4613 		r = amdgpu_device_ip_late_init(adev);  in amdgpu_device_init()
4615 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");  in amdgpu_device_init()
4616 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);  in amdgpu_device_init()
4620 		amdgpu_ras_resume(adev);  in amdgpu_device_init()
4621 		queue_delayed_work(system_wq, &adev->delayed_init_work,  in amdgpu_device_init()
4625 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_init()
4626 		amdgpu_virt_release_full_gpu(adev, true);  in amdgpu_device_init()
4627 		flush_delayed_work(&adev->delayed_init_work);  in amdgpu_device_init()
4635 	r = amdgpu_atombios_sysfs_init(adev);  in amdgpu_device_init()
4637 		drm_err(&adev->ddev,  in amdgpu_device_init()
4640 	r = amdgpu_pm_sysfs_init(adev);  in amdgpu_device_init()
4644 	r = amdgpu_ucode_sysfs_init(adev);  in amdgpu_device_init()
4646 		adev->ucode_sysfs_en = false;  in amdgpu_device_init()
4649 		adev->ucode_sysfs_en = true;  in amdgpu_device_init()
4651 	r = amdgpu_device_attr_sysfs_init(adev);  in amdgpu_device_init()
4653 		dev_err(adev->dev, "Could not create amdgpu device attr\n");  in amdgpu_device_init()
4655 	r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);  in amdgpu_device_init()
4657 		dev_err(adev->dev,  in amdgpu_device_init()
4660 	amdgpu_fru_sysfs_init(adev);  in amdgpu_device_init()
4661 	amdgpu_reg_state_sysfs_init(adev);  in amdgpu_device_init()
4662 	amdgpu_xcp_cfg_sysfs_init(adev);  in amdgpu_device_init()
4665 		r = amdgpu_pmu_init(adev);  in amdgpu_device_init()
4667 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");  in amdgpu_device_init()
4670 	if (amdgpu_device_cache_pci_state(adev->pdev))  in amdgpu_device_init()
4677 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)  in amdgpu_device_init()
4678 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);  in amdgpu_device_init()
4682 	if (px || (!dev_is_removable(&adev->pdev->dev) &&  in amdgpu_device_init()
4684 		vga_switcheroo_register_client(adev->pdev,  in amdgpu_device_init()
4688 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);  in amdgpu_device_init()
4690 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)  in amdgpu_device_init()
4691 		amdgpu_xgmi_reset_on_init(adev);  in amdgpu_device_init()
4693 	amdgpu_device_check_iommu_direct_map(adev);  in amdgpu_device_init()
4695 	adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;  in amdgpu_device_init()
4696 	r = register_pm_notifier(&adev->pm_nb);  in amdgpu_device_init()
4703 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_init()
4704 		amdgpu_virt_release_full_gpu(adev, true);  in amdgpu_device_init()
4707 	if (amdgpu_sriov_vf(adev) &&  in amdgpu_device_init()
4708 		!amdgpu_sriov_runtime(adev) &&  in amdgpu_device_init()
4709 		amdgpu_virt_mmio_blocked(adev) &&  in amdgpu_device_init()
4710 		!amdgpu_virt_wait_reset(adev)) {  in amdgpu_device_init()
4711 		dev_err(adev->dev, "VF exclusive mode timeout\n");  in amdgpu_device_init()
4713 		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;  in amdgpu_device_init()
4714 		adev->virt.ops = NULL;  in amdgpu_device_init()
4717 	amdgpu_release_ras_context(adev);  in amdgpu_device_init()
4720 	amdgpu_vf_error_trans_all(adev);  in amdgpu_device_init()
4725 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)  in amdgpu_device_unmap_mmio()  argument
4729 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);  in amdgpu_device_unmap_mmio()
4732 	amdgpu_doorbell_fini(adev);  in amdgpu_device_unmap_mmio()
4734 	iounmap(adev->rmmio);  in amdgpu_device_unmap_mmio()
4735 	adev->rmmio = NULL;  in amdgpu_device_unmap_mmio()
4736 	if (adev->mman.aper_base_kaddr)  in amdgpu_device_unmap_mmio()
4737 		iounmap(adev->mman.aper_base_kaddr);  in amdgpu_device_unmap_mmio()
4738 	adev->mman.aper_base_kaddr = NULL;  in amdgpu_device_unmap_mmio()
4741 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {  in amdgpu_device_unmap_mmio()
4742 		arch_phys_wc_del(adev->gmc.vram_mtrr);  in amdgpu_device_unmap_mmio()
4743 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);  in amdgpu_device_unmap_mmio()
4755 void amdgpu_device_fini_hw(struct amdgpu_device *adev)  in amdgpu_device_fini_hw()  argument
4757 	dev_info(adev->dev, "amdgpu: finishing device.\n");  in amdgpu_device_fini_hw()
4758 	flush_delayed_work(&adev->delayed_init_work);  in amdgpu_device_fini_hw()
4760 	if (adev->mman.initialized)  in amdgpu_device_fini_hw()
4761 		drain_workqueue(adev->mman.bdev.wq);  in amdgpu_device_fini_hw()
4762 	adev->shutdown = true;  in amdgpu_device_fini_hw()
4764 	unregister_pm_notifier(&adev->pm_nb);  in amdgpu_device_fini_hw()
4769 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_fini_hw()
4770 		amdgpu_virt_request_full_gpu(adev, false);  in amdgpu_device_fini_hw()
4771 		amdgpu_virt_fini_data_exchange(adev);  in amdgpu_device_fini_hw()
4775 	amdgpu_irq_disable_all(adev);  in amdgpu_device_fini_hw()
4776 	if (adev->mode_info.mode_config_initialized) {  in amdgpu_device_fini_hw()
4777 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))  in amdgpu_device_fini_hw()
4778 			drm_helper_force_disable_all(adev_to_drm(adev));  in amdgpu_device_fini_hw()
4780 			drm_atomic_helper_shutdown(adev_to_drm(adev));  in amdgpu_device_fini_hw()
4782 	amdgpu_fence_driver_hw_fini(adev);  in amdgpu_device_fini_hw()
4784 	if (adev->pm.sysfs_initialized)  in amdgpu_device_fini_hw()
4785 		amdgpu_pm_sysfs_fini(adev);  in amdgpu_device_fini_hw()
4786 	if (adev->ucode_sysfs_en)  in amdgpu_device_fini_hw()
4787 		amdgpu_ucode_sysfs_fini(adev);  in amdgpu_device_fini_hw()
4788 	amdgpu_device_attr_sysfs_fini(adev);  in amdgpu_device_fini_hw()
4789 	amdgpu_fru_sysfs_fini(adev);  in amdgpu_device_fini_hw()
4791 	amdgpu_reg_state_sysfs_fini(adev);  in amdgpu_device_fini_hw()
4792 	amdgpu_xcp_cfg_sysfs_fini(adev);  in amdgpu_device_fini_hw()
4795 	amdgpu_ras_pre_fini(adev);  in amdgpu_device_fini_hw()
4797 	amdgpu_ttm_set_buffer_funcs_status(adev, false);  in amdgpu_device_fini_hw()
4799 	amdgpu_device_ip_fini_early(adev);  in amdgpu_device_fini_hw()
4801 	amdgpu_irq_fini_hw(adev);  in amdgpu_device_fini_hw()
4803 	if (adev->mman.initialized)  in amdgpu_device_fini_hw()
4804 		ttm_device_clear_dma_mappings(&adev->mman.bdev);  in amdgpu_device_fini_hw()
4806 	amdgpu_gart_dummy_page_fini(adev);  in amdgpu_device_fini_hw()
4808 	if (drm_dev_is_unplugged(adev_to_drm(adev)))  in amdgpu_device_fini_hw()
4809 		amdgpu_device_unmap_mmio(adev);  in amdgpu_device_fini_hw()
4813 void amdgpu_device_fini_sw(struct amdgpu_device *adev)  in amdgpu_device_fini_sw()  argument
4818 	amdgpu_device_ip_fini(adev);  in amdgpu_device_fini_sw()
4819 	amdgpu_fence_driver_sw_fini(adev);  in amdgpu_device_fini_sw()
4820 	amdgpu_ucode_release(&adev->firmware.gpu_info_fw);  in amdgpu_device_fini_sw()
4821 	adev->accel_working = false;  in amdgpu_device_fini_sw()
4822 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));  in amdgpu_device_fini_sw()
4824 		dma_fence_put(adev->isolation[i].spearhead);  in amdgpu_device_fini_sw()
4825 		amdgpu_sync_free(&adev->isolation[i].active);  in amdgpu_device_fini_sw()
4826 		amdgpu_sync_free(&adev->isolation[i].prev);  in amdgpu_device_fini_sw()
4829 	amdgpu_reset_fini(adev);  in amdgpu_device_fini_sw()
4832 	amdgpu_i2c_fini(adev);  in amdgpu_device_fini_sw()
4834 	if (adev->bios) {  in amdgpu_device_fini_sw()
4836 			amdgpu_atombios_fini(adev);  in amdgpu_device_fini_sw()
4837 		amdgpu_bios_release(adev);  in amdgpu_device_fini_sw()
4840 	kfree(adev->fru_info);  in amdgpu_device_fini_sw()
4841 	adev->fru_info = NULL;  in amdgpu_device_fini_sw()
4843 	kfree(adev->xcp_mgr);  in amdgpu_device_fini_sw()
4844 	adev->xcp_mgr = NULL;  in amdgpu_device_fini_sw()
4846 	px = amdgpu_device_supports_px(adev_to_drm(adev));  in amdgpu_device_fini_sw()
4848 	if (px || (!dev_is_removable(&adev->pdev->dev) &&  in amdgpu_device_fini_sw()
4850 		vga_switcheroo_unregister_client(adev->pdev);  in amdgpu_device_fini_sw()
4853 		vga_switcheroo_fini_domain_pm_ops(adev->dev);  in amdgpu_device_fini_sw()
4855 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)  in amdgpu_device_fini_sw()
4856 		vga_client_unregister(adev->pdev);  in amdgpu_device_fini_sw()
4858 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {  in amdgpu_device_fini_sw()
4860 		iounmap(adev->rmmio);  in amdgpu_device_fini_sw()
4861 		adev->rmmio = NULL;  in amdgpu_device_fini_sw()
4866 		amdgpu_pmu_fini(adev);  in amdgpu_device_fini_sw()
4867 	if (adev->mman.discovery_bin)  in amdgpu_device_fini_sw()
4868 		amdgpu_discovery_fini(adev);  in amdgpu_device_fini_sw()
4870 	amdgpu_reset_put_reset_domain(adev->reset_domain);  in amdgpu_device_fini_sw()
4871 	adev->reset_domain = NULL;  in amdgpu_device_fini_sw()
4873 	kfree(adev->pci_state);  in amdgpu_device_fini_sw()
4886 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)  in amdgpu_device_evict_resources()  argument
4891 	if (!adev->in_s4 && (adev->flags & AMD_IS_APU))  in amdgpu_device_evict_resources()
4894 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);  in amdgpu_device_evict_resources()
4916 	struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb);  in amdgpu_device_pm_notifier()  local
4920 		adev->in_s4 = true;  in amdgpu_device_pm_notifier()
4923 		adev->in_s4 = false;  in amdgpu_device_pm_notifier()
4941 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_prepare()  local
4948 	r = amdgpu_device_evict_resources(adev);  in amdgpu_device_prepare()
4952 	flush_delayed_work(&adev->gfx.gfx_off_delay_work);  in amdgpu_device_prepare()
4954 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_prepare()
4955 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_prepare()
4957 		if (!adev->ip_blocks[i].version->funcs->prepare_suspend)  in amdgpu_device_prepare()
4959 		r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]);  in amdgpu_device_prepare()
4979 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_suspend()  local
4985 	adev->in_suspend = true;  in amdgpu_device_suspend()
4987 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_suspend()
4988 		amdgpu_virt_fini_data_exchange(adev);  in amdgpu_device_suspend()
4989 		r = amdgpu_virt_request_full_gpu(adev, false);  in amdgpu_device_suspend()
4998 		drm_client_dev_suspend(adev_to_drm(adev), false);  in amdgpu_device_suspend()
5000 	cancel_delayed_work_sync(&adev->delayed_init_work);  in amdgpu_device_suspend()
5002 	amdgpu_ras_suspend(adev);  in amdgpu_device_suspend()
5004 	amdgpu_device_ip_suspend_phase1(adev);  in amdgpu_device_suspend()
5006 	if (!adev->in_s0ix)  in amdgpu_device_suspend()
5007 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);  in amdgpu_device_suspend()
5009 	r = amdgpu_device_evict_resources(adev);  in amdgpu_device_suspend()
5013 	amdgpu_ttm_set_buffer_funcs_status(adev, false);  in amdgpu_device_suspend()
5015 	amdgpu_fence_driver_hw_fini(adev);  in amdgpu_device_suspend()
5017 	amdgpu_device_ip_suspend_phase2(adev);  in amdgpu_device_suspend()
5019 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_suspend()
5020 		amdgpu_virt_release_full_gpu(adev, false);  in amdgpu_device_suspend()
5022 	r = amdgpu_dpm_notify_rlc_state(adev, false);  in amdgpu_device_suspend()
5041 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_resume()  local
5044 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_resume()
5045 		r = amdgpu_virt_request_full_gpu(adev, true);  in amdgpu_device_resume()
5053 	if (adev->in_s0ix)  in amdgpu_device_resume()
5054 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);  in amdgpu_device_resume()
5057 	if (amdgpu_device_need_post(adev)) {  in amdgpu_device_resume()
5058 		r = amdgpu_device_asic_init(adev);  in amdgpu_device_resume()
5060 			dev_err(adev->dev, "amdgpu asic init failed\n");  in amdgpu_device_resume()
5063 	r = amdgpu_device_ip_resume(adev);  in amdgpu_device_resume()
5066 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);  in amdgpu_device_resume()
5070 	if (!adev->in_s0ix) {  in amdgpu_device_resume()
5071 		r = amdgpu_amdkfd_resume(adev, adev->in_runpm);  in amdgpu_device_resume()
5076 	r = amdgpu_device_ip_late_init(adev);  in amdgpu_device_resume()
5080 	queue_delayed_work(system_wq, &adev->delayed_init_work,  in amdgpu_device_resume()
5083 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_resume()
5084 		amdgpu_virt_init_data_exchange(adev);  in amdgpu_device_resume()
5085 		amdgpu_virt_release_full_gpu(adev, true);  in amdgpu_device_resume()
5092 	flush_delayed_work(&adev->delayed_init_work);  in amdgpu_device_resume()
5095 		drm_client_dev_resume(adev_to_drm(adev), false);  in amdgpu_device_resume()
5097 	amdgpu_ras_resume(adev);  in amdgpu_device_resume()
5099 	if (adev->mode_info.num_crtc) {  in amdgpu_device_resume()
5112 		if (!adev->dc_enabled)  in amdgpu_device_resume()
5120 	adev->in_suspend = false;  in amdgpu_device_resume()
5122 	if (adev->enable_mes)  in amdgpu_device_resume()
5123 		amdgpu_mes_self_test(adev);  in amdgpu_device_resume()
5141 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_check_soft_reset()  argument
5146 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_ip_check_soft_reset()
5149 	if (amdgpu_asic_need_full_reset(adev))  in amdgpu_device_ip_check_soft_reset()
5152 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_check_soft_reset()
5153 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_check_soft_reset()
5155 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)  in amdgpu_device_ip_check_soft_reset()
5156 			adev->ip_blocks[i].status.hang =  in amdgpu_device_ip_check_soft_reset()
5157 				adev->ip_blocks[i].version->funcs->check_soft_reset(  in amdgpu_device_ip_check_soft_reset()
5158 					&adev->ip_blocks[i]);  in amdgpu_device_ip_check_soft_reset()
5159 		if (adev->ip_blocks[i].status.hang) {  in amdgpu_device_ip_check_soft_reset()
5160 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);  in amdgpu_device_ip_check_soft_reset()
5178 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_pre_soft_reset()  argument
5182 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_pre_soft_reset()
5183 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_pre_soft_reset()
5185 		if (adev->ip_blocks[i].status.hang &&  in amdgpu_device_ip_pre_soft_reset()
5186 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {  in amdgpu_device_ip_pre_soft_reset()
5187 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]);  in amdgpu_device_ip_pre_soft_reset()
5205 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_need_full_reset()  argument
5209 	if (amdgpu_asic_need_full_reset(adev))  in amdgpu_device_ip_need_full_reset()
5212 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_need_full_reset()
5213 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_need_full_reset()
5215 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||  in amdgpu_device_ip_need_full_reset()
5216 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||  in amdgpu_device_ip_need_full_reset()
5217 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||  in amdgpu_device_ip_need_full_reset()
5218 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||  in amdgpu_device_ip_need_full_reset()
5219 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {  in amdgpu_device_ip_need_full_reset()
5220 			if (adev->ip_blocks[i].status.hang) {  in amdgpu_device_ip_need_full_reset()
5221 				dev_info(adev->dev, "Some block need full reset!\n");  in amdgpu_device_ip_need_full_reset()
5240 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_soft_reset()  argument
5244 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_soft_reset()
5245 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_soft_reset()
5247 		if (adev->ip_blocks[i].status.hang &&  in amdgpu_device_ip_soft_reset()
5248 		    adev->ip_blocks[i].version->funcs->soft_reset) {  in amdgpu_device_ip_soft_reset()
5249 			r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]);  in amdgpu_device_ip_soft_reset()
5269 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_post_soft_reset()  argument
5273 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_post_soft_reset()
5274 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_post_soft_reset()
5276 		if (adev->ip_blocks[i].status.hang &&  in amdgpu_device_ip_post_soft_reset()
5277 		    adev->ip_blocks[i].version->funcs->post_soft_reset)  in amdgpu_device_ip_post_soft_reset()
5278 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]);  in amdgpu_device_ip_post_soft_reset()
5295 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,  in amdgpu_device_reset_sriov()  argument
5302 		if (!amdgpu_ras_get_fed_status(adev))  in amdgpu_device_reset_sriov()
5303 			amdgpu_virt_ready_to_reset(adev);  in amdgpu_device_reset_sriov()
5304 		amdgpu_virt_wait_reset(adev);  in amdgpu_device_reset_sriov()
5306 		r = amdgpu_virt_request_full_gpu(adev, true);  in amdgpu_device_reset_sriov()
5308 		r = amdgpu_virt_reset_gpu(adev);  in amdgpu_device_reset_sriov()
5313 	amdgpu_ras_clear_err_state(adev);  in amdgpu_device_reset_sriov()
5314 	amdgpu_irq_gpu_reset_resume_helper(adev);  in amdgpu_device_reset_sriov()
5317 	amdgpu_virt_post_reset(adev);  in amdgpu_device_reset_sriov()
5320 	r = amdgpu_device_ip_reinit_early_sriov(adev);  in amdgpu_device_reset_sriov()
5324 	amdgpu_virt_init_data_exchange(adev);  in amdgpu_device_reset_sriov()
5326 	r = amdgpu_device_fw_loading(adev);  in amdgpu_device_reset_sriov()
5331 	r = amdgpu_device_ip_reinit_late_sriov(adev);  in amdgpu_device_reset_sriov()
5335 	hive = amdgpu_get_xgmi_hive(adev);  in amdgpu_device_reset_sriov()
5337 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)  in amdgpu_device_reset_sriov()
5338 		r = amdgpu_xgmi_update_topology(hive, adev);  in amdgpu_device_reset_sriov()
5344 	r = amdgpu_ib_ring_tests(adev);  in amdgpu_device_reset_sriov()
5348 	if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST)  in amdgpu_device_reset_sriov()
5349 		amdgpu_inc_vram_lost(adev);  in amdgpu_device_reset_sriov()
5354 	amdgpu_amdkfd_post_reset(adev);  in amdgpu_device_reset_sriov()
5355 	amdgpu_virt_release_full_gpu(adev, true);  in amdgpu_device_reset_sriov()
5358 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||  in amdgpu_device_reset_sriov()
5359 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||  in amdgpu_device_reset_sriov()
5360 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||  in amdgpu_device_reset_sriov()
5361 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||  in amdgpu_device_reset_sriov()
5362 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))  in amdgpu_device_reset_sriov()
5363 		amdgpu_ras_resume(adev);  in amdgpu_device_reset_sriov()
5365 	amdgpu_virt_ras_telemetry_post_reset(adev);  in amdgpu_device_reset_sriov()
5380 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)  in amdgpu_device_has_job_running()  argument
5385 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_device_has_job_running()
5404 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)  in amdgpu_device_should_recover_gpu()  argument
5411 	if (!amdgpu_ras_is_poison_mode_supported(adev))  in amdgpu_device_should_recover_gpu()
5414 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_should_recover_gpu()
5418 		switch (adev->asic_type) {  in amdgpu_device_should_recover_gpu()
5443 		dev_info(adev->dev, "GPU recovery disabled.\n");  in amdgpu_device_should_recover_gpu()
5447 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)  in amdgpu_device_mode1_reset()  argument
5452 	if (adev->bios)  in amdgpu_device_mode1_reset()
5453 		amdgpu_atombios_scratch_regs_engine_hung(adev, true);  in amdgpu_device_mode1_reset()
5455 	dev_info(adev->dev, "GPU mode1 reset\n");  in amdgpu_device_mode1_reset()
5460 	amdgpu_device_cache_pci_state(adev->pdev);  in amdgpu_device_mode1_reset()
5463 	pci_clear_master(adev->pdev);  in amdgpu_device_mode1_reset()
5465 	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {  in amdgpu_device_mode1_reset()
5466 		dev_info(adev->dev, "GPU smu mode1 reset\n");  in amdgpu_device_mode1_reset()
5467 		ret = amdgpu_dpm_mode1_reset(adev);  in amdgpu_device_mode1_reset()
5469 		dev_info(adev->dev, "GPU psp mode1 reset\n");  in amdgpu_device_mode1_reset()
5470 		ret = psp_gpu_reset(adev);  in amdgpu_device_mode1_reset()
5476 	amdgpu_device_load_pci_state(adev->pdev);  in amdgpu_device_mode1_reset()
5477 	ret = amdgpu_psp_wait_for_bootloader(adev);  in amdgpu_device_mode1_reset()
5482 	for (i = 0; i < adev->usec_timeout; i++) {  in amdgpu_device_mode1_reset()
5483 		u32 memsize = adev->nbio.funcs->get_memsize(adev);  in amdgpu_device_mode1_reset()
5490 	if (i >= adev->usec_timeout) {  in amdgpu_device_mode1_reset()
5495 	if (adev->bios)  in amdgpu_device_mode1_reset()
5496 		amdgpu_atombios_scratch_regs_engine_hung(adev, false);  in amdgpu_device_mode1_reset()
5501 	dev_err(adev->dev, "GPU mode1 reset failed\n");  in amdgpu_device_mode1_reset()
5505 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,  in amdgpu_device_pre_asic_reset()  argument
5514 	if (reset_context->reset_req_dev == adev)  in amdgpu_device_pre_asic_reset()
5517 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_pre_asic_reset()
5518 		amdgpu_virt_pre_reset(adev);  in amdgpu_device_pre_asic_reset()
5520 	amdgpu_fence_driver_isr_toggle(adev, true);  in amdgpu_device_pre_asic_reset()
5524 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_device_pre_asic_reset()
5538 	amdgpu_fence_driver_isr_toggle(adev, false);  in amdgpu_device_pre_asic_reset()
5543 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);  in amdgpu_device_pre_asic_reset()
5551 	if (!amdgpu_sriov_vf(adev)) {  in amdgpu_device_pre_asic_reset()
5554 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);  in amdgpu_device_pre_asic_reset()
5557 		    amdgpu_device_ip_check_soft_reset(adev)) {  in amdgpu_device_pre_asic_reset()
5558 			amdgpu_device_ip_pre_soft_reset(adev);  in amdgpu_device_pre_asic_reset()
5559 			r = amdgpu_device_ip_soft_reset(adev);  in amdgpu_device_pre_asic_reset()
5560 			amdgpu_device_ip_post_soft_reset(adev);  in amdgpu_device_pre_asic_reset()
5561 			if (r || amdgpu_device_ip_check_soft_reset(adev)) {  in amdgpu_device_pre_asic_reset()
5562 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");  in amdgpu_device_pre_asic_reset()
5578 			r = amdgpu_device_ip_suspend(adev);  in amdgpu_device_pre_asic_reset()
5804 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)  in amdgpu_device_set_mp1_state()  argument
5807 	switch (amdgpu_asic_reset_method(adev)) {  in amdgpu_device_set_mp1_state()
5809 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;  in amdgpu_device_set_mp1_state()
5812 		adev->mp1_state = PP_MP1_STATE_RESET;  in amdgpu_device_set_mp1_state()
5815 		adev->mp1_state = PP_MP1_STATE_NONE;  in amdgpu_device_set_mp1_state()
5820 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)  in amdgpu_device_unset_mp1_state()  argument
5822 	amdgpu_vf_error_trans_all(adev);  in amdgpu_device_unset_mp1_state()
5823 	adev->mp1_state = PP_MP1_STATE_NONE;  in amdgpu_device_unset_mp1_state()
5826 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)  in amdgpu_device_resume_display_audio()  argument
5830 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),  in amdgpu_device_resume_display_audio()
5831 			adev->pdev->bus->number, 1);  in amdgpu_device_resume_display_audio()
5840 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)  in amdgpu_device_suspend_display_audio()  argument
5850 	reset_method = amdgpu_asic_reset_method(adev);  in amdgpu_device_suspend_display_audio()
5855 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),  in amdgpu_device_suspend_display_audio()
5856 			adev->pdev->bus->number, 1);  in amdgpu_device_suspend_display_audio()
5875 			dev_warn(adev->dev, "failed to suspend display audio\n");  in amdgpu_device_suspend_display_audio()
5888 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)  in amdgpu_device_stop_pending_resets()  argument
5890 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);  in amdgpu_device_stop_pending_resets()
5893 	if (!amdgpu_sriov_vf(adev))  in amdgpu_device_stop_pending_resets()
5894 		cancel_work(&adev->reset_work);  in amdgpu_device_stop_pending_resets()
5897 	if (adev->kfd.dev)  in amdgpu_device_stop_pending_resets()
5898 		cancel_work(&adev->kfd.reset_work);  in amdgpu_device_stop_pending_resets()
5900 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_stop_pending_resets()
5901 		cancel_work(&adev->virt.flr_work);  in amdgpu_device_stop_pending_resets()
5903 	if (con && adev->ras_enabled)  in amdgpu_device_stop_pending_resets()
5937 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,  in amdgpu_device_gpu_recover()  argument
5954 	if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) &&  in amdgpu_device_gpu_recover()
5955 	    !amdgpu_sriov_vf(adev) &&  in amdgpu_device_gpu_recover()
5957 		dev_dbg(adev->dev,  in amdgpu_device_gpu_recover()
5965 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);  in amdgpu_device_gpu_recover()
5971 	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&  in amdgpu_device_gpu_recover()
5972 		amdgpu_ras_get_context(adev)->reboot) {  in amdgpu_device_gpu_recover()
5979 	dev_info(adev->dev, "GPU %s begin!\n",  in amdgpu_device_gpu_recover()
5982 	if (!amdgpu_sriov_vf(adev))  in amdgpu_device_gpu_recover()
5983 		hive = amdgpu_get_xgmi_hive(adev);  in amdgpu_device_gpu_recover()
5995 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {  in amdgpu_device_gpu_recover()
5998 			if (adev->shutdown)  in amdgpu_device_gpu_recover()
6001 		if (!list_is_first(&adev->reset_list, &device_list))  in amdgpu_device_gpu_recover()
6002 			list_rotate_to_front(&adev->reset_list, &device_list);  in amdgpu_device_gpu_recover()
6005 		list_add_tail(&adev->reset_list, &device_list);  in amdgpu_device_gpu_recover()
6009 	if (!amdgpu_sriov_vf(adev)) {  in amdgpu_device_gpu_recover()
6082 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");  in amdgpu_device_gpu_recover()
6099 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_device_gpu_recover()
6100 		if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {  in amdgpu_device_gpu_recover()
6101 			dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");  in amdgpu_device_gpu_recover()
6102 			amdgpu_ras_set_fed(adev, true);  in amdgpu_device_gpu_recover()
6106 		r = amdgpu_device_reset_sriov(adev, reset_context);  in amdgpu_device_gpu_recover()
6108 			amdgpu_virt_release_full_gpu(adev, true);  in amdgpu_device_gpu_recover()
6112 			adev->asic_reset_res = r;  in amdgpu_device_gpu_recover()
6177 		if (!adev->kfd.init_complete)  in amdgpu_device_gpu_recover()
6178 			amdgpu_amdkfd_device_init(adev);  in amdgpu_device_gpu_recover()
6199 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);  in amdgpu_device_gpu_recover()
6201 	atomic_set(&adev->reset_domain->reset_res, r);  in amdgpu_device_gpu_recover()
6204 		drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE);  in amdgpu_device_gpu_recover()
6220 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,  in amdgpu_device_partner_bandwidth()  argument
6224 	struct pci_dev *parent = adev->pdev;  in amdgpu_device_partner_bandwidth()
6232 	if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {  in amdgpu_device_partner_bandwidth()
6243 		pcie_bandwidth_available(adev->pdev, NULL, speed, width);  in amdgpu_device_partner_bandwidth()
6257 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev,  in amdgpu_device_gpu_bandwidth()  argument
6261 	struct pci_dev *parent = adev->pdev;  in amdgpu_device_gpu_bandwidth()
6280 		*speed = pcie_get_speed_cap(adev->pdev);  in amdgpu_device_gpu_bandwidth()
6281 		*width = pcie_get_width_cap(adev->pdev);  in amdgpu_device_gpu_bandwidth()
6294 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)  in amdgpu_device_get_pcie_info()  argument
6300 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;  in amdgpu_device_get_pcie_info()
6303 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;  in amdgpu_device_get_pcie_info()
6306 	if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {  in amdgpu_device_get_pcie_info()
6307 		if (adev->pm.pcie_gen_mask == 0)  in amdgpu_device_get_pcie_info()
6308 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;  in amdgpu_device_get_pcie_info()
6309 		if (adev->pm.pcie_mlw_mask == 0)  in amdgpu_device_get_pcie_info()
6310 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;  in amdgpu_device_get_pcie_info()
6314 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)  in amdgpu_device_get_pcie_info()
6317 	amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,  in amdgpu_device_get_pcie_info()
6319 	amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width);  in amdgpu_device_get_pcie_info()
6321 	if (adev->pm.pcie_gen_mask == 0) {  in amdgpu_device_get_pcie_info()
6324 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6329 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6335 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6340 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6344 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6347 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;  in amdgpu_device_get_pcie_info()
6351 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6355 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6361 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6366 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6370 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
6373 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;  in amdgpu_device_get_pcie_info()
6377 	if (adev->pm.pcie_mlw_mask == 0) {  in amdgpu_device_get_pcie_info()
6380 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK;  in amdgpu_device_get_pcie_info()
6384 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 |  in amdgpu_device_get_pcie_info()
6393 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |  in amdgpu_device_get_pcie_info()
6401 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |  in amdgpu_device_get_pcie_info()
6408 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |  in amdgpu_device_get_pcie_info()
6414 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |  in amdgpu_device_get_pcie_info()
6419 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |  in amdgpu_device_get_pcie_info()
6423 				adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1;  in amdgpu_device_get_pcie_info()
6431 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;  in amdgpu_device_get_pcie_info()
6435 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |  in amdgpu_device_get_pcie_info()
6444 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |  in amdgpu_device_get_pcie_info()
6452 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |  in amdgpu_device_get_pcie_info()
6459 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |  in amdgpu_device_get_pcie_info()
6465 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |  in amdgpu_device_get_pcie_info()
6470 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |  in amdgpu_device_get_pcie_info()
6474 				adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;  in amdgpu_device_get_pcie_info()
6493 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,  in amdgpu_device_is_peer_accessible()  argument
6498 		!adev->gmc.xgmi.connected_to_cpu &&  in amdgpu_device_is_peer_accessible()
6499 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);  in amdgpu_device_is_peer_accessible()
6501 		dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n",  in amdgpu_device_is_peer_accessible()
6504 	bool is_large_bar = adev->gmc.visible_vram_size &&  in amdgpu_device_is_peer_accessible()
6505 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size;  in amdgpu_device_is_peer_accessible()
6512 			adev->gmc.aper_base + adev->gmc.aper_size - 1;  in amdgpu_device_is_peer_accessible()
6514 		p2p_addressable = !(adev->gmc.aper_base & address_mask ||  in amdgpu_device_is_peer_accessible()
6525 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_baco_enter()  local
6526 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);  in amdgpu_device_baco_enter()
6531 	if (ras && adev->ras_enabled &&  in amdgpu_device_baco_enter()
6532 	    adev->nbio.funcs->enable_doorbell_interrupt)  in amdgpu_device_baco_enter()
6533 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);  in amdgpu_device_baco_enter()
6535 	return amdgpu_dpm_baco_enter(adev);  in amdgpu_device_baco_enter()
6540 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_baco_exit()  local
6541 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);  in amdgpu_device_baco_exit()
6547 	ret = amdgpu_dpm_baco_exit(adev);  in amdgpu_device_baco_exit()
6551 	if (ras && adev->ras_enabled &&  in amdgpu_device_baco_exit()
6552 	    adev->nbio.funcs->enable_doorbell_interrupt)  in amdgpu_device_baco_exit()
6553 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);  in amdgpu_device_baco_exit()
6555 	if (amdgpu_passthrough(adev) && adev->nbio.funcs &&  in amdgpu_device_baco_exit()
6556 	    adev->nbio.funcs->clear_doorbell_interrupt)  in amdgpu_device_baco_exit()
6557 		adev->nbio.funcs->clear_doorbell_interrupt(adev);  in amdgpu_device_baco_exit()
6574 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_error_detected()  local
6579 	if (adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_pci_error_detected()
6584 	adev->pci_channel_state = state;  in amdgpu_pci_error_detected()
6595 		amdgpu_device_lock_reset_domain(adev->reset_domain);  in amdgpu_pci_error_detected()
6596 		amdgpu_device_set_mp1_state(adev);  in amdgpu_pci_error_detected()
6603 			struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_pci_error_detected()
6610 		atomic_inc(&adev->gpu_reset_counter);  in amdgpu_pci_error_detected()
6650 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_slot_reset()  local
6657 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||  in amdgpu_pci_slot_reset()
6658 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) &&  in amdgpu_pci_slot_reset()
6659 	    amdgpu_ras_in_recovery(adev))  in amdgpu_pci_slot_reset()
6667 	list_add_tail(&adev->reset_list, &device_list);  in amdgpu_pci_slot_reset()
6676 	for (i = 0; i < adev->usec_timeout; i++) {  in amdgpu_pci_slot_reset()
6677 		memsize = amdgpu_asic_get_config_memsize(adev);  in amdgpu_pci_slot_reset()
6689 	reset_context.reset_req_dev = adev;  in amdgpu_pci_slot_reset()
6693 	adev->no_hw_access = true;  in amdgpu_pci_slot_reset()
6694 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);  in amdgpu_pci_slot_reset()
6695 	adev->no_hw_access = false;  in amdgpu_pci_slot_reset()
6703 		if (amdgpu_device_cache_pci_state(adev->pdev))  in amdgpu_pci_slot_reset()
6704 			pci_restore_state(adev->pdev);  in amdgpu_pci_slot_reset()
6709 		amdgpu_device_unset_mp1_state(adev);  in amdgpu_pci_slot_reset()
6710 		amdgpu_device_unlock_reset_domain(adev->reset_domain);  in amdgpu_pci_slot_reset()
6726 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_resume()  local
6733 	if (adev->pci_channel_state != pci_channel_io_frozen)  in amdgpu_pci_resume()
6737 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_pci_resume()
6745 	amdgpu_device_unset_mp1_state(adev);  in amdgpu_pci_resume()
6746 	amdgpu_device_unlock_reset_domain(adev->reset_domain);  in amdgpu_pci_resume()
6752 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_cache_pci_state()  local
6755 	if (amdgpu_sriov_vf(adev))  in amdgpu_device_cache_pci_state()
6760 		kfree(adev->pci_state);  in amdgpu_device_cache_pci_state()
6762 		adev->pci_state = pci_store_saved_state(pdev);  in amdgpu_device_cache_pci_state()
6764 		if (!adev->pci_state) {  in amdgpu_device_cache_pci_state()
6779 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_load_pci_state()  local
6782 	if (!adev->pci_state)  in amdgpu_device_load_pci_state()
6785 	r = pci_load_saved_state(pdev, adev->pci_state);  in amdgpu_device_load_pci_state()
6797 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,  in amdgpu_device_flush_hdp()  argument
6801 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))  in amdgpu_device_flush_hdp()
6804 	if (adev->gmc.xgmi.connected_to_cpu)  in amdgpu_device_flush_hdp()
6810 		amdgpu_asic_flush_hdp(adev, ring);  in amdgpu_device_flush_hdp()
6813 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,  in amdgpu_device_invalidate_hdp()  argument
6817 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))  in amdgpu_device_invalidate_hdp()
6820 	if (adev->gmc.xgmi.connected_to_cpu)  in amdgpu_device_invalidate_hdp()
6823 	amdgpu_asic_invalidate_hdp(adev, ring);  in amdgpu_device_invalidate_hdp()
6826 int amdgpu_in_reset(struct amdgpu_device *adev)  in amdgpu_in_reset()  argument
6828 	return atomic_read(&adev->reset_domain->in_gpu_reset);  in amdgpu_in_reset()
6851 void amdgpu_device_halt(struct amdgpu_device *adev)  in amdgpu_device_halt()  argument
6853 	struct pci_dev *pdev = adev->pdev;  in amdgpu_device_halt()
6854 	struct drm_device *ddev = adev_to_drm(adev);  in amdgpu_device_halt()
6856 	amdgpu_xcp_dev_unplug(adev);  in amdgpu_device_halt()
6859 	amdgpu_irq_disable_all(adev);  in amdgpu_device_halt()
6861 	amdgpu_fence_driver_hw_fini(adev);  in amdgpu_device_halt()
6863 	adev->no_hw_access = true;  in amdgpu_device_halt()
6865 	amdgpu_device_unmap_mmio(adev);  in amdgpu_device_halt()
6871 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,  in amdgpu_device_pcie_port_rreg()  argument
6877 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);  in amdgpu_device_pcie_port_rreg()
6878 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);  in amdgpu_device_pcie_port_rreg()
6880 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_pcie_port_rreg()
6884 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_pcie_port_rreg()
6888 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,  in amdgpu_device_pcie_port_wreg()  argument
6893 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);  in amdgpu_device_pcie_port_wreg()
6894 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);  in amdgpu_device_pcie_port_wreg()
6896 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_pcie_port_wreg()
6901 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_pcie_port_wreg()
6910 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)  in amdgpu_device_get_gang()  argument
6915 	fence = dma_fence_get_rcu_safe(&adev->gang_submit);  in amdgpu_device_get_gang()
6929 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,  in amdgpu_device_switch_gang()  argument
6937 		old = amdgpu_device_get_gang(adev);  in amdgpu_device_switch_gang()
6946 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,  in amdgpu_device_switch_gang()
6968 struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,  in amdgpu_device_enforce_isolation()  argument
6972 	struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];  in amdgpu_device_enforce_isolation()
6993 	mutex_lock(&adev->enforce_isolation_mutex);  in amdgpu_device_enforce_isolation()
7014 			dep = amdgpu_device_get_gang(adev);  in amdgpu_device_enforce_isolation()
7041 	mutex_unlock(&adev->enforce_isolation_mutex);  in amdgpu_device_enforce_isolation()
7045 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)  in amdgpu_device_has_display_hardware()  argument
7047 	switch (adev->asic_type) {  in amdgpu_device_has_display_hardware()
7079 		if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||  in amdgpu_device_has_display_hardware()
7080 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))  in amdgpu_device_has_display_hardware()
7086 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,  in amdgpu_device_wait_on_rreg()  argument
7093 	uint32_t loop = adev->usec_timeout;  in amdgpu_device_wait_on_rreg()
7097 			loop = adev->usec_timeout;  in amdgpu_device_wait_on_rreg()
7118 	if (!ring || !ring->adev)  in amdgpu_get_soft_full_reset_mask()
7121 	if (amdgpu_device_should_recover_gpu(ring->adev))  in amdgpu_get_soft_full_reset_mask()
7124 	if (unlikely(!ring->adev->debug_disable_soft_recovery) &&  in amdgpu_get_soft_full_reset_mask()
7125 	    !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)  in amdgpu_get_soft_full_reset_mask()