1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "amdgpu_ring_mux.h" 51 #include "gfx_v9_4.h" 52 #include "gfx_v9_0.h" 53 #include "gfx_v9_0_cleaner_shader.h" 54 #include "gfx_v9_4_2.h" 55 56 #include "asic_reg/pwr/pwr_10_0_offset.h" 57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 58 #include "asic_reg/gc/gc_9_0_default.h" 59 60 #define GFX9_NUM_GFX_RINGS 1 61 #define GFX9_NUM_SW_GFX_RINGS 2 62 #define GFX9_MEC_HPD_SIZE 4096 63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 65 66 #define mmGCEA_PROBE_MAP 0x070c 67 #define mmGCEA_PROBE_MAP_BASE_IDX 0 68 69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 75 76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 82 83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 89 90 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 93 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 96 97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 104 105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 115 116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 118 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 121 122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 128 129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin"); 133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin"); 134 135 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 137 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 139 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 141 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 143 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 145 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 147 148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025 149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1 150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026 151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1 152 153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = { 154 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS), 155 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2), 156 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1), 157 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2), 158 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1), 159 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1), 160 SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT), 161 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT), 162 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT), 163 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS), 164 SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR), 165 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE), 166 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR), 167 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR), 168 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE), 169 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR), 170 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR), 171 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE), 172 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR), 173 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR), 174 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE), 175 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR), 176 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR), 177 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ), 178 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ), 179 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ), 180 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ), 181 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO), 182 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI), 183 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ), 184 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO), 185 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI), 186 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ), 187 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO), 188 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI), 189 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ), 190 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO), 191 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI), 192 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ), 193 SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS), 194 SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS), 195 SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS), 196 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT), 197 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT), 198 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS), 199 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL), 200 SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS), 201 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS), 202 SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS), 203 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS), 204 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS), 205 SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS), 206 SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS), 207 SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS), 208 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL), 209 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS), 210 SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG), 211 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL), 212 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR), 213 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR), 214 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR), 215 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR), 216 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR), 217 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS), 218 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT), 219 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND), 220 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE), 221 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1), 222 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2), 223 SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE), 224 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE), 225 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE), 226 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT), 227 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6), 228 /* cp header registers */ 229 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 230 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 231 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME2_HEADER_DUMP), 232 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 233 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 234 /* SE status registers */ 235 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0), 236 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1), 237 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2), 238 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3) 239 }; 240 241 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = { 242 /* compute queue registers */ 243 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID), 244 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE), 245 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE), 246 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY), 247 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY), 248 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM), 249 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE), 250 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI), 251 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR), 252 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), 253 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), 254 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), 255 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL), 256 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR), 257 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI), 258 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR), 259 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL), 260 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 261 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR), 262 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI), 263 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL), 264 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR), 265 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR), 266 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS), 267 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO), 268 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI), 269 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL), 270 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET), 271 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE), 272 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET), 273 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE), 274 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE), 275 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR), 276 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM), 277 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO), 278 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI), 279 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS), 280 }; 281 282 enum ta_ras_gfx_subblock { 283 /*CPC*/ 284 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 285 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 286 TA_RAS_BLOCK__GFX_CPC_UCODE, 287 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 288 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 289 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 290 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 291 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 292 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 293 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 294 /* CPF*/ 295 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 296 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 297 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 298 TA_RAS_BLOCK__GFX_CPF_TAG, 299 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 300 /* CPG*/ 301 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 302 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 303 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 304 TA_RAS_BLOCK__GFX_CPG_TAG, 305 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 306 /* GDS*/ 307 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 308 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 309 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 310 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 311 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 312 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 313 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 314 /* SPI*/ 315 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 316 /* SQ*/ 317 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 318 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 319 TA_RAS_BLOCK__GFX_SQ_LDS_D, 320 TA_RAS_BLOCK__GFX_SQ_LDS_I, 321 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 322 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 323 /* SQC (3 ranges)*/ 324 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 325 /* SQC range 0*/ 326 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 327 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 328 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 329 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 330 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 331 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 332 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 333 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 334 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 335 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 336 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 337 /* SQC range 1*/ 338 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 339 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 340 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 341 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 342 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 343 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 344 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 345 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 346 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 347 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 348 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 349 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 350 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 351 /* SQC range 2*/ 352 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 353 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 354 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 355 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 356 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 357 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 358 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 359 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 360 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 361 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 362 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 363 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 364 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 365 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 366 /* TA*/ 367 TA_RAS_BLOCK__GFX_TA_INDEX_START, 368 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 369 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 370 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 371 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 372 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 373 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 374 /* TCA*/ 375 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 376 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 377 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 378 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 379 /* TCC (5 sub-ranges)*/ 380 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 381 /* TCC range 0*/ 382 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 383 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 384 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 385 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 386 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 387 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 388 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 389 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 390 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 391 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 392 /* TCC range 1*/ 393 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 394 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 395 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 396 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 397 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 398 /* TCC range 2*/ 399 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 400 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 401 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 402 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 403 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 404 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 405 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 406 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 407 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 408 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 409 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 410 /* TCC range 3*/ 411 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 412 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 413 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 414 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 415 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 416 /* TCC range 4*/ 417 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 418 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 419 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 420 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 421 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 422 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 423 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 424 /* TCI*/ 425 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 426 /* TCP*/ 427 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 428 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 429 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 430 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 431 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 432 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 433 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 434 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 435 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 436 /* TD*/ 437 TA_RAS_BLOCK__GFX_TD_INDEX_START, 438 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 439 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 440 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 441 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 442 /* EA (3 sub-ranges)*/ 443 TA_RAS_BLOCK__GFX_EA_INDEX_START, 444 /* EA range 0*/ 445 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 446 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 447 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 448 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 449 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 450 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 451 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 452 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 453 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 454 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 455 /* EA range 1*/ 456 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 457 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 458 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 459 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 460 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 461 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 462 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 463 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 464 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 465 /* EA range 2*/ 466 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 467 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 468 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 469 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 470 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 471 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 472 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 473 /* UTC VM L2 bank*/ 474 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 475 /* UTC VM walker*/ 476 TA_RAS_BLOCK__UTC_VML2_WALKER, 477 /* UTC ATC L2 2MB cache*/ 478 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 479 /* UTC ATC L2 4KB cache*/ 480 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 481 TA_RAS_BLOCK__GFX_MAX 482 }; 483 484 struct ras_gfx_subblock { 485 unsigned char *name; 486 int ta_subblock; 487 int hw_supported_error_type; 488 int sw_supported_error_type; 489 }; 490 491 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 492 [AMDGPU_RAS_BLOCK__##subblock] = { \ 493 #subblock, \ 494 TA_RAS_BLOCK__##subblock, \ 495 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 496 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 497 } 498 499 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 500 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 501 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 502 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 508 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 509 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 510 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 511 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 512 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 513 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 514 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 515 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 516 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 517 0), 518 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 519 0), 520 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 521 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 522 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 523 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 524 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 525 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 526 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 527 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 528 0, 0), 529 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 530 0), 531 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 532 0, 0), 533 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 534 0), 535 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 536 0, 0), 537 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 538 0), 539 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 540 1), 541 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 542 0, 0, 0), 543 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 544 0), 545 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 546 0), 547 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 548 0), 549 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 550 0), 551 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 552 0), 553 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 554 0, 0), 555 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 556 0), 557 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 558 0), 559 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 560 0, 0, 0), 561 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 562 0), 563 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 564 0), 565 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 566 0), 567 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 568 0), 569 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 570 0), 571 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 572 0, 0), 573 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 574 0), 575 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 576 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 577 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 578 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 579 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 580 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 581 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 582 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 583 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 584 1), 585 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 586 1), 587 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 588 1), 589 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 590 0), 591 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 592 0), 593 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 594 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 595 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 596 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 597 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 598 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 599 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 600 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 601 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 602 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 603 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 604 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 605 0), 606 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 607 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 608 0), 609 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 610 0, 0), 611 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 612 0), 613 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 614 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 615 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 616 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 617 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 618 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 619 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 620 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 621 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 622 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 623 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 624 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 625 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 626 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 627 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 628 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 629 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 630 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 631 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 632 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 633 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 634 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 635 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 636 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 637 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 638 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 639 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 640 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 641 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 642 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 643 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 644 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 645 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 646 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 647 }; 648 649 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 650 { 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 671 }; 672 673 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 674 { 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 693 }; 694 695 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 696 { 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 708 }; 709 710 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 711 { 712 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 713 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 714 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 715 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 716 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 717 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 718 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 719 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 720 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 721 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 722 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 723 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 724 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 725 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 726 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 727 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 728 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 729 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 730 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 731 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 732 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 733 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 734 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 735 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 736 }; 737 738 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 739 { 740 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 741 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 742 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 743 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 744 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 745 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 746 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 747 }; 748 749 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 750 { 751 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 752 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 753 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 754 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 755 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 756 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 757 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 758 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 759 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 760 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 761 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 762 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 763 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 764 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 765 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 766 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 767 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 768 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 769 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 770 }; 771 772 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 773 { 774 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 775 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 776 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 777 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 778 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 779 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 780 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 781 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 782 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 783 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 784 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 785 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 786 }; 787 788 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 789 { 790 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 791 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 792 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 793 }; 794 795 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 796 { 797 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 798 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 799 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 800 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 801 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 802 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 803 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 804 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 805 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 806 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 807 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 808 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 809 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 810 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 811 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 812 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 813 }; 814 815 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 816 { 817 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 818 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 819 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 820 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 821 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 822 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 823 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 824 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 825 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 826 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 827 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 828 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 829 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 830 }; 831 832 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 833 { 834 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 835 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 836 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 837 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 838 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 839 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 840 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 841 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 842 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 843 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 844 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 845 }; 846 847 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 848 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 849 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 850 }; 851 852 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 853 { 854 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 855 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 856 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 857 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 858 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 859 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 860 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 861 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 862 }; 863 864 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 865 { 866 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 867 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 868 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 869 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 870 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 871 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 872 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 873 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 874 }; 875 876 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 877 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 878 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 879 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 880 881 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 882 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 883 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 884 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 885 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 886 struct amdgpu_cu_info *cu_info); 887 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 888 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds); 889 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 890 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 891 void *ras_error_status); 892 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 893 void *inject_if, uint32_t instance_mask); 894 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 895 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, 896 unsigned int vmid); 897 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 898 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 899 900 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 901 uint64_t queue_mask) 902 { 903 struct amdgpu_device *adev = kiq_ring->adev; 904 u64 shader_mc_addr; 905 906 /* Cleaner shader MC address */ 907 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 908 909 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 910 amdgpu_ring_write(kiq_ring, 911 PACKET3_SET_RESOURCES_VMID_MASK(0) | 912 /* vmid_mask:0* queue_type:0 (KIQ) */ 913 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 914 amdgpu_ring_write(kiq_ring, 915 lower_32_bits(queue_mask)); /* queue mask lo */ 916 amdgpu_ring_write(kiq_ring, 917 upper_32_bits(queue_mask)); /* queue mask hi */ 918 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 919 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 920 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 921 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 922 } 923 924 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 925 struct amdgpu_ring *ring) 926 { 927 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 928 uint64_t wptr_addr = ring->wptr_gpu_addr; 929 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 930 931 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 932 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 933 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 934 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 935 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 936 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 937 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 938 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 939 /*queue_type: normal compute queue */ 940 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 941 /* alloc format: all_on_one_pipe */ 942 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 943 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 944 /* num_queues: must be 1 */ 945 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 946 amdgpu_ring_write(kiq_ring, 947 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 948 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 949 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 950 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 951 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 952 } 953 954 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 955 struct amdgpu_ring *ring, 956 enum amdgpu_unmap_queues_action action, 957 u64 gpu_addr, u64 seq) 958 { 959 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 960 961 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 962 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 963 PACKET3_UNMAP_QUEUES_ACTION(action) | 964 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 965 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 966 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 967 amdgpu_ring_write(kiq_ring, 968 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 969 970 if (action == PREEMPT_QUEUES_NO_UNMAP) { 971 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask)); 972 amdgpu_ring_write(kiq_ring, 0); 973 amdgpu_ring_write(kiq_ring, 0); 974 975 } else { 976 amdgpu_ring_write(kiq_ring, 0); 977 amdgpu_ring_write(kiq_ring, 0); 978 amdgpu_ring_write(kiq_ring, 0); 979 } 980 } 981 982 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 983 struct amdgpu_ring *ring, 984 u64 addr, 985 u64 seq) 986 { 987 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 988 989 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 990 amdgpu_ring_write(kiq_ring, 991 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 992 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 993 PACKET3_QUERY_STATUS_COMMAND(2)); 994 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 995 amdgpu_ring_write(kiq_ring, 996 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 997 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 998 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 999 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 1000 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 1001 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 1002 } 1003 1004 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 1005 uint16_t pasid, uint32_t flush_type, 1006 bool all_hub) 1007 { 1008 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 1009 amdgpu_ring_write(kiq_ring, 1010 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 1011 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 1012 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 1013 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 1014 } 1015 1016 1017 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type, 1018 uint32_t me_id, uint32_t pipe_id, uint32_t queue_id, 1019 uint32_t xcc_id, uint32_t vmid) 1020 { 1021 struct amdgpu_device *adev = kiq_ring->adev; 1022 unsigned i; 1023 1024 /* enter save mode */ 1025 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id); 1026 mutex_lock(&adev->srbm_mutex); 1027 soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0); 1028 1029 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) { 1030 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2); 1031 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1); 1032 /* wait till dequeue take effects */ 1033 for (i = 0; i < adev->usec_timeout; i++) { 1034 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 1035 break; 1036 udelay(1); 1037 } 1038 if (i >= adev->usec_timeout) 1039 dev_err(adev->dev, "fail to wait on hqd deactive\n"); 1040 } else { 1041 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type); 1042 } 1043 1044 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 1045 mutex_unlock(&adev->srbm_mutex); 1046 /* exit safe mode */ 1047 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id); 1048 } 1049 1050 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 1051 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 1052 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 1053 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 1054 .kiq_query_status = gfx_v9_0_kiq_query_status, 1055 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 1056 .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue, 1057 .set_resources_size = 8, 1058 .map_queues_size = 7, 1059 .unmap_queues_size = 6, 1060 .query_status_size = 7, 1061 .invalidate_tlbs_size = 2, 1062 }; 1063 1064 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 1065 { 1066 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs; 1067 } 1068 1069 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 1070 { 1071 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1072 case IP_VERSION(9, 0, 1): 1073 soc15_program_register_sequence(adev, 1074 golden_settings_gc_9_0, 1075 ARRAY_SIZE(golden_settings_gc_9_0)); 1076 soc15_program_register_sequence(adev, 1077 golden_settings_gc_9_0_vg10, 1078 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 1079 break; 1080 case IP_VERSION(9, 2, 1): 1081 soc15_program_register_sequence(adev, 1082 golden_settings_gc_9_2_1, 1083 ARRAY_SIZE(golden_settings_gc_9_2_1)); 1084 soc15_program_register_sequence(adev, 1085 golden_settings_gc_9_2_1_vg12, 1086 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 1087 break; 1088 case IP_VERSION(9, 4, 0): 1089 soc15_program_register_sequence(adev, 1090 golden_settings_gc_9_0, 1091 ARRAY_SIZE(golden_settings_gc_9_0)); 1092 soc15_program_register_sequence(adev, 1093 golden_settings_gc_9_0_vg20, 1094 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 1095 break; 1096 case IP_VERSION(9, 4, 1): 1097 soc15_program_register_sequence(adev, 1098 golden_settings_gc_9_4_1_arct, 1099 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 1100 break; 1101 case IP_VERSION(9, 2, 2): 1102 case IP_VERSION(9, 1, 0): 1103 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 1104 ARRAY_SIZE(golden_settings_gc_9_1)); 1105 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1106 soc15_program_register_sequence(adev, 1107 golden_settings_gc_9_1_rv2, 1108 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 1109 else 1110 soc15_program_register_sequence(adev, 1111 golden_settings_gc_9_1_rv1, 1112 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 1113 break; 1114 case IP_VERSION(9, 3, 0): 1115 soc15_program_register_sequence(adev, 1116 golden_settings_gc_9_1_rn, 1117 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 1118 return; /* for renoir, don't need common goldensetting */ 1119 case IP_VERSION(9, 4, 2): 1120 gfx_v9_4_2_init_golden_registers(adev, 1121 adev->smuio.funcs->get_die_id(adev)); 1122 break; 1123 default: 1124 break; 1125 } 1126 1127 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && 1128 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))) 1129 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 1130 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 1131 } 1132 1133 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1134 bool wc, uint32_t reg, uint32_t val) 1135 { 1136 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1137 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1138 WRITE_DATA_DST_SEL(0) | 1139 (wc ? WR_CONFIRM : 0)); 1140 amdgpu_ring_write(ring, reg); 1141 amdgpu_ring_write(ring, 0); 1142 amdgpu_ring_write(ring, val); 1143 } 1144 1145 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1146 int mem_space, int opt, uint32_t addr0, 1147 uint32_t addr1, uint32_t ref, uint32_t mask, 1148 uint32_t inv) 1149 { 1150 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1151 amdgpu_ring_write(ring, 1152 /* memory (1) or register (0) */ 1153 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1154 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1155 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1156 WAIT_REG_MEM_ENGINE(eng_sel))); 1157 1158 if (mem_space) 1159 BUG_ON(addr0 & 0x3); /* Dword align */ 1160 amdgpu_ring_write(ring, addr0); 1161 amdgpu_ring_write(ring, addr1); 1162 amdgpu_ring_write(ring, ref); 1163 amdgpu_ring_write(ring, mask); 1164 amdgpu_ring_write(ring, inv); /* poll interval */ 1165 } 1166 1167 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1168 { 1169 struct amdgpu_device *adev = ring->adev; 1170 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1171 uint32_t tmp = 0; 1172 unsigned i; 1173 int r; 1174 1175 WREG32(scratch, 0xCAFEDEAD); 1176 r = amdgpu_ring_alloc(ring, 3); 1177 if (r) 1178 return r; 1179 1180 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1181 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START); 1182 amdgpu_ring_write(ring, 0xDEADBEEF); 1183 amdgpu_ring_commit(ring); 1184 1185 for (i = 0; i < adev->usec_timeout; i++) { 1186 tmp = RREG32(scratch); 1187 if (tmp == 0xDEADBEEF) 1188 break; 1189 udelay(1); 1190 } 1191 1192 if (i >= adev->usec_timeout) 1193 r = -ETIMEDOUT; 1194 return r; 1195 } 1196 1197 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1198 { 1199 struct amdgpu_device *adev = ring->adev; 1200 struct amdgpu_ib ib; 1201 struct dma_fence *f = NULL; 1202 1203 unsigned index; 1204 uint64_t gpu_addr; 1205 uint32_t tmp; 1206 long r; 1207 1208 r = amdgpu_device_wb_get(adev, &index); 1209 if (r) 1210 return r; 1211 1212 gpu_addr = adev->wb.gpu_addr + (index * 4); 1213 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1214 memset(&ib, 0, sizeof(ib)); 1215 1216 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 1217 if (r) 1218 goto err1; 1219 1220 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1221 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1222 ib.ptr[2] = lower_32_bits(gpu_addr); 1223 ib.ptr[3] = upper_32_bits(gpu_addr); 1224 ib.ptr[4] = 0xDEADBEEF; 1225 ib.length_dw = 5; 1226 1227 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1228 if (r) 1229 goto err2; 1230 1231 r = dma_fence_wait_timeout(f, false, timeout); 1232 if (r == 0) { 1233 r = -ETIMEDOUT; 1234 goto err2; 1235 } else if (r < 0) { 1236 goto err2; 1237 } 1238 1239 tmp = adev->wb.wb[index]; 1240 if (tmp == 0xDEADBEEF) 1241 r = 0; 1242 else 1243 r = -EINVAL; 1244 1245 err2: 1246 amdgpu_ib_free(adev, &ib, NULL); 1247 dma_fence_put(f); 1248 err1: 1249 amdgpu_device_wb_free(adev, index); 1250 return r; 1251 } 1252 1253 1254 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1255 { 1256 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1257 amdgpu_ucode_release(&adev->gfx.me_fw); 1258 amdgpu_ucode_release(&adev->gfx.ce_fw); 1259 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1260 amdgpu_ucode_release(&adev->gfx.mec_fw); 1261 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1262 1263 kfree(adev->gfx.rlc.register_list_format); 1264 } 1265 1266 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1267 { 1268 adev->gfx.me_fw_write_wait = false; 1269 adev->gfx.mec_fw_write_wait = false; 1270 1271 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && 1272 ((adev->gfx.mec_fw_version < 0x000001a5) || 1273 (adev->gfx.mec_feature_version < 46) || 1274 (adev->gfx.pfp_fw_version < 0x000000b7) || 1275 (adev->gfx.pfp_feature_version < 46))) 1276 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1277 1278 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1279 case IP_VERSION(9, 0, 1): 1280 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1281 (adev->gfx.me_feature_version >= 42) && 1282 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1283 (adev->gfx.pfp_feature_version >= 42)) 1284 adev->gfx.me_fw_write_wait = true; 1285 1286 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1287 (adev->gfx.mec_feature_version >= 42)) 1288 adev->gfx.mec_fw_write_wait = true; 1289 break; 1290 case IP_VERSION(9, 2, 1): 1291 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1292 (adev->gfx.me_feature_version >= 44) && 1293 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1294 (adev->gfx.pfp_feature_version >= 44)) 1295 adev->gfx.me_fw_write_wait = true; 1296 1297 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1298 (adev->gfx.mec_feature_version >= 44)) 1299 adev->gfx.mec_fw_write_wait = true; 1300 break; 1301 case IP_VERSION(9, 4, 0): 1302 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1303 (adev->gfx.me_feature_version >= 44) && 1304 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1305 (adev->gfx.pfp_feature_version >= 44)) 1306 adev->gfx.me_fw_write_wait = true; 1307 1308 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1309 (adev->gfx.mec_feature_version >= 44)) 1310 adev->gfx.mec_fw_write_wait = true; 1311 break; 1312 case IP_VERSION(9, 1, 0): 1313 case IP_VERSION(9, 2, 2): 1314 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1315 (adev->gfx.me_feature_version >= 42) && 1316 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1317 (adev->gfx.pfp_feature_version >= 42)) 1318 adev->gfx.me_fw_write_wait = true; 1319 1320 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1321 (adev->gfx.mec_feature_version >= 42)) 1322 adev->gfx.mec_fw_write_wait = true; 1323 break; 1324 default: 1325 adev->gfx.me_fw_write_wait = true; 1326 adev->gfx.mec_fw_write_wait = true; 1327 break; 1328 } 1329 } 1330 1331 struct amdgpu_gfxoff_quirk { 1332 u16 chip_vendor; 1333 u16 chip_device; 1334 u16 subsys_vendor; 1335 u16 subsys_device; 1336 u8 revision; 1337 }; 1338 1339 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1340 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1341 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1342 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1343 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1344 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1345 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1346 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */ 1347 { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 }, 1348 /* https://bbs.openkylin.top/t/topic/171497 */ 1349 { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 }, 1350 /* HP 705G4 DM with R5 2400G */ 1351 { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 }, 1352 { 0, 0, 0, 0, 0 }, 1353 }; 1354 1355 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1356 { 1357 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1358 1359 while (p && p->chip_device != 0) { 1360 if (pdev->vendor == p->chip_vendor && 1361 pdev->device == p->chip_device && 1362 pdev->subsystem_vendor == p->subsys_vendor && 1363 pdev->subsystem_device == p->subsys_device && 1364 pdev->revision == p->revision) { 1365 return true; 1366 } 1367 ++p; 1368 } 1369 return false; 1370 } 1371 1372 static bool is_raven_kicker(struct amdgpu_device *adev) 1373 { 1374 if (adev->pm.fw_version >= 0x41e2b) 1375 return true; 1376 else 1377 return false; 1378 } 1379 1380 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1381 { 1382 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) && 1383 (adev->gfx.me_fw_version >= 0x000000a5) && 1384 (adev->gfx.me_feature_version >= 52)) 1385 return true; 1386 else 1387 return false; 1388 } 1389 1390 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1391 { 1392 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1393 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1394 1395 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1396 case IP_VERSION(9, 0, 1): 1397 case IP_VERSION(9, 2, 1): 1398 case IP_VERSION(9, 4, 0): 1399 break; 1400 case IP_VERSION(9, 2, 2): 1401 case IP_VERSION(9, 1, 0): 1402 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1403 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1404 ((!is_raven_kicker(adev) && 1405 adev->gfx.rlc_fw_version < 531) || 1406 (adev->gfx.rlc_feature_version < 1) || 1407 !adev->gfx.rlc.is_rlc_v2_1)) 1408 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1409 1410 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1411 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1412 AMD_PG_SUPPORT_CP | 1413 AMD_PG_SUPPORT_RLC_SMU_HS; 1414 break; 1415 case IP_VERSION(9, 3, 0): 1416 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1417 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1418 AMD_PG_SUPPORT_CP | 1419 AMD_PG_SUPPORT_RLC_SMU_HS; 1420 break; 1421 default: 1422 break; 1423 } 1424 } 1425 1426 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1427 char *chip_name) 1428 { 1429 int err; 1430 1431 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 1432 "amdgpu/%s_pfp.bin", chip_name); 1433 if (err) 1434 goto out; 1435 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 1436 1437 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1438 "amdgpu/%s_me.bin", chip_name); 1439 if (err) 1440 goto out; 1441 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 1442 1443 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1444 "amdgpu/%s_ce.bin", chip_name); 1445 if (err) 1446 goto out; 1447 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE); 1448 1449 out: 1450 if (err) { 1451 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1452 amdgpu_ucode_release(&adev->gfx.me_fw); 1453 amdgpu_ucode_release(&adev->gfx.ce_fw); 1454 } 1455 return err; 1456 } 1457 1458 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1459 char *chip_name) 1460 { 1461 int err; 1462 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1463 uint16_t version_major; 1464 uint16_t version_minor; 1465 uint32_t smu_version; 1466 1467 /* 1468 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1469 * instead of picasso_rlc.bin. 1470 * Judgment method: 1471 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1472 * or revision >= 0xD8 && revision <= 0xDF 1473 * otherwise is PCO FP5 1474 */ 1475 if (!strcmp(chip_name, "picasso") && 1476 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1477 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1478 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1479 "amdgpu/%s_rlc_am4.bin", chip_name); 1480 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1481 (smu_version >= 0x41e2b)) 1482 /** 1483 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1484 */ 1485 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1486 "amdgpu/%s_kicker_rlc.bin", chip_name); 1487 else 1488 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1489 "amdgpu/%s_rlc.bin", chip_name); 1490 if (err) 1491 goto out; 1492 1493 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1494 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1495 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1496 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 1497 out: 1498 if (err) 1499 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1500 1501 return err; 1502 } 1503 1504 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1505 { 1506 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 1507 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || 1508 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) 1509 return false; 1510 1511 return true; 1512 } 1513 1514 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1515 char *chip_name) 1516 { 1517 int err; 1518 1519 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) 1520 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1521 "amdgpu/%s_sjt_mec.bin", chip_name); 1522 else 1523 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1524 "amdgpu/%s_mec.bin", chip_name); 1525 if (err) 1526 goto out; 1527 1528 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 1529 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 1530 1531 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1532 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) 1533 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1534 "amdgpu/%s_sjt_mec2.bin", chip_name); 1535 else 1536 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1537 "amdgpu/%s_mec2.bin", chip_name); 1538 if (!err) { 1539 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2); 1540 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT); 1541 } else { 1542 err = 0; 1543 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1544 } 1545 } else { 1546 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1547 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1548 } 1549 1550 gfx_v9_0_check_if_need_gfxoff(adev); 1551 gfx_v9_0_check_fw_write_wait(adev); 1552 1553 out: 1554 if (err) 1555 amdgpu_ucode_release(&adev->gfx.mec_fw); 1556 return err; 1557 } 1558 1559 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1560 { 1561 char ucode_prefix[30]; 1562 int r; 1563 1564 DRM_DEBUG("\n"); 1565 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 1566 1567 /* No CPG in Arcturus */ 1568 if (adev->gfx.num_gfx_rings) { 1569 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix); 1570 if (r) 1571 return r; 1572 } 1573 1574 r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix); 1575 if (r) 1576 return r; 1577 1578 r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix); 1579 if (r) 1580 return r; 1581 1582 return r; 1583 } 1584 1585 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1586 { 1587 u32 count = 0; 1588 const struct cs_section_def *sect = NULL; 1589 const struct cs_extent_def *ext = NULL; 1590 1591 /* begin clear state */ 1592 count += 2; 1593 /* context control state */ 1594 count += 3; 1595 1596 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1597 for (ext = sect->section; ext->extent != NULL; ++ext) { 1598 if (sect->id == SECT_CONTEXT) 1599 count += 2 + ext->reg_count; 1600 else 1601 return 0; 1602 } 1603 } 1604 1605 /* end clear state */ 1606 count += 2; 1607 /* clear state */ 1608 count += 2; 1609 1610 return count; 1611 } 1612 1613 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1614 volatile u32 *buffer) 1615 { 1616 u32 count = 0, i; 1617 const struct cs_section_def *sect = NULL; 1618 const struct cs_extent_def *ext = NULL; 1619 1620 if (adev->gfx.rlc.cs_data == NULL) 1621 return; 1622 if (buffer == NULL) 1623 return; 1624 1625 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1626 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1627 1628 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1629 buffer[count++] = cpu_to_le32(0x80000000); 1630 buffer[count++] = cpu_to_le32(0x80000000); 1631 1632 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1633 for (ext = sect->section; ext->extent != NULL; ++ext) { 1634 if (sect->id == SECT_CONTEXT) { 1635 buffer[count++] = 1636 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1637 buffer[count++] = cpu_to_le32(ext->reg_index - 1638 PACKET3_SET_CONTEXT_REG_START); 1639 for (i = 0; i < ext->reg_count; i++) 1640 buffer[count++] = cpu_to_le32(ext->extent[i]); 1641 } else { 1642 return; 1643 } 1644 } 1645 } 1646 1647 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1648 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1649 1650 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1651 buffer[count++] = cpu_to_le32(0); 1652 } 1653 1654 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1655 { 1656 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1657 uint32_t pg_always_on_cu_num = 2; 1658 uint32_t always_on_cu_num; 1659 uint32_t i, j, k; 1660 uint32_t mask, cu_bitmap, counter; 1661 1662 if (adev->flags & AMD_IS_APU) 1663 always_on_cu_num = 4; 1664 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1)) 1665 always_on_cu_num = 8; 1666 else 1667 always_on_cu_num = 12; 1668 1669 mutex_lock(&adev->grbm_idx_mutex); 1670 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1671 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1672 mask = 1; 1673 cu_bitmap = 0; 1674 counter = 0; 1675 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 1676 1677 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1678 if (cu_info->bitmap[0][i][j] & mask) { 1679 if (counter == pg_always_on_cu_num) 1680 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1681 if (counter < always_on_cu_num) 1682 cu_bitmap |= mask; 1683 else 1684 break; 1685 counter++; 1686 } 1687 mask <<= 1; 1688 } 1689 1690 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1691 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1692 } 1693 } 1694 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1695 mutex_unlock(&adev->grbm_idx_mutex); 1696 } 1697 1698 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1699 { 1700 uint32_t data; 1701 1702 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1703 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1704 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1705 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1706 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1707 1708 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1709 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1710 1711 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1712 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1713 1714 mutex_lock(&adev->grbm_idx_mutex); 1715 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1716 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1717 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1718 1719 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1720 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1721 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1722 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1723 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1724 1725 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1726 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1727 data &= 0x0000FFFF; 1728 data |= 0x00C00000; 1729 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1730 1731 /* 1732 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1733 * programmed in gfx_v9_0_init_always_on_cu_mask() 1734 */ 1735 1736 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1737 * but used for RLC_LB_CNTL configuration */ 1738 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1739 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1740 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1741 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1742 mutex_unlock(&adev->grbm_idx_mutex); 1743 1744 gfx_v9_0_init_always_on_cu_mask(adev); 1745 } 1746 1747 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1748 { 1749 uint32_t data; 1750 1751 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1752 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1753 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1754 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1755 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1756 1757 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1758 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1759 1760 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1761 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1762 1763 mutex_lock(&adev->grbm_idx_mutex); 1764 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1765 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1766 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1767 1768 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1769 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1770 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1771 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1772 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1773 1774 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1775 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1776 data &= 0x0000FFFF; 1777 data |= 0x00C00000; 1778 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1779 1780 /* 1781 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1782 * programmed in gfx_v9_0_init_always_on_cu_mask() 1783 */ 1784 1785 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1786 * but used for RLC_LB_CNTL configuration */ 1787 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1788 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1789 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1790 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1791 mutex_unlock(&adev->grbm_idx_mutex); 1792 1793 gfx_v9_0_init_always_on_cu_mask(adev); 1794 } 1795 1796 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1797 { 1798 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1799 } 1800 1801 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1802 { 1803 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1804 return 5; 1805 else 1806 return 4; 1807 } 1808 1809 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 1810 { 1811 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1812 1813 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 1814 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1815 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1); 1816 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2); 1817 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3); 1818 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL); 1819 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX); 1820 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT); 1821 adev->gfx.rlc.rlcg_reg_access_supported = true; 1822 } 1823 1824 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1825 { 1826 const struct cs_section_def *cs_data; 1827 int r; 1828 1829 adev->gfx.rlc.cs_data = gfx9_cs_data; 1830 1831 cs_data = adev->gfx.rlc.cs_data; 1832 1833 if (cs_data) { 1834 /* init clear state block */ 1835 r = amdgpu_gfx_rlc_init_csb(adev); 1836 if (r) 1837 return r; 1838 } 1839 1840 if (adev->flags & AMD_IS_APU) { 1841 /* TODO: double check the cp_table_size for RV */ 1842 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1843 r = amdgpu_gfx_rlc_init_cpt(adev); 1844 if (r) 1845 return r; 1846 } 1847 1848 return 0; 1849 } 1850 1851 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1852 { 1853 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1854 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1855 } 1856 1857 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1858 { 1859 int r; 1860 u32 *hpd; 1861 const __le32 *fw_data; 1862 unsigned fw_size; 1863 u32 *fw; 1864 size_t mec_hpd_size; 1865 1866 const struct gfx_firmware_header_v1_0 *mec_hdr; 1867 1868 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1869 1870 /* take ownership of the relevant compute queues */ 1871 amdgpu_gfx_compute_queue_acquire(adev); 1872 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1873 if (mec_hpd_size) { 1874 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1875 AMDGPU_GEM_DOMAIN_VRAM | 1876 AMDGPU_GEM_DOMAIN_GTT, 1877 &adev->gfx.mec.hpd_eop_obj, 1878 &adev->gfx.mec.hpd_eop_gpu_addr, 1879 (void **)&hpd); 1880 if (r) { 1881 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1882 gfx_v9_0_mec_fini(adev); 1883 return r; 1884 } 1885 1886 memset(hpd, 0, mec_hpd_size); 1887 1888 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1889 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1890 } 1891 1892 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1893 1894 fw_data = (const __le32 *) 1895 (adev->gfx.mec_fw->data + 1896 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1897 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1898 1899 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1900 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1901 &adev->gfx.mec.mec_fw_obj, 1902 &adev->gfx.mec.mec_fw_gpu_addr, 1903 (void **)&fw); 1904 if (r) { 1905 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1906 gfx_v9_0_mec_fini(adev); 1907 return r; 1908 } 1909 1910 memcpy(fw, fw_data, fw_size); 1911 1912 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1913 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1914 1915 return 0; 1916 } 1917 1918 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1919 { 1920 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1921 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1922 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1923 (address << SQ_IND_INDEX__INDEX__SHIFT) | 1924 (SQ_IND_INDEX__FORCE_READ_MASK)); 1925 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1926 } 1927 1928 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 1929 uint32_t wave, uint32_t thread, 1930 uint32_t regno, uint32_t num, uint32_t *out) 1931 { 1932 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1933 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1934 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1935 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1936 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 1937 (SQ_IND_INDEX__FORCE_READ_MASK) | 1938 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1939 while (num--) 1940 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1941 } 1942 1943 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1944 { 1945 /* type 1 wave data */ 1946 dst[(*no_fields)++] = 1; 1947 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 1948 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 1949 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 1950 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 1951 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 1952 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 1953 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 1954 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 1955 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 1956 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 1957 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 1958 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 1959 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 1960 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 1961 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 1962 } 1963 1964 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1965 uint32_t wave, uint32_t start, 1966 uint32_t size, uint32_t *dst) 1967 { 1968 wave_read_regs( 1969 adev, simd, wave, 0, 1970 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 1971 } 1972 1973 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1974 uint32_t wave, uint32_t thread, 1975 uint32_t start, uint32_t size, 1976 uint32_t *dst) 1977 { 1978 wave_read_regs( 1979 adev, simd, wave, thread, 1980 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1981 } 1982 1983 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 1984 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1985 { 1986 soc15_grbm_select(adev, me, pipe, q, vm, 0); 1987 } 1988 1989 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 1990 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 1991 .select_se_sh = &gfx_v9_0_select_se_sh, 1992 .read_wave_data = &gfx_v9_0_read_wave_data, 1993 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 1994 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 1995 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 1996 }; 1997 1998 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = { 1999 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2000 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2001 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2002 }; 2003 2004 static struct amdgpu_gfx_ras gfx_v9_0_ras = { 2005 .ras_block = { 2006 .hw_ops = &gfx_v9_0_ras_ops, 2007 }, 2008 }; 2009 2010 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2011 { 2012 u32 gb_addr_config; 2013 int err; 2014 2015 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2016 case IP_VERSION(9, 0, 1): 2017 adev->gfx.config.max_hw_contexts = 8; 2018 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2019 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2020 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2021 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2022 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2023 break; 2024 case IP_VERSION(9, 2, 1): 2025 adev->gfx.config.max_hw_contexts = 8; 2026 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2027 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2028 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2029 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2030 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2031 DRM_INFO("fix gfx.config for vega12\n"); 2032 break; 2033 case IP_VERSION(9, 4, 0): 2034 adev->gfx.ras = &gfx_v9_0_ras; 2035 adev->gfx.config.max_hw_contexts = 8; 2036 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2037 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2038 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2039 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2040 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2041 gb_addr_config &= ~0xf3e777ff; 2042 gb_addr_config |= 0x22014042; 2043 /* check vbios table if gpu info is not available */ 2044 err = amdgpu_atomfirmware_get_gfx_info(adev); 2045 if (err) 2046 return err; 2047 break; 2048 case IP_VERSION(9, 2, 2): 2049 case IP_VERSION(9, 1, 0): 2050 adev->gfx.config.max_hw_contexts = 8; 2051 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2052 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2053 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2054 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2055 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2056 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2057 else 2058 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2059 break; 2060 case IP_VERSION(9, 4, 1): 2061 adev->gfx.ras = &gfx_v9_4_ras; 2062 adev->gfx.config.max_hw_contexts = 8; 2063 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2064 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2065 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2066 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2067 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2068 gb_addr_config &= ~0xf3e777ff; 2069 gb_addr_config |= 0x22014042; 2070 break; 2071 case IP_VERSION(9, 3, 0): 2072 adev->gfx.config.max_hw_contexts = 8; 2073 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2074 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2075 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2076 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2077 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2078 gb_addr_config &= ~0xf3e777ff; 2079 gb_addr_config |= 0x22010042; 2080 break; 2081 case IP_VERSION(9, 4, 2): 2082 adev->gfx.ras = &gfx_v9_4_2_ras; 2083 adev->gfx.config.max_hw_contexts = 8; 2084 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2085 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2086 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2087 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2088 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2089 gb_addr_config &= ~0xf3e777ff; 2090 gb_addr_config |= 0x22014042; 2091 /* check vbios table if gpu info is not available */ 2092 err = amdgpu_atomfirmware_get_gfx_info(adev); 2093 if (err) 2094 return err; 2095 break; 2096 default: 2097 BUG(); 2098 break; 2099 } 2100 2101 adev->gfx.config.gb_addr_config = gb_addr_config; 2102 2103 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2104 REG_GET_FIELD( 2105 adev->gfx.config.gb_addr_config, 2106 GB_ADDR_CONFIG, 2107 NUM_PIPES); 2108 2109 adev->gfx.config.max_tile_pipes = 2110 adev->gfx.config.gb_addr_config_fields.num_pipes; 2111 2112 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2113 REG_GET_FIELD( 2114 adev->gfx.config.gb_addr_config, 2115 GB_ADDR_CONFIG, 2116 NUM_BANKS); 2117 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2118 REG_GET_FIELD( 2119 adev->gfx.config.gb_addr_config, 2120 GB_ADDR_CONFIG, 2121 MAX_COMPRESSED_FRAGS); 2122 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2123 REG_GET_FIELD( 2124 adev->gfx.config.gb_addr_config, 2125 GB_ADDR_CONFIG, 2126 NUM_RB_PER_SE); 2127 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2128 REG_GET_FIELD( 2129 adev->gfx.config.gb_addr_config, 2130 GB_ADDR_CONFIG, 2131 NUM_SHADER_ENGINES); 2132 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2133 REG_GET_FIELD( 2134 adev->gfx.config.gb_addr_config, 2135 GB_ADDR_CONFIG, 2136 PIPE_INTERLEAVE_SIZE)); 2137 2138 return 0; 2139 } 2140 2141 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2142 int mec, int pipe, int queue) 2143 { 2144 unsigned irq_type; 2145 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2146 unsigned int hw_prio; 2147 2148 ring = &adev->gfx.compute_ring[ring_id]; 2149 2150 /* mec0 is me1 */ 2151 ring->me = mec + 1; 2152 ring->pipe = pipe; 2153 ring->queue = queue; 2154 2155 ring->ring_obj = NULL; 2156 ring->use_doorbell = true; 2157 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2158 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2159 + (ring_id * GFX9_MEC_HPD_SIZE); 2160 ring->vm_hub = AMDGPU_GFXHUB(0); 2161 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2162 2163 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2164 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2165 + ring->pipe; 2166 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2167 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT; 2168 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2169 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2170 hw_prio, NULL); 2171 } 2172 2173 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev) 2174 { 2175 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9); 2176 uint32_t *ptr; 2177 uint32_t inst; 2178 2179 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 2180 if (!ptr) { 2181 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 2182 adev->gfx.ip_dump_core = NULL; 2183 } else { 2184 adev->gfx.ip_dump_core = ptr; 2185 } 2186 2187 /* Allocate memory for compute queue registers for all the instances */ 2188 reg_count = ARRAY_SIZE(gc_cp_reg_list_9); 2189 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 2190 adev->gfx.mec.num_queue_per_pipe; 2191 2192 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 2193 if (!ptr) { 2194 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 2195 adev->gfx.ip_dump_compute_queues = NULL; 2196 } else { 2197 adev->gfx.ip_dump_compute_queues = ptr; 2198 } 2199 } 2200 2201 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block) 2202 { 2203 int i, j, k, r, ring_id; 2204 int xcc_id = 0; 2205 struct amdgpu_ring *ring; 2206 struct amdgpu_device *adev = ip_block->adev; 2207 unsigned int hw_prio; 2208 2209 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2210 case IP_VERSION(9, 0, 1): 2211 case IP_VERSION(9, 2, 1): 2212 case IP_VERSION(9, 4, 0): 2213 case IP_VERSION(9, 2, 2): 2214 case IP_VERSION(9, 1, 0): 2215 case IP_VERSION(9, 4, 1): 2216 case IP_VERSION(9, 3, 0): 2217 case IP_VERSION(9, 4, 2): 2218 adev->gfx.mec.num_mec = 2; 2219 break; 2220 default: 2221 adev->gfx.mec.num_mec = 1; 2222 break; 2223 } 2224 2225 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2226 default: 2227 adev->gfx.enable_cleaner_shader = false; 2228 break; 2229 } 2230 2231 adev->gfx.mec.num_pipe_per_mec = 4; 2232 adev->gfx.mec.num_queue_per_pipe = 8; 2233 2234 /* EOP Event */ 2235 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2236 if (r) 2237 return r; 2238 2239 /* Bad opcode Event */ 2240 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 2241 GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR, 2242 &adev->gfx.bad_op_irq); 2243 if (r) 2244 return r; 2245 2246 /* Privileged reg */ 2247 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2248 &adev->gfx.priv_reg_irq); 2249 if (r) 2250 return r; 2251 2252 /* Privileged inst */ 2253 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2254 &adev->gfx.priv_inst_irq); 2255 if (r) 2256 return r; 2257 2258 /* ECC error */ 2259 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2260 &adev->gfx.cp_ecc_error_irq); 2261 if (r) 2262 return r; 2263 2264 /* FUE error */ 2265 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2266 &adev->gfx.cp_ecc_error_irq); 2267 if (r) 2268 return r; 2269 2270 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2271 2272 if (adev->gfx.rlc.funcs) { 2273 if (adev->gfx.rlc.funcs->init) { 2274 r = adev->gfx.rlc.funcs->init(adev); 2275 if (r) { 2276 dev_err(adev->dev, "Failed to init rlc BOs!\n"); 2277 return r; 2278 } 2279 } 2280 } 2281 2282 r = gfx_v9_0_mec_init(adev); 2283 if (r) { 2284 DRM_ERROR("Failed to init MEC BOs!\n"); 2285 return r; 2286 } 2287 2288 /* set up the gfx ring */ 2289 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2290 ring = &adev->gfx.gfx_ring[i]; 2291 ring->ring_obj = NULL; 2292 if (!i) 2293 sprintf(ring->name, "gfx"); 2294 else 2295 sprintf(ring->name, "gfx_%d", i); 2296 ring->use_doorbell = true; 2297 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2298 2299 /* disable scheduler on the real ring */ 2300 ring->no_scheduler = adev->gfx.mcbp; 2301 ring->vm_hub = AMDGPU_GFXHUB(0); 2302 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2303 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2304 AMDGPU_RING_PRIO_DEFAULT, NULL); 2305 if (r) 2306 return r; 2307 } 2308 2309 /* set up the software rings */ 2310 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { 2311 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) { 2312 ring = &adev->gfx.sw_gfx_ring[i]; 2313 ring->ring_obj = NULL; 2314 sprintf(ring->name, amdgpu_sw_ring_name(i)); 2315 ring->use_doorbell = true; 2316 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2317 ring->is_sw_ring = true; 2318 hw_prio = amdgpu_sw_ring_priority(i); 2319 ring->vm_hub = AMDGPU_GFXHUB(0); 2320 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2321 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio, 2322 NULL); 2323 if (r) 2324 return r; 2325 ring->wptr = 0; 2326 } 2327 2328 /* init the muxer and add software rings */ 2329 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0], 2330 GFX9_NUM_SW_GFX_RINGS); 2331 if (r) { 2332 DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r); 2333 return r; 2334 } 2335 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) { 2336 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer, 2337 &adev->gfx.sw_gfx_ring[i]); 2338 if (r) { 2339 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r); 2340 return r; 2341 } 2342 } 2343 } 2344 2345 /* set up the compute queues - allocate horizontally across pipes */ 2346 ring_id = 0; 2347 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2348 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2349 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2350 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 2351 k, j)) 2352 continue; 2353 2354 r = gfx_v9_0_compute_ring_init(adev, 2355 ring_id, 2356 i, k, j); 2357 if (r) 2358 return r; 2359 2360 ring_id++; 2361 } 2362 } 2363 } 2364 2365 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0); 2366 if (r) { 2367 DRM_ERROR("Failed to init KIQ BOs!\n"); 2368 return r; 2369 } 2370 2371 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 2372 if (r) 2373 return r; 2374 2375 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2376 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0); 2377 if (r) 2378 return r; 2379 2380 adev->gfx.ce_ram_size = 0x8000; 2381 2382 r = gfx_v9_0_gpu_early_init(adev); 2383 if (r) 2384 return r; 2385 2386 if (amdgpu_gfx_ras_sw_init(adev)) { 2387 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 2388 return -EINVAL; 2389 } 2390 2391 gfx_v9_0_alloc_ip_dump(adev); 2392 2393 r = amdgpu_gfx_sysfs_isolation_shader_init(adev); 2394 if (r) 2395 return r; 2396 2397 return 0; 2398 } 2399 2400 2401 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block) 2402 { 2403 int i; 2404 struct amdgpu_device *adev = ip_block->adev; 2405 2406 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { 2407 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 2408 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]); 2409 amdgpu_ring_mux_fini(&adev->gfx.muxer); 2410 } 2411 2412 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2413 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2414 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2415 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2416 2417 amdgpu_gfx_mqd_sw_fini(adev, 0); 2418 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 2419 amdgpu_gfx_kiq_fini(adev, 0); 2420 2421 amdgpu_gfx_cleaner_shader_sw_fini(adev); 2422 2423 gfx_v9_0_mec_fini(adev); 2424 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2425 &adev->gfx.rlc.clear_state_gpu_addr, 2426 (void **)&adev->gfx.rlc.cs_ptr); 2427 if (adev->flags & AMD_IS_APU) { 2428 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2429 &adev->gfx.rlc.cp_table_gpu_addr, 2430 (void **)&adev->gfx.rlc.cp_table_ptr); 2431 } 2432 gfx_v9_0_free_microcode(adev); 2433 2434 amdgpu_gfx_sysfs_isolation_shader_fini(adev); 2435 2436 kfree(adev->gfx.ip_dump_core); 2437 kfree(adev->gfx.ip_dump_compute_queues); 2438 2439 return 0; 2440 } 2441 2442 2443 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2444 { 2445 /* TODO */ 2446 } 2447 2448 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2449 u32 instance, int xcc_id) 2450 { 2451 u32 data; 2452 2453 if (instance == 0xffffffff) 2454 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2455 else 2456 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2457 2458 if (se_num == 0xffffffff) 2459 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2460 else 2461 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2462 2463 if (sh_num == 0xffffffff) 2464 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2465 else 2466 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2467 2468 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2469 } 2470 2471 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2472 { 2473 u32 data, mask; 2474 2475 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2476 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2477 2478 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2479 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2480 2481 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2482 adev->gfx.config.max_sh_per_se); 2483 2484 return (~data) & mask; 2485 } 2486 2487 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2488 { 2489 int i, j; 2490 u32 data; 2491 u32 active_rbs = 0; 2492 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2493 adev->gfx.config.max_sh_per_se; 2494 2495 mutex_lock(&adev->grbm_idx_mutex); 2496 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2497 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2498 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 2499 data = gfx_v9_0_get_rb_active_bitmap(adev); 2500 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2501 rb_bitmap_width_per_sh); 2502 } 2503 } 2504 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 2505 mutex_unlock(&adev->grbm_idx_mutex); 2506 2507 adev->gfx.config.backend_enable_mask = active_rbs; 2508 adev->gfx.config.num_rbs = hweight32(active_rbs); 2509 } 2510 2511 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev, 2512 uint32_t first_vmid, 2513 uint32_t last_vmid) 2514 { 2515 uint32_t data; 2516 uint32_t trap_config_vmid_mask = 0; 2517 int i; 2518 2519 /* Calculate trap config vmid mask */ 2520 for (i = first_vmid; i < last_vmid; i++) 2521 trap_config_vmid_mask |= (1 << i); 2522 2523 data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG, 2524 VMID_SEL, trap_config_vmid_mask); 2525 data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, 2526 TRAP_EN, 1); 2527 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); 2528 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); 2529 2530 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0); 2531 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0); 2532 } 2533 2534 #define DEFAULT_SH_MEM_BASES (0x6000) 2535 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2536 { 2537 int i; 2538 uint32_t sh_mem_config; 2539 uint32_t sh_mem_bases; 2540 2541 /* 2542 * Configure apertures: 2543 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2544 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2545 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2546 */ 2547 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2548 2549 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2550 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2551 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2552 2553 mutex_lock(&adev->srbm_mutex); 2554 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2555 soc15_grbm_select(adev, 0, 0, 0, i, 0); 2556 /* CP and shaders */ 2557 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2558 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2559 } 2560 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 2561 mutex_unlock(&adev->srbm_mutex); 2562 2563 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2564 access. These should be enabled by FW for target VMIDs. */ 2565 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2566 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2567 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2568 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2569 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2570 } 2571 } 2572 2573 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2574 { 2575 int vmid; 2576 2577 /* 2578 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2579 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2580 * the driver can enable them for graphics. VMID0 should maintain 2581 * access so that HWS firmware can save/restore entries. 2582 */ 2583 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2584 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2585 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2586 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2587 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2588 } 2589 } 2590 2591 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2592 { 2593 uint32_t tmp; 2594 2595 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2596 case IP_VERSION(9, 4, 1): 2597 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2598 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT, 2599 !READ_ONCE(adev->barrier_has_auto_waitcnt)); 2600 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2601 break; 2602 default: 2603 break; 2604 } 2605 } 2606 2607 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2608 { 2609 u32 tmp; 2610 int i; 2611 2612 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2613 2614 gfx_v9_0_tiling_mode_table_init(adev); 2615 2616 if (adev->gfx.num_gfx_rings) 2617 gfx_v9_0_setup_rb(adev); 2618 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2619 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2620 2621 /* XXX SH_MEM regs */ 2622 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2623 mutex_lock(&adev->srbm_mutex); 2624 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2625 soc15_grbm_select(adev, 0, 0, 0, i, 0); 2626 /* CP and shaders */ 2627 if (i == 0) { 2628 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2629 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2630 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2631 !!adev->gmc.noretry); 2632 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2633 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2634 } else { 2635 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2636 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2637 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2638 !!adev->gmc.noretry); 2639 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2640 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2641 (adev->gmc.private_aperture_start >> 48)); 2642 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2643 (adev->gmc.shared_aperture_start >> 48)); 2644 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2645 } 2646 } 2647 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 2648 2649 mutex_unlock(&adev->srbm_mutex); 2650 2651 gfx_v9_0_init_compute_vmid(adev); 2652 gfx_v9_0_init_gds_vmid(adev); 2653 gfx_v9_0_init_sq_config(adev); 2654 } 2655 2656 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2657 { 2658 u32 i, j, k; 2659 u32 mask; 2660 2661 mutex_lock(&adev->grbm_idx_mutex); 2662 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2663 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2664 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 2665 for (k = 0; k < adev->usec_timeout; k++) { 2666 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2667 break; 2668 udelay(1); 2669 } 2670 if (k == adev->usec_timeout) { 2671 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 2672 0xffffffff, 0xffffffff, 0); 2673 mutex_unlock(&adev->grbm_idx_mutex); 2674 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2675 i, j); 2676 return; 2677 } 2678 } 2679 } 2680 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 2681 mutex_unlock(&adev->grbm_idx_mutex); 2682 2683 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2684 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2685 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2686 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2687 for (k = 0; k < adev->usec_timeout; k++) { 2688 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2689 break; 2690 udelay(1); 2691 } 2692 } 2693 2694 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2695 bool enable) 2696 { 2697 u32 tmp; 2698 2699 /* These interrupts should be enabled to drive DS clock */ 2700 2701 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2702 2703 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2704 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2705 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2706 if (adev->gfx.num_gfx_rings) 2707 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2708 2709 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2710 } 2711 2712 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2713 { 2714 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2715 /* csib */ 2716 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2717 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2718 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2719 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2720 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2721 adev->gfx.rlc.clear_state_size); 2722 } 2723 2724 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2725 int indirect_offset, 2726 int list_size, 2727 int *unique_indirect_regs, 2728 int unique_indirect_reg_count, 2729 int *indirect_start_offsets, 2730 int *indirect_start_offsets_count, 2731 int max_start_offsets_count) 2732 { 2733 int idx; 2734 2735 for (; indirect_offset < list_size; indirect_offset++) { 2736 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2737 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2738 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2739 2740 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2741 indirect_offset += 2; 2742 2743 /* look for the matching indice */ 2744 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2745 if (unique_indirect_regs[idx] == 2746 register_list_format[indirect_offset] || 2747 !unique_indirect_regs[idx]) 2748 break; 2749 } 2750 2751 BUG_ON(idx >= unique_indirect_reg_count); 2752 2753 if (!unique_indirect_regs[idx]) 2754 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2755 2756 indirect_offset++; 2757 } 2758 } 2759 } 2760 2761 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2762 { 2763 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2764 int unique_indirect_reg_count = 0; 2765 2766 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2767 int indirect_start_offsets_count = 0; 2768 2769 int list_size = 0; 2770 int i = 0, j = 0; 2771 u32 tmp = 0; 2772 2773 u32 *register_list_format = 2774 kmemdup(adev->gfx.rlc.register_list_format, 2775 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2776 if (!register_list_format) 2777 return -ENOMEM; 2778 2779 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2780 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2781 gfx_v9_1_parse_ind_reg_list(register_list_format, 2782 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2783 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2784 unique_indirect_regs, 2785 unique_indirect_reg_count, 2786 indirect_start_offsets, 2787 &indirect_start_offsets_count, 2788 ARRAY_SIZE(indirect_start_offsets)); 2789 2790 /* enable auto inc in case it is disabled */ 2791 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2792 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2793 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2794 2795 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2796 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2797 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2798 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2799 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2800 adev->gfx.rlc.register_restore[i]); 2801 2802 /* load indirect register */ 2803 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2804 adev->gfx.rlc.reg_list_format_start); 2805 2806 /* direct register portion */ 2807 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2808 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2809 register_list_format[i]); 2810 2811 /* indirect register portion */ 2812 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2813 if (register_list_format[i] == 0xFFFFFFFF) { 2814 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2815 continue; 2816 } 2817 2818 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2819 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2820 2821 for (j = 0; j < unique_indirect_reg_count; j++) { 2822 if (register_list_format[i] == unique_indirect_regs[j]) { 2823 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2824 break; 2825 } 2826 } 2827 2828 BUG_ON(j >= unique_indirect_reg_count); 2829 2830 i++; 2831 } 2832 2833 /* set save/restore list size */ 2834 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2835 list_size = list_size >> 1; 2836 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2837 adev->gfx.rlc.reg_restore_list_size); 2838 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2839 2840 /* write the starting offsets to RLC scratch ram */ 2841 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2842 adev->gfx.rlc.starting_offsets_start); 2843 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2844 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2845 indirect_start_offsets[i]); 2846 2847 /* load unique indirect regs*/ 2848 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2849 if (unique_indirect_regs[i] != 0) { 2850 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2851 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2852 unique_indirect_regs[i] & 0x3FFFF); 2853 2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2855 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2856 unique_indirect_regs[i] >> 20); 2857 } 2858 } 2859 2860 kfree(register_list_format); 2861 return 0; 2862 } 2863 2864 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2865 { 2866 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2867 } 2868 2869 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2870 bool enable) 2871 { 2872 uint32_t data = 0; 2873 uint32_t default_data = 0; 2874 2875 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2876 if (enable) { 2877 /* enable GFXIP control over CGPG */ 2878 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2879 if(default_data != data) 2880 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2881 2882 /* update status */ 2883 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2884 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2885 if(default_data != data) 2886 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2887 } else { 2888 /* restore GFXIP control over GCPG */ 2889 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2890 if(default_data != data) 2891 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2892 } 2893 } 2894 2895 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2896 { 2897 uint32_t data = 0; 2898 2899 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2900 AMD_PG_SUPPORT_GFX_SMG | 2901 AMD_PG_SUPPORT_GFX_DMG)) { 2902 /* init IDLE_POLL_COUNT = 60 */ 2903 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2904 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2905 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2906 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2907 2908 /* init RLC PG Delay */ 2909 data = 0; 2910 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2911 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2912 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2913 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2914 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2915 2916 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2917 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2918 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2919 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2920 2921 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2922 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2923 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2924 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2925 2926 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2927 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2928 2929 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2930 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2931 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2932 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0)) 2933 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2934 } 2935 } 2936 2937 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2938 bool enable) 2939 { 2940 uint32_t data = 0; 2941 uint32_t default_data = 0; 2942 2943 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2944 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2945 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2946 enable ? 1 : 0); 2947 if (default_data != data) 2948 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2949 } 2950 2951 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2952 bool enable) 2953 { 2954 uint32_t data = 0; 2955 uint32_t default_data = 0; 2956 2957 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2958 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2959 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2960 enable ? 1 : 0); 2961 if(default_data != data) 2962 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2963 } 2964 2965 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2966 bool enable) 2967 { 2968 uint32_t data = 0; 2969 uint32_t default_data = 0; 2970 2971 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2972 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2973 CP_PG_DISABLE, 2974 enable ? 0 : 1); 2975 if(default_data != data) 2976 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2977 } 2978 2979 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2980 bool enable) 2981 { 2982 uint32_t data, default_data; 2983 2984 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2985 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2986 GFX_POWER_GATING_ENABLE, 2987 enable ? 1 : 0); 2988 if(default_data != data) 2989 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2990 } 2991 2992 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2993 bool enable) 2994 { 2995 uint32_t data, default_data; 2996 2997 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2998 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2999 GFX_PIPELINE_PG_ENABLE, 3000 enable ? 1 : 0); 3001 if(default_data != data) 3002 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3003 3004 if (!enable) 3005 /* read any GFX register to wake up GFX */ 3006 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 3007 } 3008 3009 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 3010 bool enable) 3011 { 3012 uint32_t data, default_data; 3013 3014 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3015 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3016 STATIC_PER_CU_PG_ENABLE, 3017 enable ? 1 : 0); 3018 if(default_data != data) 3019 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3020 } 3021 3022 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3023 bool enable) 3024 { 3025 uint32_t data, default_data; 3026 3027 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3028 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3029 DYN_PER_CU_PG_ENABLE, 3030 enable ? 1 : 0); 3031 if(default_data != data) 3032 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3033 } 3034 3035 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3036 { 3037 gfx_v9_0_init_csb(adev); 3038 3039 /* 3040 * Rlc save restore list is workable since v2_1. 3041 * And it's needed by gfxoff feature. 3042 */ 3043 if (adev->gfx.rlc.is_rlc_v2_1) { 3044 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 3045 IP_VERSION(9, 2, 1) || 3046 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3047 gfx_v9_1_init_rlc_save_restore_list(adev); 3048 gfx_v9_0_enable_save_restore_machine(adev); 3049 } 3050 3051 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3052 AMD_PG_SUPPORT_GFX_SMG | 3053 AMD_PG_SUPPORT_GFX_DMG | 3054 AMD_PG_SUPPORT_CP | 3055 AMD_PG_SUPPORT_GDS | 3056 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3057 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE, 3058 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3059 gfx_v9_0_init_gfx_power_gating(adev); 3060 } 3061 } 3062 3063 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3064 { 3065 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3066 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3067 gfx_v9_0_wait_for_rlc_serdes(adev); 3068 } 3069 3070 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3071 { 3072 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3073 udelay(50); 3074 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3075 udelay(50); 3076 } 3077 3078 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3079 { 3080 #ifdef AMDGPU_RLC_DEBUG_RETRY 3081 u32 rlc_ucode_ver; 3082 #endif 3083 3084 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3085 udelay(50); 3086 3087 /* carrizo do enable cp interrupt after cp inited */ 3088 if (!(adev->flags & AMD_IS_APU)) { 3089 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3090 udelay(50); 3091 } 3092 3093 #ifdef AMDGPU_RLC_DEBUG_RETRY 3094 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3095 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3096 if(rlc_ucode_ver == 0x108) { 3097 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3098 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3099 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3100 * default is 0x9C4 to create a 100us interval */ 3101 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3102 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3103 * to disable the page fault retry interrupts, default is 3104 * 0x100 (256) */ 3105 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3106 } 3107 #endif 3108 } 3109 3110 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3111 { 3112 const struct rlc_firmware_header_v2_0 *hdr; 3113 const __le32 *fw_data; 3114 unsigned i, fw_size; 3115 3116 if (!adev->gfx.rlc_fw) 3117 return -EINVAL; 3118 3119 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3120 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3121 3122 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3123 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3124 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3125 3126 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3127 RLCG_UCODE_LOADING_START_ADDRESS); 3128 for (i = 0; i < fw_size; i++) 3129 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3130 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3131 3132 return 0; 3133 } 3134 3135 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3136 { 3137 int r; 3138 3139 if (amdgpu_sriov_vf(adev)) { 3140 gfx_v9_0_init_csb(adev); 3141 return 0; 3142 } 3143 3144 adev->gfx.rlc.funcs->stop(adev); 3145 3146 /* disable CG */ 3147 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3148 3149 gfx_v9_0_init_pg(adev); 3150 3151 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3152 /* legacy rlc firmware loading */ 3153 r = gfx_v9_0_rlc_load_microcode(adev); 3154 if (r) 3155 return r; 3156 } 3157 3158 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 3159 case IP_VERSION(9, 2, 2): 3160 case IP_VERSION(9, 1, 0): 3161 gfx_v9_0_init_lbpw(adev); 3162 if (amdgpu_lbpw == 0) 3163 gfx_v9_0_enable_lbpw(adev, false); 3164 else 3165 gfx_v9_0_enable_lbpw(adev, true); 3166 break; 3167 case IP_VERSION(9, 4, 0): 3168 gfx_v9_4_init_lbpw(adev); 3169 if (amdgpu_lbpw > 0) 3170 gfx_v9_0_enable_lbpw(adev, true); 3171 else 3172 gfx_v9_0_enable_lbpw(adev, false); 3173 break; 3174 default: 3175 break; 3176 } 3177 3178 gfx_v9_0_update_spm_vmid_internal(adev, 0xf); 3179 3180 adev->gfx.rlc.funcs->start(adev); 3181 3182 return 0; 3183 } 3184 3185 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3186 { 3187 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3188 3189 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1); 3190 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1); 3191 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1); 3192 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1); 3193 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1); 3194 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1); 3195 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1); 3196 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1); 3197 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1); 3198 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3199 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3200 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3201 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3202 udelay(50); 3203 } 3204 3205 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3206 { 3207 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3208 const struct gfx_firmware_header_v1_0 *ce_hdr; 3209 const struct gfx_firmware_header_v1_0 *me_hdr; 3210 const __le32 *fw_data; 3211 unsigned i, fw_size; 3212 3213 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3214 return -EINVAL; 3215 3216 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3217 adev->gfx.pfp_fw->data; 3218 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3219 adev->gfx.ce_fw->data; 3220 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3221 adev->gfx.me_fw->data; 3222 3223 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3224 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3225 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3226 3227 gfx_v9_0_cp_gfx_enable(adev, false); 3228 3229 /* PFP */ 3230 fw_data = (const __le32 *) 3231 (adev->gfx.pfp_fw->data + 3232 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3233 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3234 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3235 for (i = 0; i < fw_size; i++) 3236 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3237 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3238 3239 /* CE */ 3240 fw_data = (const __le32 *) 3241 (adev->gfx.ce_fw->data + 3242 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3243 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3244 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3245 for (i = 0; i < fw_size; i++) 3246 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3247 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3248 3249 /* ME */ 3250 fw_data = (const __le32 *) 3251 (adev->gfx.me_fw->data + 3252 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3253 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3254 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3255 for (i = 0; i < fw_size; i++) 3256 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3257 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3258 3259 return 0; 3260 } 3261 3262 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3263 { 3264 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3265 const struct cs_section_def *sect = NULL; 3266 const struct cs_extent_def *ext = NULL; 3267 int r, i, tmp; 3268 3269 /* init the CP */ 3270 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3271 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3272 3273 gfx_v9_0_cp_gfx_enable(adev, true); 3274 3275 /* Now only limit the quirk on the APU gfx9 series and already 3276 * confirmed that the APU gfx10/gfx11 needn't such update. 3277 */ 3278 if (adev->flags & AMD_IS_APU && 3279 adev->in_s3 && !adev->suspend_complete) { 3280 DRM_INFO(" Will skip the CSB packet resubmit\n"); 3281 return 0; 3282 } 3283 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3284 if (r) { 3285 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3286 return r; 3287 } 3288 3289 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3290 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3291 3292 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3293 amdgpu_ring_write(ring, 0x80000000); 3294 amdgpu_ring_write(ring, 0x80000000); 3295 3296 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3297 for (ext = sect->section; ext->extent != NULL; ++ext) { 3298 if (sect->id == SECT_CONTEXT) { 3299 amdgpu_ring_write(ring, 3300 PACKET3(PACKET3_SET_CONTEXT_REG, 3301 ext->reg_count)); 3302 amdgpu_ring_write(ring, 3303 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3304 for (i = 0; i < ext->reg_count; i++) 3305 amdgpu_ring_write(ring, ext->extent[i]); 3306 } 3307 } 3308 } 3309 3310 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3311 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3312 3313 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3314 amdgpu_ring_write(ring, 0); 3315 3316 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3317 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3318 amdgpu_ring_write(ring, 0x8000); 3319 amdgpu_ring_write(ring, 0x8000); 3320 3321 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3322 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3323 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3324 amdgpu_ring_write(ring, tmp); 3325 amdgpu_ring_write(ring, 0); 3326 3327 amdgpu_ring_commit(ring); 3328 3329 return 0; 3330 } 3331 3332 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3333 { 3334 struct amdgpu_ring *ring; 3335 u32 tmp; 3336 u32 rb_bufsz; 3337 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3338 3339 /* Set the write pointer delay */ 3340 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3341 3342 /* set the RB to use vmid 0 */ 3343 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3344 3345 /* Set ring buffer size */ 3346 ring = &adev->gfx.gfx_ring[0]; 3347 rb_bufsz = order_base_2(ring->ring_size / 8); 3348 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3349 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3350 #ifdef __BIG_ENDIAN 3351 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3352 #endif 3353 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3354 3355 /* Initialize the ring buffer's write pointers */ 3356 ring->wptr = 0; 3357 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3358 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3359 3360 /* set the wb address whether it's enabled or not */ 3361 rptr_addr = ring->rptr_gpu_addr; 3362 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3363 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3364 3365 wptr_gpu_addr = ring->wptr_gpu_addr; 3366 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3367 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3368 3369 mdelay(1); 3370 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3371 3372 rb_addr = ring->gpu_addr >> 8; 3373 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3374 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3375 3376 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3377 if (ring->use_doorbell) { 3378 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3379 DOORBELL_OFFSET, ring->doorbell_index); 3380 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3381 DOORBELL_EN, 1); 3382 } else { 3383 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3384 } 3385 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3386 3387 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3388 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3389 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3390 3391 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3392 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3393 3394 3395 /* start the ring */ 3396 gfx_v9_0_cp_gfx_start(adev); 3397 3398 return 0; 3399 } 3400 3401 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3402 { 3403 if (enable) { 3404 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3405 } else { 3406 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3407 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK | 3408 CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK | 3409 CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK | 3410 CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK | 3411 CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK | 3412 CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK | 3413 CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK | 3414 CP_MEC_CNTL__MEC_ME1_HALT_MASK | 3415 CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3416 adev->gfx.kiq[0].ring.sched.ready = false; 3417 } 3418 udelay(50); 3419 } 3420 3421 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3422 { 3423 const struct gfx_firmware_header_v1_0 *mec_hdr; 3424 const __le32 *fw_data; 3425 unsigned i; 3426 u32 tmp; 3427 3428 if (!adev->gfx.mec_fw) 3429 return -EINVAL; 3430 3431 gfx_v9_0_cp_compute_enable(adev, false); 3432 3433 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3434 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3435 3436 fw_data = (const __le32 *) 3437 (adev->gfx.mec_fw->data + 3438 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3439 tmp = 0; 3440 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3441 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3442 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3443 3444 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3445 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3446 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3447 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3448 3449 /* MEC1 */ 3450 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3451 mec_hdr->jt_offset); 3452 for (i = 0; i < mec_hdr->jt_size; i++) 3453 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3454 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3455 3456 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3457 adev->gfx.mec_fw_version); 3458 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3459 3460 return 0; 3461 } 3462 3463 /* KIQ functions */ 3464 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3465 { 3466 uint32_t tmp; 3467 struct amdgpu_device *adev = ring->adev; 3468 3469 /* tell RLC which is KIQ queue */ 3470 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3471 tmp &= 0xffffff00; 3472 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3473 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3474 tmp |= 0x80; 3475 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3476 } 3477 3478 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3479 { 3480 struct amdgpu_device *adev = ring->adev; 3481 3482 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3483 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3484 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3485 mqd->cp_hqd_queue_priority = 3486 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3487 } 3488 } 3489 } 3490 3491 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3492 { 3493 struct amdgpu_device *adev = ring->adev; 3494 struct v9_mqd *mqd = ring->mqd_ptr; 3495 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3496 uint32_t tmp; 3497 3498 mqd->header = 0xC0310800; 3499 mqd->compute_pipelinestat_enable = 0x00000001; 3500 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3501 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3502 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3503 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3504 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3505 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3506 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3507 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3508 mqd->compute_misc_reserved = 0x00000003; 3509 3510 mqd->dynamic_cu_mask_addr_lo = 3511 lower_32_bits(ring->mqd_gpu_addr 3512 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3513 mqd->dynamic_cu_mask_addr_hi = 3514 upper_32_bits(ring->mqd_gpu_addr 3515 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3516 3517 eop_base_addr = ring->eop_gpu_addr >> 8; 3518 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3519 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3520 3521 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3522 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3523 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3524 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3525 3526 mqd->cp_hqd_eop_control = tmp; 3527 3528 /* enable doorbell? */ 3529 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3530 3531 if (ring->use_doorbell) { 3532 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3533 DOORBELL_OFFSET, ring->doorbell_index); 3534 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3535 DOORBELL_EN, 1); 3536 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3537 DOORBELL_SOURCE, 0); 3538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3539 DOORBELL_HIT, 0); 3540 } else { 3541 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3542 DOORBELL_EN, 0); 3543 } 3544 3545 mqd->cp_hqd_pq_doorbell_control = tmp; 3546 3547 /* disable the queue if it's active */ 3548 ring->wptr = 0; 3549 mqd->cp_hqd_dequeue_request = 0; 3550 mqd->cp_hqd_pq_rptr = 0; 3551 mqd->cp_hqd_pq_wptr_lo = 0; 3552 mqd->cp_hqd_pq_wptr_hi = 0; 3553 3554 /* set the pointer to the MQD */ 3555 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3556 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3557 3558 /* set MQD vmid to 0 */ 3559 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3560 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3561 mqd->cp_mqd_control = tmp; 3562 3563 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3564 hqd_gpu_addr = ring->gpu_addr >> 8; 3565 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3566 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3567 3568 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3569 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3570 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3571 (order_base_2(ring->ring_size / 4) - 1)); 3572 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3573 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 3574 #ifdef __BIG_ENDIAN 3575 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3576 #endif 3577 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3578 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3579 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3580 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3581 mqd->cp_hqd_pq_control = tmp; 3582 3583 /* set the wb address whether it's enabled or not */ 3584 wb_gpu_addr = ring->rptr_gpu_addr; 3585 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3586 mqd->cp_hqd_pq_rptr_report_addr_hi = 3587 upper_32_bits(wb_gpu_addr) & 0xffff; 3588 3589 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3590 wb_gpu_addr = ring->wptr_gpu_addr; 3591 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3592 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3593 3594 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3595 ring->wptr = 0; 3596 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3597 3598 /* set the vmid for the queue */ 3599 mqd->cp_hqd_vmid = 0; 3600 3601 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3602 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3603 mqd->cp_hqd_persistent_state = tmp; 3604 3605 /* set MIN_IB_AVAIL_SIZE */ 3606 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3607 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3608 mqd->cp_hqd_ib_control = tmp; 3609 3610 /* set static priority for a queue/ring */ 3611 gfx_v9_0_mqd_set_priority(ring, mqd); 3612 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); 3613 3614 /* map_queues packet doesn't need activate the queue, 3615 * so only kiq need set this field. 3616 */ 3617 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3618 mqd->cp_hqd_active = 1; 3619 3620 return 0; 3621 } 3622 3623 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3624 { 3625 struct amdgpu_device *adev = ring->adev; 3626 struct v9_mqd *mqd = ring->mqd_ptr; 3627 int j; 3628 3629 /* disable wptr polling */ 3630 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3631 3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3633 mqd->cp_hqd_eop_base_addr_lo); 3634 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3635 mqd->cp_hqd_eop_base_addr_hi); 3636 3637 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3638 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3639 mqd->cp_hqd_eop_control); 3640 3641 /* enable doorbell? */ 3642 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3643 mqd->cp_hqd_pq_doorbell_control); 3644 3645 /* disable the queue if it's active */ 3646 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3647 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3648 for (j = 0; j < adev->usec_timeout; j++) { 3649 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3650 break; 3651 udelay(1); 3652 } 3653 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3654 mqd->cp_hqd_dequeue_request); 3655 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3656 mqd->cp_hqd_pq_rptr); 3657 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3658 mqd->cp_hqd_pq_wptr_lo); 3659 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3660 mqd->cp_hqd_pq_wptr_hi); 3661 } 3662 3663 /* set the pointer to the MQD */ 3664 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3665 mqd->cp_mqd_base_addr_lo); 3666 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3667 mqd->cp_mqd_base_addr_hi); 3668 3669 /* set MQD vmid to 0 */ 3670 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3671 mqd->cp_mqd_control); 3672 3673 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3674 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3675 mqd->cp_hqd_pq_base_lo); 3676 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3677 mqd->cp_hqd_pq_base_hi); 3678 3679 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3680 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3681 mqd->cp_hqd_pq_control); 3682 3683 /* set the wb address whether it's enabled or not */ 3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3685 mqd->cp_hqd_pq_rptr_report_addr_lo); 3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3687 mqd->cp_hqd_pq_rptr_report_addr_hi); 3688 3689 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3691 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3692 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3693 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3694 3695 /* enable the doorbell if requested */ 3696 if (ring->use_doorbell) { 3697 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3698 (adev->doorbell_index.kiq * 2) << 2); 3699 /* If GC has entered CGPG, ringing doorbell > first page 3700 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3701 * workaround this issue. And this change has to align with firmware 3702 * update. 3703 */ 3704 if (check_if_enlarge_doorbell_range(adev)) 3705 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3706 (adev->doorbell.size - 4)); 3707 else 3708 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3709 (adev->doorbell_index.userqueue_end * 2) << 2); 3710 } 3711 3712 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3713 mqd->cp_hqd_pq_doorbell_control); 3714 3715 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3716 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3717 mqd->cp_hqd_pq_wptr_lo); 3718 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3719 mqd->cp_hqd_pq_wptr_hi); 3720 3721 /* set the vmid for the queue */ 3722 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3723 3724 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3725 mqd->cp_hqd_persistent_state); 3726 3727 /* activate the queue */ 3728 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3729 mqd->cp_hqd_active); 3730 3731 if (ring->use_doorbell) 3732 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3733 3734 return 0; 3735 } 3736 3737 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3738 { 3739 struct amdgpu_device *adev = ring->adev; 3740 int j; 3741 3742 /* disable the queue if it's active */ 3743 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3744 3745 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3746 3747 for (j = 0; j < adev->usec_timeout; j++) { 3748 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3749 break; 3750 udelay(1); 3751 } 3752 3753 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3754 DRM_DEBUG("KIQ dequeue request failed.\n"); 3755 3756 /* Manual disable if dequeue request times out */ 3757 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3758 } 3759 3760 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3761 0); 3762 } 3763 3764 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3765 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3766 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3767 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3768 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3769 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3770 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3771 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3772 3773 return 0; 3774 } 3775 3776 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3777 { 3778 struct amdgpu_device *adev = ring->adev; 3779 struct v9_mqd *mqd = ring->mqd_ptr; 3780 struct v9_mqd *tmp_mqd; 3781 3782 gfx_v9_0_kiq_setting(ring); 3783 3784 /* GPU could be in bad state during probe, driver trigger the reset 3785 * after load the SMU, in this case , the mqd is not be initialized. 3786 * driver need to re-init the mqd. 3787 * check mqd->cp_hqd_pq_control since this value should not be 0 3788 */ 3789 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup; 3790 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3791 /* for GPU_RESET case , reset MQD to a clean status */ 3792 if (adev->gfx.kiq[0].mqd_backup) 3793 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation)); 3794 3795 /* reset ring buffer */ 3796 ring->wptr = 0; 3797 amdgpu_ring_clear_ring(ring); 3798 3799 mutex_lock(&adev->srbm_mutex); 3800 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3801 gfx_v9_0_kiq_init_register(ring); 3802 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3803 mutex_unlock(&adev->srbm_mutex); 3804 } else { 3805 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3806 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3807 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3808 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 3809 amdgpu_ring_clear_ring(ring); 3810 mutex_lock(&adev->srbm_mutex); 3811 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3812 gfx_v9_0_mqd_init(ring); 3813 gfx_v9_0_kiq_init_register(ring); 3814 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3815 mutex_unlock(&adev->srbm_mutex); 3816 3817 if (adev->gfx.kiq[0].mqd_backup) 3818 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation)); 3819 } 3820 3821 return 0; 3822 } 3823 3824 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore) 3825 { 3826 struct amdgpu_device *adev = ring->adev; 3827 struct v9_mqd *mqd = ring->mqd_ptr; 3828 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3829 struct v9_mqd *tmp_mqd; 3830 3831 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3832 * is not be initialized before 3833 */ 3834 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3835 3836 if (!restore && (!tmp_mqd->cp_hqd_pq_control || 3837 (!amdgpu_in_reset(adev) && !adev->in_suspend))) { 3838 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3839 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3840 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3841 mutex_lock(&adev->srbm_mutex); 3842 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3843 gfx_v9_0_mqd_init(ring); 3844 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3845 mutex_unlock(&adev->srbm_mutex); 3846 3847 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3848 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3849 } else { 3850 /* restore MQD to a clean status */ 3851 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3852 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3853 /* reset ring buffer */ 3854 ring->wptr = 0; 3855 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 3856 amdgpu_ring_clear_ring(ring); 3857 } 3858 3859 return 0; 3860 } 3861 3862 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3863 { 3864 struct amdgpu_ring *ring; 3865 int r; 3866 3867 ring = &adev->gfx.kiq[0].ring; 3868 3869 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3870 if (unlikely(r != 0)) 3871 return r; 3872 3873 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3874 if (unlikely(r != 0)) { 3875 amdgpu_bo_unreserve(ring->mqd_obj); 3876 return r; 3877 } 3878 3879 gfx_v9_0_kiq_init_queue(ring); 3880 amdgpu_bo_kunmap(ring->mqd_obj); 3881 ring->mqd_ptr = NULL; 3882 amdgpu_bo_unreserve(ring->mqd_obj); 3883 return 0; 3884 } 3885 3886 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3887 { 3888 struct amdgpu_ring *ring = NULL; 3889 int r = 0, i; 3890 3891 gfx_v9_0_cp_compute_enable(adev, true); 3892 3893 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3894 ring = &adev->gfx.compute_ring[i]; 3895 3896 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3897 if (unlikely(r != 0)) 3898 goto done; 3899 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3900 if (!r) { 3901 r = gfx_v9_0_kcq_init_queue(ring, false); 3902 amdgpu_bo_kunmap(ring->mqd_obj); 3903 ring->mqd_ptr = NULL; 3904 } 3905 amdgpu_bo_unreserve(ring->mqd_obj); 3906 if (r) 3907 goto done; 3908 } 3909 3910 r = amdgpu_gfx_enable_kcq(adev, 0); 3911 done: 3912 return r; 3913 } 3914 3915 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3916 { 3917 int r, i; 3918 struct amdgpu_ring *ring; 3919 3920 if (!(adev->flags & AMD_IS_APU)) 3921 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3922 3923 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3924 if (adev->gfx.num_gfx_rings) { 3925 /* legacy firmware loading */ 3926 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3927 if (r) 3928 return r; 3929 } 3930 3931 r = gfx_v9_0_cp_compute_load_microcode(adev); 3932 if (r) 3933 return r; 3934 } 3935 3936 if (adev->gfx.num_gfx_rings) 3937 gfx_v9_0_cp_gfx_enable(adev, false); 3938 gfx_v9_0_cp_compute_enable(adev, false); 3939 3940 r = gfx_v9_0_kiq_resume(adev); 3941 if (r) 3942 return r; 3943 3944 if (adev->gfx.num_gfx_rings) { 3945 r = gfx_v9_0_cp_gfx_resume(adev); 3946 if (r) 3947 return r; 3948 } 3949 3950 r = gfx_v9_0_kcq_resume(adev); 3951 if (r) 3952 return r; 3953 3954 if (adev->gfx.num_gfx_rings) { 3955 ring = &adev->gfx.gfx_ring[0]; 3956 r = amdgpu_ring_test_helper(ring); 3957 if (r) 3958 return r; 3959 } 3960 3961 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3962 ring = &adev->gfx.compute_ring[i]; 3963 amdgpu_ring_test_helper(ring); 3964 } 3965 3966 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3967 3968 return 0; 3969 } 3970 3971 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3972 { 3973 u32 tmp; 3974 3975 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) && 3976 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) 3977 return; 3978 3979 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3980 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3981 adev->df.hash_status.hash_64k); 3982 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3983 adev->df.hash_status.hash_2m); 3984 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3985 adev->df.hash_status.hash_1g); 3986 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3987 } 3988 3989 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3990 { 3991 if (adev->gfx.num_gfx_rings) 3992 gfx_v9_0_cp_gfx_enable(adev, enable); 3993 gfx_v9_0_cp_compute_enable(adev, enable); 3994 } 3995 3996 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) 3997 { 3998 int r; 3999 struct amdgpu_device *adev = ip_block->adev; 4000 4001 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4002 adev->gfx.cleaner_shader_ptr); 4003 4004 if (!amdgpu_sriov_vf(adev)) 4005 gfx_v9_0_init_golden_registers(adev); 4006 4007 gfx_v9_0_constants_init(adev); 4008 4009 gfx_v9_0_init_tcp_config(adev); 4010 4011 r = adev->gfx.rlc.funcs->resume(adev); 4012 if (r) 4013 return r; 4014 4015 r = gfx_v9_0_cp_resume(adev); 4016 if (r) 4017 return r; 4018 4019 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4020 gfx_v9_4_2_set_power_brake_sequence(adev); 4021 4022 return r; 4023 } 4024 4025 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) 4026 { 4027 struct amdgpu_device *adev = ip_block->adev; 4028 4029 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4030 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4031 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4032 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4033 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 4034 4035 /* DF freeze and kcq disable will fail */ 4036 if (!amdgpu_ras_intr_triggered()) 4037 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4038 amdgpu_gfx_disable_kcq(adev, 0); 4039 4040 if (amdgpu_sriov_vf(adev)) { 4041 gfx_v9_0_cp_gfx_enable(adev, false); 4042 /* must disable polling for SRIOV when hw finished, otherwise 4043 * CPC engine may still keep fetching WB address which is already 4044 * invalid after sw finished and trigger DMAR reading error in 4045 * hypervisor side. 4046 */ 4047 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4048 return 0; 4049 } 4050 4051 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4052 * otherwise KIQ is hanging when binding back 4053 */ 4054 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4055 mutex_lock(&adev->srbm_mutex); 4056 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me, 4057 adev->gfx.kiq[0].ring.pipe, 4058 adev->gfx.kiq[0].ring.queue, 0, 0); 4059 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring); 4060 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 4061 mutex_unlock(&adev->srbm_mutex); 4062 } 4063 4064 gfx_v9_0_cp_enable(adev, false); 4065 4066 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */ 4067 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) || 4068 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) { 4069 dev_dbg(adev->dev, "Skipping RLC halt\n"); 4070 return 0; 4071 } 4072 4073 adev->gfx.rlc.funcs->stop(adev); 4074 return 0; 4075 } 4076 4077 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block) 4078 { 4079 return gfx_v9_0_hw_fini(ip_block); 4080 } 4081 4082 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block) 4083 { 4084 return gfx_v9_0_hw_init(ip_block); 4085 } 4086 4087 static bool gfx_v9_0_is_idle(void *handle) 4088 { 4089 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4090 4091 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4092 GRBM_STATUS, GUI_ACTIVE)) 4093 return false; 4094 else 4095 return true; 4096 } 4097 4098 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 4099 { 4100 unsigned i; 4101 struct amdgpu_device *adev = ip_block->adev; 4102 4103 for (i = 0; i < adev->usec_timeout; i++) { 4104 if (gfx_v9_0_is_idle(adev)) 4105 return 0; 4106 udelay(1); 4107 } 4108 return -ETIMEDOUT; 4109 } 4110 4111 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block) 4112 { 4113 u32 grbm_soft_reset = 0; 4114 u32 tmp; 4115 struct amdgpu_device *adev = ip_block->adev; 4116 4117 /* GRBM_STATUS */ 4118 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4119 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4120 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4121 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4122 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4123 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4124 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4125 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4126 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4127 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4128 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4129 } 4130 4131 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4132 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4133 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4134 } 4135 4136 /* GRBM_STATUS2 */ 4137 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4138 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4139 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4140 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4141 4142 4143 if (grbm_soft_reset) { 4144 /* stop the rlc */ 4145 adev->gfx.rlc.funcs->stop(adev); 4146 4147 if (adev->gfx.num_gfx_rings) 4148 /* Disable GFX parsing/prefetching */ 4149 gfx_v9_0_cp_gfx_enable(adev, false); 4150 4151 /* Disable MEC parsing/prefetching */ 4152 gfx_v9_0_cp_compute_enable(adev, false); 4153 4154 if (grbm_soft_reset) { 4155 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4156 tmp |= grbm_soft_reset; 4157 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4158 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4159 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4160 4161 udelay(50); 4162 4163 tmp &= ~grbm_soft_reset; 4164 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4165 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4166 } 4167 4168 /* Wait a little for things to settle down */ 4169 udelay(50); 4170 } 4171 return 0; 4172 } 4173 4174 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4175 { 4176 signed long r, cnt = 0; 4177 unsigned long flags; 4178 uint32_t seq, reg_val_offs = 0; 4179 uint64_t value = 0; 4180 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 4181 struct amdgpu_ring *ring = &kiq->ring; 4182 4183 BUG_ON(!ring->funcs->emit_rreg); 4184 4185 spin_lock_irqsave(&kiq->ring_lock, flags); 4186 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4187 pr_err("critical bug! too many kiq readers\n"); 4188 goto failed_unlock; 4189 } 4190 amdgpu_ring_alloc(ring, 32); 4191 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4192 amdgpu_ring_write(ring, 9 | /* src: register*/ 4193 (5 << 8) | /* dst: memory */ 4194 (1 << 16) | /* count sel */ 4195 (1 << 20)); /* write confirm */ 4196 amdgpu_ring_write(ring, 0); 4197 amdgpu_ring_write(ring, 0); 4198 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4199 reg_val_offs * 4)); 4200 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4201 reg_val_offs * 4)); 4202 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4203 if (r) 4204 goto failed_undo; 4205 4206 amdgpu_ring_commit(ring); 4207 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4208 4209 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4210 4211 /* don't wait anymore for gpu reset case because this way may 4212 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4213 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4214 * never return if we keep waiting in virt_kiq_rreg, which cause 4215 * gpu_recover() hang there. 4216 * 4217 * also don't wait anymore for IRQ context 4218 * */ 4219 if (r < 1 && (amdgpu_in_reset(adev))) 4220 goto failed_kiq_read; 4221 4222 might_sleep(); 4223 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4224 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4225 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4226 } 4227 4228 if (cnt > MAX_KIQ_REG_TRY) 4229 goto failed_kiq_read; 4230 4231 mb(); 4232 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4233 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4234 amdgpu_device_wb_free(adev, reg_val_offs); 4235 return value; 4236 4237 failed_undo: 4238 amdgpu_ring_undo(ring); 4239 failed_unlock: 4240 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4241 failed_kiq_read: 4242 if (reg_val_offs) 4243 amdgpu_device_wb_free(adev, reg_val_offs); 4244 pr_err("failed to read gpu clock\n"); 4245 return ~0; 4246 } 4247 4248 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4249 { 4250 uint64_t clock, clock_lo, clock_hi, hi_check; 4251 4252 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 4253 case IP_VERSION(9, 3, 0): 4254 preempt_disable(); 4255 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 4256 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4257 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 4258 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over 4259 * roughly every 42 seconds. 4260 */ 4261 if (hi_check != clock_hi) { 4262 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4263 clock_hi = hi_check; 4264 } 4265 preempt_enable(); 4266 clock = clock_lo | (clock_hi << 32ULL); 4267 break; 4268 default: 4269 amdgpu_gfx_off_ctrl(adev, false); 4270 mutex_lock(&adev->gfx.gpu_clock_mutex); 4271 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 4272 IP_VERSION(9, 0, 1) && 4273 amdgpu_sriov_runtime(adev)) { 4274 clock = gfx_v9_0_kiq_read_clock(adev); 4275 } else { 4276 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4277 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4278 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4279 } 4280 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4281 amdgpu_gfx_off_ctrl(adev, true); 4282 break; 4283 } 4284 return clock; 4285 } 4286 4287 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4288 uint32_t vmid, 4289 uint32_t gds_base, uint32_t gds_size, 4290 uint32_t gws_base, uint32_t gws_size, 4291 uint32_t oa_base, uint32_t oa_size) 4292 { 4293 struct amdgpu_device *adev = ring->adev; 4294 4295 /* GDS Base */ 4296 gfx_v9_0_write_data_to_reg(ring, 0, false, 4297 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4298 gds_base); 4299 4300 /* GDS Size */ 4301 gfx_v9_0_write_data_to_reg(ring, 0, false, 4302 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4303 gds_size); 4304 4305 /* GWS */ 4306 gfx_v9_0_write_data_to_reg(ring, 0, false, 4307 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4308 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4309 4310 /* OA */ 4311 gfx_v9_0_write_data_to_reg(ring, 0, false, 4312 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4313 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4314 } 4315 4316 static const u32 vgpr_init_compute_shader[] = 4317 { 4318 0xb07c0000, 0xbe8000ff, 4319 0x000000f8, 0xbf110800, 4320 0x7e000280, 0x7e020280, 4321 0x7e040280, 0x7e060280, 4322 0x7e080280, 0x7e0a0280, 4323 0x7e0c0280, 0x7e0e0280, 4324 0x80808800, 0xbe803200, 4325 0xbf84fff5, 0xbf9c0000, 4326 0xd28c0001, 0x0001007f, 4327 0xd28d0001, 0x0002027e, 4328 0x10020288, 0xb8810904, 4329 0xb7814000, 0xd1196a01, 4330 0x00000301, 0xbe800087, 4331 0xbefc00c1, 0xd89c4000, 4332 0x00020201, 0xd89cc080, 4333 0x00040401, 0x320202ff, 4334 0x00000800, 0x80808100, 4335 0xbf84fff8, 0x7e020280, 4336 0xbf810000, 0x00000000, 4337 }; 4338 4339 static const u32 sgpr_init_compute_shader[] = 4340 { 4341 0xb07c0000, 0xbe8000ff, 4342 0x0000005f, 0xbee50080, 4343 0xbe812c65, 0xbe822c65, 4344 0xbe832c65, 0xbe842c65, 4345 0xbe852c65, 0xb77c0005, 4346 0x80808500, 0xbf84fff8, 4347 0xbe800080, 0xbf810000, 4348 }; 4349 4350 static const u32 vgpr_init_compute_shader_arcturus[] = { 4351 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4352 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4353 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4354 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4355 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4356 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4357 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4358 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4359 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4360 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4361 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4362 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4363 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4364 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4365 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4366 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4367 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4368 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4369 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4370 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4371 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4372 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4373 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4374 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4375 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4376 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4377 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4378 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4379 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4380 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4381 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4382 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4383 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4384 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4385 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4386 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4387 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4388 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4389 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4390 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4391 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4392 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4393 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4394 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4395 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4396 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4397 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4398 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4399 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4400 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4401 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4402 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4403 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4404 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4405 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4406 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4407 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4408 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4409 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4410 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4411 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4412 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4413 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4414 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4415 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4416 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4417 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4418 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4419 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4420 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4421 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4422 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4423 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4424 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4425 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4426 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4427 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4428 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4429 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4430 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4431 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4432 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4433 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4434 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4435 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4436 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4437 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4438 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4439 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4440 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4441 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4442 0xbf84fff8, 0xbf810000, 4443 }; 4444 4445 /* When below register arrays changed, please update gpr_reg_size, 4446 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4447 to cover all gfx9 ASICs */ 4448 static const struct soc15_reg_entry vgpr_init_regs[] = { 4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4451 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4452 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4453 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4454 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4455 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4456 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4457 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4458 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4459 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4460 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4461 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4462 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4463 }; 4464 4465 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4466 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4467 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4468 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4469 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4470 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4471 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4472 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4473 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4474 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4475 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4476 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4477 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4478 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4479 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4480 }; 4481 4482 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4483 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4484 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4485 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4486 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4487 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4488 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4489 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4490 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4491 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4492 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4493 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4494 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4495 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4496 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4497 }; 4498 4499 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4500 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4501 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4502 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4503 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4504 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4505 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4506 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4507 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4508 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4509 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4510 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4511 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4512 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4513 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4514 }; 4515 4516 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4517 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4518 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4519 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4520 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4521 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4522 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4523 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4524 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4525 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4526 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4527 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4528 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4529 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4530 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4531 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4532 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4533 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4534 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4535 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4536 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4537 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4538 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4539 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4540 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4541 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4542 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4543 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4544 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4545 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4546 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4547 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4548 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4549 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4550 }; 4551 4552 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4553 { 4554 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4555 int i, r; 4556 4557 /* only support when RAS is enabled */ 4558 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4559 return 0; 4560 4561 r = amdgpu_ring_alloc(ring, 7); 4562 if (r) { 4563 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4564 ring->name, r); 4565 return r; 4566 } 4567 4568 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4569 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4570 4571 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4572 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4573 PACKET3_DMA_DATA_DST_SEL(1) | 4574 PACKET3_DMA_DATA_SRC_SEL(2) | 4575 PACKET3_DMA_DATA_ENGINE(0))); 4576 amdgpu_ring_write(ring, 0); 4577 amdgpu_ring_write(ring, 0); 4578 amdgpu_ring_write(ring, 0); 4579 amdgpu_ring_write(ring, 0); 4580 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4581 adev->gds.gds_size); 4582 4583 amdgpu_ring_commit(ring); 4584 4585 for (i = 0; i < adev->usec_timeout; i++) { 4586 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4587 break; 4588 udelay(1); 4589 } 4590 4591 if (i >= adev->usec_timeout) 4592 r = -ETIMEDOUT; 4593 4594 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4595 4596 return r; 4597 } 4598 4599 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4600 { 4601 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4602 struct amdgpu_ib ib; 4603 struct dma_fence *f = NULL; 4604 int r, i; 4605 unsigned total_size, vgpr_offset, sgpr_offset; 4606 u64 gpu_addr; 4607 4608 int compute_dim_x = adev->gfx.config.max_shader_engines * 4609 adev->gfx.config.max_cu_per_sh * 4610 adev->gfx.config.max_sh_per_se; 4611 int sgpr_work_group_size = 5; 4612 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4613 int vgpr_init_shader_size; 4614 const u32 *vgpr_init_shader_ptr; 4615 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4616 4617 /* only support when RAS is enabled */ 4618 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4619 return 0; 4620 4621 /* bail if the compute ring is not ready */ 4622 if (!ring->sched.ready) 4623 return 0; 4624 4625 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) { 4626 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4627 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4628 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4629 } else { 4630 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4631 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4632 vgpr_init_regs_ptr = vgpr_init_regs; 4633 } 4634 4635 total_size = 4636 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4637 total_size += 4638 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4639 total_size += 4640 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4641 total_size = ALIGN(total_size, 256); 4642 vgpr_offset = total_size; 4643 total_size += ALIGN(vgpr_init_shader_size, 256); 4644 sgpr_offset = total_size; 4645 total_size += sizeof(sgpr_init_compute_shader); 4646 4647 /* allocate an indirect buffer to put the commands in */ 4648 memset(&ib, 0, sizeof(ib)); 4649 r = amdgpu_ib_get(adev, NULL, total_size, 4650 AMDGPU_IB_POOL_DIRECT, &ib); 4651 if (r) { 4652 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4653 return r; 4654 } 4655 4656 /* load the compute shaders */ 4657 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4658 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4659 4660 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4661 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4662 4663 /* init the ib length to 0 */ 4664 ib.length_dw = 0; 4665 4666 /* VGPR */ 4667 /* write the register state for the compute dispatch */ 4668 for (i = 0; i < gpr_reg_size; i++) { 4669 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4670 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4671 - PACKET3_SET_SH_REG_START; 4672 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4673 } 4674 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4675 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4676 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4677 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4678 - PACKET3_SET_SH_REG_START; 4679 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4680 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4681 4682 /* write dispatch packet */ 4683 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4684 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4685 ib.ptr[ib.length_dw++] = 1; /* y */ 4686 ib.ptr[ib.length_dw++] = 1; /* z */ 4687 ib.ptr[ib.length_dw++] = 4688 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4689 4690 /* write CS partial flush packet */ 4691 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4692 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4693 4694 /* SGPR1 */ 4695 /* write the register state for the compute dispatch */ 4696 for (i = 0; i < gpr_reg_size; i++) { 4697 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4698 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4699 - PACKET3_SET_SH_REG_START; 4700 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4701 } 4702 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4703 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4704 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4705 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4706 - PACKET3_SET_SH_REG_START; 4707 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4708 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4709 4710 /* write dispatch packet */ 4711 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4712 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4713 ib.ptr[ib.length_dw++] = 1; /* y */ 4714 ib.ptr[ib.length_dw++] = 1; /* z */ 4715 ib.ptr[ib.length_dw++] = 4716 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4717 4718 /* write CS partial flush packet */ 4719 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4720 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4721 4722 /* SGPR2 */ 4723 /* write the register state for the compute dispatch */ 4724 for (i = 0; i < gpr_reg_size; i++) { 4725 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4726 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4727 - PACKET3_SET_SH_REG_START; 4728 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4729 } 4730 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4731 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4732 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4733 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4734 - PACKET3_SET_SH_REG_START; 4735 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4736 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4737 4738 /* write dispatch packet */ 4739 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4740 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4741 ib.ptr[ib.length_dw++] = 1; /* y */ 4742 ib.ptr[ib.length_dw++] = 1; /* z */ 4743 ib.ptr[ib.length_dw++] = 4744 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4745 4746 /* write CS partial flush packet */ 4747 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4748 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4749 4750 /* shedule the ib on the ring */ 4751 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4752 if (r) { 4753 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4754 goto fail; 4755 } 4756 4757 /* wait for the GPU to finish processing the IB */ 4758 r = dma_fence_wait(f, false); 4759 if (r) { 4760 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4761 goto fail; 4762 } 4763 4764 fail: 4765 amdgpu_ib_free(adev, &ib, NULL); 4766 dma_fence_put(f); 4767 4768 return r; 4769 } 4770 4771 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block) 4772 { 4773 struct amdgpu_device *adev = ip_block->adev; 4774 4775 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 4776 4777 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || 4778 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4779 adev->gfx.num_gfx_rings = 0; 4780 else 4781 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4782 adev->gfx.xcc_mask = 1; 4783 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4784 AMDGPU_MAX_COMPUTE_RINGS); 4785 gfx_v9_0_set_kiq_pm4_funcs(adev); 4786 gfx_v9_0_set_ring_funcs(adev); 4787 gfx_v9_0_set_irq_funcs(adev); 4788 gfx_v9_0_set_gds_init(adev); 4789 gfx_v9_0_set_rlc_funcs(adev); 4790 4791 /* init rlcg reg access ctrl */ 4792 gfx_v9_0_init_rlcg_reg_access_ctrl(adev); 4793 4794 return gfx_v9_0_init_microcode(adev); 4795 } 4796 4797 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block) 4798 { 4799 struct amdgpu_device *adev = ip_block->adev; 4800 int r; 4801 4802 /* 4803 * Temp workaround to fix the issue that CP firmware fails to 4804 * update read pointer when CPDMA is writing clearing operation 4805 * to GDS in suspend/resume sequence on several cards. So just 4806 * limit this operation in cold boot sequence. 4807 */ 4808 if ((!adev->in_suspend) && 4809 (adev->gds.gds_size)) { 4810 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4811 if (r) 4812 return r; 4813 } 4814 4815 /* requires IBs so do in late init after IB pool is initialized */ 4816 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4817 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev); 4818 else 4819 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4820 4821 if (r) 4822 return r; 4823 4824 if (adev->gfx.ras && 4825 adev->gfx.ras->enable_watchdog_timer) 4826 adev->gfx.ras->enable_watchdog_timer(adev); 4827 4828 return 0; 4829 } 4830 4831 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block) 4832 { 4833 struct amdgpu_device *adev = ip_block->adev; 4834 int r; 4835 4836 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4837 if (r) 4838 return r; 4839 4840 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4841 if (r) 4842 return r; 4843 4844 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 4845 if (r) 4846 return r; 4847 4848 r = gfx_v9_0_ecc_late_init(ip_block); 4849 if (r) 4850 return r; 4851 4852 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4853 gfx_v9_4_2_debug_trap_config_init(adev, 4854 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); 4855 else 4856 gfx_v9_0_debug_trap_config_init(adev, 4857 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); 4858 4859 return 0; 4860 } 4861 4862 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4863 { 4864 uint32_t rlc_setting; 4865 4866 /* if RLC is not enabled, do nothing */ 4867 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4868 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4869 return false; 4870 4871 return true; 4872 } 4873 4874 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 4875 { 4876 uint32_t data; 4877 unsigned i; 4878 4879 data = RLC_SAFE_MODE__CMD_MASK; 4880 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4881 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4882 4883 /* wait for RLC_SAFE_MODE */ 4884 for (i = 0; i < adev->usec_timeout; i++) { 4885 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4886 break; 4887 udelay(1); 4888 } 4889 } 4890 4891 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 4892 { 4893 uint32_t data; 4894 4895 data = RLC_SAFE_MODE__CMD_MASK; 4896 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4897 } 4898 4899 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4900 bool enable) 4901 { 4902 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4903 4904 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4905 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4906 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4907 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4908 } else { 4909 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4910 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4911 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4912 } 4913 4914 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4915 } 4916 4917 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4918 bool enable) 4919 { 4920 /* TODO: double check if we need to perform under safe mode */ 4921 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4922 4923 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4924 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4925 else 4926 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4927 4928 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4929 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4930 else 4931 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4932 4933 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4934 } 4935 4936 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4937 bool enable) 4938 { 4939 uint32_t data, def; 4940 4941 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4942 4943 /* It is disabled by HW by default */ 4944 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4945 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4946 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4947 4948 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) 4949 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4950 4951 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4952 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4953 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4954 4955 /* only for Vega10 & Raven1 */ 4956 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4957 4958 if (def != data) 4959 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4960 4961 /* MGLS is a global flag to control all MGLS in GFX */ 4962 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4963 /* 2 - RLC memory Light sleep */ 4964 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4965 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4966 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4967 if (def != data) 4968 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4969 } 4970 /* 3 - CP memory Light sleep */ 4971 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4972 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4973 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4974 if (def != data) 4975 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4976 } 4977 } 4978 } else { 4979 /* 1 - MGCG_OVERRIDE */ 4980 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4981 4982 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) 4983 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4984 4985 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4986 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4987 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4988 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4989 4990 if (def != data) 4991 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4992 4993 /* 2 - disable MGLS in RLC */ 4994 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4995 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4996 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4997 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4998 } 4999 5000 /* 3 - disable MGLS in CP */ 5001 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 5002 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 5003 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5004 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 5005 } 5006 } 5007 5008 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5009 } 5010 5011 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 5012 bool enable) 5013 { 5014 uint32_t data, def; 5015 5016 if (!adev->gfx.num_gfx_rings) 5017 return; 5018 5019 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5020 5021 /* Enable 3D CGCG/CGLS */ 5022 if (enable) { 5023 /* write cmd to clear cgcg/cgls ov */ 5024 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5025 /* unset CGCG override */ 5026 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5027 /* update CGCG and CGLS override bits */ 5028 if (def != data) 5029 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5030 5031 /* enable 3Dcgcg FSM(0x0000363f) */ 5032 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 5033 5034 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5035 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5036 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5037 else 5038 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 5039 5040 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5041 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5042 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5043 if (def != data) 5044 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5045 5046 /* set IDLE_POLL_COUNT(0x00900100) */ 5047 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5048 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5049 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5050 if (def != data) 5051 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5052 } else { 5053 /* Disable CGCG/CGLS */ 5054 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 5055 /* disable cgcg, cgls should be disabled */ 5056 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 5057 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 5058 /* disable cgcg and cgls in FSM */ 5059 if (def != data) 5060 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5061 } 5062 5063 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5064 } 5065 5066 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5067 bool enable) 5068 { 5069 uint32_t def, data; 5070 5071 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5072 5073 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5074 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5075 /* unset CGCG override */ 5076 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5077 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5078 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5079 else 5080 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5081 /* update CGCG and CGLS override bits */ 5082 if (def != data) 5083 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5084 5085 /* enable cgcg FSM(0x0000363F) */ 5086 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5087 5088 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) 5089 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5090 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5091 else 5092 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5093 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5094 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5095 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5096 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5097 if (def != data) 5098 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5099 5100 /* set IDLE_POLL_COUNT(0x00900100) */ 5101 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5102 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5103 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5104 if (def != data) 5105 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5106 } else { 5107 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5108 /* reset CGCG/CGLS bits */ 5109 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5110 /* disable cgcg and cgls in FSM */ 5111 if (def != data) 5112 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5113 } 5114 5115 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5116 } 5117 5118 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5119 bool enable) 5120 { 5121 if (enable) { 5122 /* CGCG/CGLS should be enabled after MGCG/MGLS 5123 * === MGCG + MGLS === 5124 */ 5125 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5126 /* === CGCG /CGLS for GFX 3D Only === */ 5127 gfx_v9_0_update_3d_clock_gating(adev, enable); 5128 /* === CGCG + CGLS === */ 5129 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5130 } else { 5131 /* CGCG/CGLS should be disabled before MGCG/MGLS 5132 * === CGCG + CGLS === 5133 */ 5134 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5135 /* === CGCG /CGLS for GFX 3D Only === */ 5136 gfx_v9_0_update_3d_clock_gating(adev, enable); 5137 /* === MGCG + MGLS === */ 5138 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5139 } 5140 return 0; 5141 } 5142 5143 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, 5144 unsigned int vmid) 5145 { 5146 u32 reg, data; 5147 5148 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5149 if (amdgpu_sriov_is_pp_one_vf(adev)) 5150 data = RREG32_NO_KIQ(reg); 5151 else 5152 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL); 5153 5154 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5155 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5156 5157 if (amdgpu_sriov_is_pp_one_vf(adev)) 5158 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5159 else 5160 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5161 } 5162 5163 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid) 5164 { 5165 amdgpu_gfx_off_ctrl(adev, false); 5166 5167 gfx_v9_0_update_spm_vmid_internal(adev, vmid); 5168 5169 amdgpu_gfx_off_ctrl(adev, true); 5170 } 5171 5172 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5173 uint32_t offset, 5174 struct soc15_reg_rlcg *entries, int arr_size) 5175 { 5176 int i; 5177 uint32_t reg; 5178 5179 if (!entries) 5180 return false; 5181 5182 for (i = 0; i < arr_size; i++) { 5183 const struct soc15_reg_rlcg *entry; 5184 5185 entry = &entries[i]; 5186 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5187 if (offset == reg) 5188 return true; 5189 } 5190 5191 return false; 5192 } 5193 5194 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5195 { 5196 return gfx_v9_0_check_rlcg_range(adev, offset, 5197 (void *)rlcg_access_gc_9_0, 5198 ARRAY_SIZE(rlcg_access_gc_9_0)); 5199 } 5200 5201 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5202 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5203 .set_safe_mode = gfx_v9_0_set_safe_mode, 5204 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5205 .init = gfx_v9_0_rlc_init, 5206 .get_csb_size = gfx_v9_0_get_csb_size, 5207 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5208 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5209 .resume = gfx_v9_0_rlc_resume, 5210 .stop = gfx_v9_0_rlc_stop, 5211 .reset = gfx_v9_0_rlc_reset, 5212 .start = gfx_v9_0_rlc_start, 5213 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5214 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5215 }; 5216 5217 static int gfx_v9_0_set_powergating_state(void *handle, 5218 enum amd_powergating_state state) 5219 { 5220 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5221 bool enable = (state == AMD_PG_STATE_GATE); 5222 5223 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5224 case IP_VERSION(9, 2, 2): 5225 case IP_VERSION(9, 1, 0): 5226 case IP_VERSION(9, 3, 0): 5227 if (!enable) 5228 amdgpu_gfx_off_ctrl(adev, false); 5229 5230 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5231 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5232 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5233 } else { 5234 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5235 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5236 } 5237 5238 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5239 gfx_v9_0_enable_cp_power_gating(adev, true); 5240 else 5241 gfx_v9_0_enable_cp_power_gating(adev, false); 5242 5243 /* update gfx cgpg state */ 5244 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5245 5246 /* update mgcg state */ 5247 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5248 5249 if (enable) 5250 amdgpu_gfx_off_ctrl(adev, true); 5251 break; 5252 case IP_VERSION(9, 2, 1): 5253 amdgpu_gfx_off_ctrl(adev, enable); 5254 break; 5255 default: 5256 break; 5257 } 5258 5259 return 0; 5260 } 5261 5262 static int gfx_v9_0_set_clockgating_state(void *handle, 5263 enum amd_clockgating_state state) 5264 { 5265 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5266 5267 if (amdgpu_sriov_vf(adev)) 5268 return 0; 5269 5270 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5271 case IP_VERSION(9, 0, 1): 5272 case IP_VERSION(9, 2, 1): 5273 case IP_VERSION(9, 4, 0): 5274 case IP_VERSION(9, 2, 2): 5275 case IP_VERSION(9, 1, 0): 5276 case IP_VERSION(9, 4, 1): 5277 case IP_VERSION(9, 3, 0): 5278 case IP_VERSION(9, 4, 2): 5279 gfx_v9_0_update_gfx_clock_gating(adev, 5280 state == AMD_CG_STATE_GATE); 5281 break; 5282 default: 5283 break; 5284 } 5285 return 0; 5286 } 5287 5288 static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags) 5289 { 5290 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5291 int data; 5292 5293 if (amdgpu_sriov_vf(adev)) 5294 *flags = 0; 5295 5296 /* AMD_CG_SUPPORT_GFX_MGCG */ 5297 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5298 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5299 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5300 5301 /* AMD_CG_SUPPORT_GFX_CGCG */ 5302 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5303 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5304 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5305 5306 /* AMD_CG_SUPPORT_GFX_CGLS */ 5307 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5308 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5309 5310 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5311 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5312 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5313 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5314 5315 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5316 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5317 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5318 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5319 5320 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) { 5321 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5322 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5323 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5324 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5325 5326 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5327 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5328 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5329 } 5330 } 5331 5332 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5333 { 5334 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/ 5335 } 5336 5337 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5338 { 5339 struct amdgpu_device *adev = ring->adev; 5340 u64 wptr; 5341 5342 /* XXX check if swapping is necessary on BE */ 5343 if (ring->use_doorbell) { 5344 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5345 } else { 5346 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5347 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5348 } 5349 5350 return wptr; 5351 } 5352 5353 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5354 { 5355 struct amdgpu_device *adev = ring->adev; 5356 5357 if (ring->use_doorbell) { 5358 /* XXX check if swapping is necessary on BE */ 5359 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); 5360 WDOORBELL64(ring->doorbell_index, ring->wptr); 5361 } else { 5362 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5363 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5364 } 5365 } 5366 5367 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5368 { 5369 struct amdgpu_device *adev = ring->adev; 5370 u32 ref_and_mask, reg_mem_engine; 5371 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5372 5373 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5374 switch (ring->me) { 5375 case 1: 5376 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5377 break; 5378 case 2: 5379 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5380 break; 5381 default: 5382 return; 5383 } 5384 reg_mem_engine = 0; 5385 } else { 5386 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5387 reg_mem_engine = 1; /* pfp */ 5388 } 5389 5390 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5391 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5392 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5393 ref_and_mask, ref_and_mask, 0x20); 5394 } 5395 5396 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5397 struct amdgpu_job *job, 5398 struct amdgpu_ib *ib, 5399 uint32_t flags) 5400 { 5401 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5402 u32 header, control = 0; 5403 5404 if (ib->flags & AMDGPU_IB_FLAG_CE) 5405 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5406 else 5407 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5408 5409 control |= ib->length_dw | (vmid << 24); 5410 5411 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 5412 control |= INDIRECT_BUFFER_PRE_ENB(1); 5413 5414 if (flags & AMDGPU_IB_PREEMPTED) 5415 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5416 5417 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5418 gfx_v9_0_ring_emit_de_meta(ring, 5419 (!amdgpu_sriov_vf(ring->adev) && 5420 flags & AMDGPU_IB_PREEMPTED) ? 5421 true : false, 5422 job->gds_size > 0 && job->gds_base != 0); 5423 } 5424 5425 amdgpu_ring_write(ring, header); 5426 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5427 amdgpu_ring_write(ring, 5428 #ifdef __BIG_ENDIAN 5429 (2 << 0) | 5430 #endif 5431 lower_32_bits(ib->gpu_addr)); 5432 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5433 amdgpu_ring_ib_on_emit_cntl(ring); 5434 amdgpu_ring_write(ring, control); 5435 } 5436 5437 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring, 5438 unsigned offset) 5439 { 5440 u32 control = ring->ring[offset]; 5441 5442 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5443 ring->ring[offset] = control; 5444 } 5445 5446 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring, 5447 unsigned offset) 5448 { 5449 struct amdgpu_device *adev = ring->adev; 5450 void *ce_payload_cpu_addr; 5451 uint64_t payload_offset, payload_size; 5452 5453 payload_size = sizeof(struct v9_ce_ib_state); 5454 5455 if (ring->is_mes_queue) { 5456 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5457 gfx[0].gfx_meta_data) + 5458 offsetof(struct v9_gfx_meta_data, ce_payload); 5459 ce_payload_cpu_addr = 5460 amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset); 5461 } else { 5462 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload); 5463 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; 5464 } 5465 5466 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { 5467 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size); 5468 } else { 5469 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, 5470 (ring->buf_mask + 1 - offset) << 2); 5471 payload_size -= (ring->buf_mask + 1 - offset) << 2; 5472 memcpy((void *)&ring->ring[0], 5473 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), 5474 payload_size); 5475 } 5476 } 5477 5478 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring, 5479 unsigned offset) 5480 { 5481 struct amdgpu_device *adev = ring->adev; 5482 void *de_payload_cpu_addr; 5483 uint64_t payload_offset, payload_size; 5484 5485 payload_size = sizeof(struct v9_de_ib_state); 5486 5487 if (ring->is_mes_queue) { 5488 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5489 gfx[0].gfx_meta_data) + 5490 offsetof(struct v9_gfx_meta_data, de_payload); 5491 de_payload_cpu_addr = 5492 amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset); 5493 } else { 5494 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload); 5495 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; 5496 } 5497 5498 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status = 5499 IB_COMPLETION_STATUS_PREEMPTED; 5500 5501 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { 5502 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size); 5503 } else { 5504 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, 5505 (ring->buf_mask + 1 - offset) << 2); 5506 payload_size -= (ring->buf_mask + 1 - offset) << 2; 5507 memcpy((void *)&ring->ring[0], 5508 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), 5509 payload_size); 5510 } 5511 } 5512 5513 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5514 struct amdgpu_job *job, 5515 struct amdgpu_ib *ib, 5516 uint32_t flags) 5517 { 5518 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5519 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5520 5521 /* Currently, there is a high possibility to get wave ID mismatch 5522 * between ME and GDS, leading to a hw deadlock, because ME generates 5523 * different wave IDs than the GDS expects. This situation happens 5524 * randomly when at least 5 compute pipes use GDS ordered append. 5525 * The wave IDs generated by ME are also wrong after suspend/resume. 5526 * Those are probably bugs somewhere else in the kernel driver. 5527 * 5528 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5529 * GDS to 0 for this ring (me/pipe). 5530 */ 5531 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5532 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5533 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5534 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5535 } 5536 5537 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5538 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5539 amdgpu_ring_write(ring, 5540 #ifdef __BIG_ENDIAN 5541 (2 << 0) | 5542 #endif 5543 lower_32_bits(ib->gpu_addr)); 5544 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5545 amdgpu_ring_write(ring, control); 5546 } 5547 5548 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5549 u64 seq, unsigned flags) 5550 { 5551 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5552 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5553 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5554 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; 5555 uint32_t dw2 = 0; 5556 5557 /* RELEASE_MEM - flush caches, send int */ 5558 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5559 5560 if (writeback) { 5561 dw2 = EOP_TC_NC_ACTION_EN; 5562 } else { 5563 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN | 5564 EOP_TC_MD_ACTION_EN; 5565 } 5566 dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5567 EVENT_INDEX(5); 5568 if (exec) 5569 dw2 |= EOP_EXEC; 5570 5571 amdgpu_ring_write(ring, dw2); 5572 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5573 5574 /* 5575 * the address should be Qword aligned if 64bit write, Dword 5576 * aligned if only send 32bit data low (discard data high) 5577 */ 5578 if (write64bit) 5579 BUG_ON(addr & 0x7); 5580 else 5581 BUG_ON(addr & 0x3); 5582 amdgpu_ring_write(ring, lower_32_bits(addr)); 5583 amdgpu_ring_write(ring, upper_32_bits(addr)); 5584 amdgpu_ring_write(ring, lower_32_bits(seq)); 5585 amdgpu_ring_write(ring, upper_32_bits(seq)); 5586 amdgpu_ring_write(ring, 0); 5587 } 5588 5589 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5590 { 5591 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5592 uint32_t seq = ring->fence_drv.sync_seq; 5593 uint64_t addr = ring->fence_drv.gpu_addr; 5594 5595 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5596 lower_32_bits(addr), upper_32_bits(addr), 5597 seq, 0xffffffff, 4); 5598 } 5599 5600 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5601 unsigned vmid, uint64_t pd_addr) 5602 { 5603 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5604 5605 /* compute doesn't have PFP */ 5606 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5607 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5608 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5609 amdgpu_ring_write(ring, 0x0); 5610 } 5611 } 5612 5613 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5614 { 5615 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */ 5616 } 5617 5618 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5619 { 5620 u64 wptr; 5621 5622 /* XXX check if swapping is necessary on BE */ 5623 if (ring->use_doorbell) 5624 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5625 else 5626 BUG(); 5627 return wptr; 5628 } 5629 5630 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5631 { 5632 struct amdgpu_device *adev = ring->adev; 5633 5634 /* XXX check if swapping is necessary on BE */ 5635 if (ring->use_doorbell) { 5636 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); 5637 WDOORBELL64(ring->doorbell_index, ring->wptr); 5638 } else{ 5639 BUG(); /* only DOORBELL method supported on gfx9 now */ 5640 } 5641 } 5642 5643 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5644 u64 seq, unsigned int flags) 5645 { 5646 struct amdgpu_device *adev = ring->adev; 5647 5648 /* we only allocate 32bit for each seq wb address */ 5649 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5650 5651 /* write fence seq to the "addr" */ 5652 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5653 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5654 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5655 amdgpu_ring_write(ring, lower_32_bits(addr)); 5656 amdgpu_ring_write(ring, upper_32_bits(addr)); 5657 amdgpu_ring_write(ring, lower_32_bits(seq)); 5658 5659 if (flags & AMDGPU_FENCE_FLAG_INT) { 5660 /* set register to trigger INT */ 5661 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5662 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5663 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5664 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5665 amdgpu_ring_write(ring, 0); 5666 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5667 } 5668 } 5669 5670 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5671 { 5672 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5673 amdgpu_ring_write(ring, 0); 5674 } 5675 5676 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume) 5677 { 5678 struct amdgpu_device *adev = ring->adev; 5679 struct v9_ce_ib_state ce_payload = {0}; 5680 uint64_t offset, ce_payload_gpu_addr; 5681 void *ce_payload_cpu_addr; 5682 int cnt; 5683 5684 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5685 5686 if (ring->is_mes_queue) { 5687 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5688 gfx[0].gfx_meta_data) + 5689 offsetof(struct v9_gfx_meta_data, ce_payload); 5690 ce_payload_gpu_addr = 5691 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 5692 ce_payload_cpu_addr = 5693 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); 5694 } else { 5695 offset = offsetof(struct v9_gfx_meta_data, ce_payload); 5696 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 5697 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 5698 } 5699 5700 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5701 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5702 WRITE_DATA_DST_SEL(8) | 5703 WR_CONFIRM) | 5704 WRITE_DATA_CACHE_POLICY(0)); 5705 amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr)); 5706 amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr)); 5707 5708 amdgpu_ring_ib_on_emit_ce(ring); 5709 5710 if (resume) 5711 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr, 5712 sizeof(ce_payload) >> 2); 5713 else 5714 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, 5715 sizeof(ce_payload) >> 2); 5716 } 5717 5718 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring) 5719 { 5720 int i, r = 0; 5721 struct amdgpu_device *adev = ring->adev; 5722 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 5723 struct amdgpu_ring *kiq_ring = &kiq->ring; 5724 unsigned long flags; 5725 5726 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 5727 return -EINVAL; 5728 5729 spin_lock_irqsave(&kiq->ring_lock, flags); 5730 5731 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { 5732 spin_unlock_irqrestore(&kiq->ring_lock, flags); 5733 return -ENOMEM; 5734 } 5735 5736 /* assert preemption condition */ 5737 amdgpu_ring_set_preempt_cond_exec(ring, false); 5738 5739 ring->trail_seq += 1; 5740 amdgpu_ring_alloc(ring, 13); 5741 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, 5742 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT); 5743 5744 /* assert IB preemption, emit the trailing fence */ 5745 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, 5746 ring->trail_fence_gpu_addr, 5747 ring->trail_seq); 5748 5749 amdgpu_ring_commit(kiq_ring); 5750 spin_unlock_irqrestore(&kiq->ring_lock, flags); 5751 5752 /* poll the trailing fence */ 5753 for (i = 0; i < adev->usec_timeout; i++) { 5754 if (ring->trail_seq == 5755 le32_to_cpu(*ring->trail_fence_cpu_addr)) 5756 break; 5757 udelay(1); 5758 } 5759 5760 if (i >= adev->usec_timeout) { 5761 r = -EINVAL; 5762 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx); 5763 } 5764 5765 /*reset the CP_VMID_PREEMPT after trailing fence*/ 5766 amdgpu_ring_emit_wreg(ring, 5767 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT), 5768 0x0); 5769 amdgpu_ring_commit(ring); 5770 5771 /* deassert preemption condition */ 5772 amdgpu_ring_set_preempt_cond_exec(ring, true); 5773 return r; 5774 } 5775 5776 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds) 5777 { 5778 struct amdgpu_device *adev = ring->adev; 5779 struct v9_de_ib_state de_payload = {0}; 5780 uint64_t offset, gds_addr, de_payload_gpu_addr; 5781 void *de_payload_cpu_addr; 5782 int cnt; 5783 5784 if (ring->is_mes_queue) { 5785 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5786 gfx[0].gfx_meta_data) + 5787 offsetof(struct v9_gfx_meta_data, de_payload); 5788 de_payload_gpu_addr = 5789 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 5790 de_payload_cpu_addr = 5791 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); 5792 5793 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5794 gfx[0].gds_backup) + 5795 offsetof(struct v9_gfx_meta_data, de_payload); 5796 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 5797 } else { 5798 offset = offsetof(struct v9_gfx_meta_data, de_payload); 5799 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 5800 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 5801 5802 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 5803 AMDGPU_CSA_SIZE - adev->gds.gds_size, 5804 PAGE_SIZE); 5805 } 5806 5807 if (usegds) { 5808 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5809 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5810 } 5811 5812 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5813 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5814 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5815 WRITE_DATA_DST_SEL(8) | 5816 WR_CONFIRM) | 5817 WRITE_DATA_CACHE_POLICY(0)); 5818 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 5819 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 5820 5821 amdgpu_ring_ib_on_emit_de(ring); 5822 if (resume) 5823 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 5824 sizeof(de_payload) >> 2); 5825 else 5826 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 5827 sizeof(de_payload) >> 2); 5828 } 5829 5830 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5831 bool secure) 5832 { 5833 uint32_t v = secure ? FRAME_TMZ : 0; 5834 5835 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5836 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5837 } 5838 5839 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5840 { 5841 uint32_t dw2 = 0; 5842 5843 gfx_v9_0_ring_emit_ce_meta(ring, 5844 (!amdgpu_sriov_vf(ring->adev) && 5845 flags & AMDGPU_IB_PREEMPTED) ? true : false); 5846 5847 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5848 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5849 /* set load_global_config & load_global_uconfig */ 5850 dw2 |= 0x8001; 5851 /* set load_cs_sh_regs */ 5852 dw2 |= 0x01000000; 5853 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5854 dw2 |= 0x10002; 5855 5856 /* set load_ce_ram if preamble presented */ 5857 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5858 dw2 |= 0x10000000; 5859 } else { 5860 /* still load_ce_ram if this is the first time preamble presented 5861 * although there is no context switch happens. 5862 */ 5863 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5864 dw2 |= 0x10000000; 5865 } 5866 5867 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5868 amdgpu_ring_write(ring, dw2); 5869 amdgpu_ring_write(ring, 0); 5870 } 5871 5872 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 5873 uint64_t addr) 5874 { 5875 unsigned ret; 5876 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5877 amdgpu_ring_write(ring, lower_32_bits(addr)); 5878 amdgpu_ring_write(ring, upper_32_bits(addr)); 5879 /* discard following DWs if *cond_exec_gpu_addr==0 */ 5880 amdgpu_ring_write(ring, 0); 5881 ret = ring->wptr & ring->buf_mask; 5882 /* patch dummy value later */ 5883 amdgpu_ring_write(ring, 0); 5884 return ret; 5885 } 5886 5887 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5888 uint32_t reg_val_offs) 5889 { 5890 struct amdgpu_device *adev = ring->adev; 5891 5892 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5893 amdgpu_ring_write(ring, 0 | /* src: register*/ 5894 (5 << 8) | /* dst: memory */ 5895 (1 << 20)); /* write confirm */ 5896 amdgpu_ring_write(ring, reg); 5897 amdgpu_ring_write(ring, 0); 5898 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5899 reg_val_offs * 4)); 5900 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5901 reg_val_offs * 4)); 5902 } 5903 5904 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5905 uint32_t val) 5906 { 5907 uint32_t cmd = 0; 5908 5909 switch (ring->funcs->type) { 5910 case AMDGPU_RING_TYPE_GFX: 5911 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5912 break; 5913 case AMDGPU_RING_TYPE_KIQ: 5914 cmd = (1 << 16); /* no inc addr */ 5915 break; 5916 default: 5917 cmd = WR_CONFIRM; 5918 break; 5919 } 5920 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5921 amdgpu_ring_write(ring, cmd); 5922 amdgpu_ring_write(ring, reg); 5923 amdgpu_ring_write(ring, 0); 5924 amdgpu_ring_write(ring, val); 5925 } 5926 5927 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5928 uint32_t val, uint32_t mask) 5929 { 5930 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5931 } 5932 5933 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5934 uint32_t reg0, uint32_t reg1, 5935 uint32_t ref, uint32_t mask) 5936 { 5937 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5938 struct amdgpu_device *adev = ring->adev; 5939 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5940 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5941 5942 if (fw_version_ok) 5943 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5944 ref, mask, 0x20); 5945 else 5946 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5947 ref, mask); 5948 } 5949 5950 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5951 { 5952 struct amdgpu_device *adev = ring->adev; 5953 uint32_t value = 0; 5954 5955 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5956 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5957 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5958 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5959 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5960 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5961 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5962 } 5963 5964 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5965 enum amdgpu_interrupt_state state) 5966 { 5967 switch (state) { 5968 case AMDGPU_IRQ_STATE_DISABLE: 5969 case AMDGPU_IRQ_STATE_ENABLE: 5970 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5971 TIME_STAMP_INT_ENABLE, 5972 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5973 break; 5974 default: 5975 break; 5976 } 5977 } 5978 5979 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5980 int me, int pipe, 5981 enum amdgpu_interrupt_state state) 5982 { 5983 u32 mec_int_cntl, mec_int_cntl_reg; 5984 5985 /* 5986 * amdgpu controls only the first MEC. That's why this function only 5987 * handles the setting of interrupts for this specific MEC. All other 5988 * pipes' interrupts are set by amdkfd. 5989 */ 5990 5991 if (me == 1) { 5992 switch (pipe) { 5993 case 0: 5994 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5995 break; 5996 case 1: 5997 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5998 break; 5999 case 2: 6000 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 6001 break; 6002 case 3: 6003 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 6004 break; 6005 default: 6006 DRM_DEBUG("invalid pipe %d\n", pipe); 6007 return; 6008 } 6009 } else { 6010 DRM_DEBUG("invalid me %d\n", me); 6011 return; 6012 } 6013 6014 switch (state) { 6015 case AMDGPU_IRQ_STATE_DISABLE: 6016 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg); 6017 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6018 TIME_STAMP_INT_ENABLE, 0); 6019 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6020 break; 6021 case AMDGPU_IRQ_STATE_ENABLE: 6022 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6023 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6024 TIME_STAMP_INT_ENABLE, 1); 6025 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6026 break; 6027 default: 6028 break; 6029 } 6030 } 6031 6032 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev, 6033 int me, int pipe) 6034 { 6035 /* 6036 * amdgpu controls only the first MEC. That's why this function only 6037 * handles the setting of interrupts for this specific MEC. All other 6038 * pipes' interrupts are set by amdkfd. 6039 */ 6040 if (me != 1) 6041 return 0; 6042 6043 switch (pipe) { 6044 case 0: 6045 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 6046 case 1: 6047 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 6048 case 2: 6049 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 6050 case 3: 6051 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 6052 default: 6053 return 0; 6054 } 6055 } 6056 6057 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6058 struct amdgpu_irq_src *source, 6059 unsigned type, 6060 enum amdgpu_interrupt_state state) 6061 { 6062 u32 cp_int_cntl_reg, cp_int_cntl; 6063 int i, j; 6064 6065 switch (state) { 6066 case AMDGPU_IRQ_STATE_DISABLE: 6067 case AMDGPU_IRQ_STATE_ENABLE: 6068 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6069 PRIV_REG_INT_ENABLE, 6070 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6071 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6072 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6073 /* MECs start at 1 */ 6074 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j); 6075 6076 if (cp_int_cntl_reg) { 6077 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6078 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6079 PRIV_REG_INT_ENABLE, 6080 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6081 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6082 } 6083 } 6084 } 6085 break; 6086 default: 6087 break; 6088 } 6089 6090 return 0; 6091 } 6092 6093 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6094 struct amdgpu_irq_src *source, 6095 unsigned type, 6096 enum amdgpu_interrupt_state state) 6097 { 6098 u32 cp_int_cntl_reg, cp_int_cntl; 6099 int i, j; 6100 6101 switch (state) { 6102 case AMDGPU_IRQ_STATE_DISABLE: 6103 case AMDGPU_IRQ_STATE_ENABLE: 6104 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6105 OPCODE_ERROR_INT_ENABLE, 6106 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6107 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6108 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6109 /* MECs start at 1 */ 6110 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j); 6111 6112 if (cp_int_cntl_reg) { 6113 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6114 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6115 OPCODE_ERROR_INT_ENABLE, 6116 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6117 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6118 } 6119 } 6120 } 6121 break; 6122 default: 6123 break; 6124 } 6125 6126 return 0; 6127 } 6128 6129 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6130 struct amdgpu_irq_src *source, 6131 unsigned type, 6132 enum amdgpu_interrupt_state state) 6133 { 6134 switch (state) { 6135 case AMDGPU_IRQ_STATE_DISABLE: 6136 case AMDGPU_IRQ_STATE_ENABLE: 6137 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6138 PRIV_INSTR_INT_ENABLE, 6139 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6140 break; 6141 default: 6142 break; 6143 } 6144 6145 return 0; 6146 } 6147 6148 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 6149 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 6150 CP_ECC_ERROR_INT_ENABLE, 1) 6151 6152 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 6153 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 6154 CP_ECC_ERROR_INT_ENABLE, 0) 6155 6156 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 6157 struct amdgpu_irq_src *source, 6158 unsigned type, 6159 enum amdgpu_interrupt_state state) 6160 { 6161 switch (state) { 6162 case AMDGPU_IRQ_STATE_DISABLE: 6163 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6164 CP_ECC_ERROR_INT_ENABLE, 0); 6165 DISABLE_ECC_ON_ME_PIPE(1, 0); 6166 DISABLE_ECC_ON_ME_PIPE(1, 1); 6167 DISABLE_ECC_ON_ME_PIPE(1, 2); 6168 DISABLE_ECC_ON_ME_PIPE(1, 3); 6169 break; 6170 6171 case AMDGPU_IRQ_STATE_ENABLE: 6172 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6173 CP_ECC_ERROR_INT_ENABLE, 1); 6174 ENABLE_ECC_ON_ME_PIPE(1, 0); 6175 ENABLE_ECC_ON_ME_PIPE(1, 1); 6176 ENABLE_ECC_ON_ME_PIPE(1, 2); 6177 ENABLE_ECC_ON_ME_PIPE(1, 3); 6178 break; 6179 default: 6180 break; 6181 } 6182 6183 return 0; 6184 } 6185 6186 6187 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6188 struct amdgpu_irq_src *src, 6189 unsigned type, 6190 enum amdgpu_interrupt_state state) 6191 { 6192 switch (type) { 6193 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6194 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 6195 break; 6196 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6197 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6198 break; 6199 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6200 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6201 break; 6202 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6203 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6204 break; 6205 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6206 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6207 break; 6208 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 6209 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 6210 break; 6211 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 6212 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 6213 break; 6214 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 6215 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 6216 break; 6217 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 6218 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 6219 break; 6220 default: 6221 break; 6222 } 6223 return 0; 6224 } 6225 6226 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 6227 struct amdgpu_irq_src *source, 6228 struct amdgpu_iv_entry *entry) 6229 { 6230 int i; 6231 u8 me_id, pipe_id, queue_id; 6232 struct amdgpu_ring *ring; 6233 6234 DRM_DEBUG("IH: CP EOP\n"); 6235 me_id = (entry->ring_id & 0x0c) >> 2; 6236 pipe_id = (entry->ring_id & 0x03) >> 0; 6237 queue_id = (entry->ring_id & 0x70) >> 4; 6238 6239 switch (me_id) { 6240 case 0: 6241 if (adev->gfx.num_gfx_rings) { 6242 if (!adev->gfx.mcbp) { 6243 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6244 } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) { 6245 /* Fence signals are handled on the software rings*/ 6246 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 6247 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]); 6248 } 6249 } 6250 break; 6251 case 1: 6252 case 2: 6253 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6254 ring = &adev->gfx.compute_ring[i]; 6255 /* Per-queue interrupt is supported for MEC starting from VI. 6256 * The interrupt can only be enabled/disabled per pipe instead of per queue. 6257 */ 6258 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 6259 amdgpu_fence_process(ring); 6260 } 6261 break; 6262 } 6263 return 0; 6264 } 6265 6266 static void gfx_v9_0_fault(struct amdgpu_device *adev, 6267 struct amdgpu_iv_entry *entry) 6268 { 6269 u8 me_id, pipe_id, queue_id; 6270 struct amdgpu_ring *ring; 6271 int i; 6272 6273 me_id = (entry->ring_id & 0x0c) >> 2; 6274 pipe_id = (entry->ring_id & 0x03) >> 0; 6275 queue_id = (entry->ring_id & 0x70) >> 4; 6276 6277 switch (me_id) { 6278 case 0: 6279 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 6280 break; 6281 case 1: 6282 case 2: 6283 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6284 ring = &adev->gfx.compute_ring[i]; 6285 if (ring->me == me_id && ring->pipe == pipe_id && 6286 ring->queue == queue_id) 6287 drm_sched_fault(&ring->sched); 6288 } 6289 break; 6290 } 6291 } 6292 6293 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 6294 struct amdgpu_irq_src *source, 6295 struct amdgpu_iv_entry *entry) 6296 { 6297 DRM_ERROR("Illegal register access in command stream\n"); 6298 gfx_v9_0_fault(adev, entry); 6299 return 0; 6300 } 6301 6302 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev, 6303 struct amdgpu_irq_src *source, 6304 struct amdgpu_iv_entry *entry) 6305 { 6306 DRM_ERROR("Illegal opcode in command stream\n"); 6307 gfx_v9_0_fault(adev, entry); 6308 return 0; 6309 } 6310 6311 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 6312 struct amdgpu_irq_src *source, 6313 struct amdgpu_iv_entry *entry) 6314 { 6315 DRM_ERROR("Illegal instruction in command stream\n"); 6316 gfx_v9_0_fault(adev, entry); 6317 return 0; 6318 } 6319 6320 6321 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 6322 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 6323 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 6324 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 6325 }, 6326 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 6327 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 6328 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 6329 }, 6330 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 6331 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 6332 0, 0 6333 }, 6334 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 6335 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 6336 0, 0 6337 }, 6338 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 6339 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 6340 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 6341 }, 6342 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6343 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 6344 0, 0 6345 }, 6346 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6347 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 6348 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 6349 }, 6350 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 6351 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 6352 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 6353 }, 6354 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 6355 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 6356 0, 0 6357 }, 6358 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 6359 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 6360 0, 0 6361 }, 6362 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 6363 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 6364 0, 0 6365 }, 6366 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6367 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 6368 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 6369 }, 6370 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6371 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 6372 0, 0 6373 }, 6374 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6375 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 6376 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 6377 }, 6378 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6379 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6380 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6381 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6382 }, 6383 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6384 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6385 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6386 0, 0 6387 }, 6388 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6389 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6390 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6391 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6392 }, 6393 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6394 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6395 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6396 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6397 }, 6398 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6399 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6400 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6401 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6402 }, 6403 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6404 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6405 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6406 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6407 }, 6408 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6409 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6410 0, 0 6411 }, 6412 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6413 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6414 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6415 }, 6416 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6417 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6418 0, 0 6419 }, 6420 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6421 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6422 0, 0 6423 }, 6424 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6425 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6426 0, 0 6427 }, 6428 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6429 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6430 0, 0 6431 }, 6432 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6433 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6434 0, 0 6435 }, 6436 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6437 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6438 0, 0 6439 }, 6440 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6441 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6442 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6443 }, 6444 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6445 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6446 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6447 }, 6448 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6449 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6450 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6451 }, 6452 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6453 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6454 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6455 }, 6456 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6457 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6458 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6459 }, 6460 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6461 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6462 0, 0 6463 }, 6464 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6465 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6466 0, 0 6467 }, 6468 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6469 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6470 0, 0 6471 }, 6472 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6473 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6474 0, 0 6475 }, 6476 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6477 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6478 0, 0 6479 }, 6480 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6481 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6482 0, 0 6483 }, 6484 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6485 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6486 0, 0 6487 }, 6488 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6489 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6490 0, 0 6491 }, 6492 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6493 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6494 0, 0 6495 }, 6496 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6497 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6498 0, 0 6499 }, 6500 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6501 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6502 0, 0 6503 }, 6504 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6505 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6506 0, 0 6507 }, 6508 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6509 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6510 0, 0 6511 }, 6512 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6513 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6514 0, 0 6515 }, 6516 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6517 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6518 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6519 }, 6520 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6521 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6522 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6523 }, 6524 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6525 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6526 0, 0 6527 }, 6528 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6529 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6530 0, 0 6531 }, 6532 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6533 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6534 0, 0 6535 }, 6536 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6537 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6538 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6539 }, 6540 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6541 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6542 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6543 }, 6544 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6545 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6546 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6547 }, 6548 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6549 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6550 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6551 }, 6552 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6553 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6554 0, 0 6555 }, 6556 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6557 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6558 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6559 }, 6560 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6561 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6562 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6563 }, 6564 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6565 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6566 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6567 }, 6568 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6569 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6570 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6571 }, 6572 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6573 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6574 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6575 }, 6576 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6577 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6578 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6579 }, 6580 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6581 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6582 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6583 }, 6584 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6585 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6586 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6587 }, 6588 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6589 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6590 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6591 }, 6592 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6593 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6594 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6595 }, 6596 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6597 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6598 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6599 }, 6600 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6601 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6602 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6603 }, 6604 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6605 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6606 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6607 }, 6608 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6609 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6610 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6611 }, 6612 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6613 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6614 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6615 }, 6616 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6617 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6618 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6619 }, 6620 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6621 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6622 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6623 }, 6624 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6625 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6626 0, 0 6627 }, 6628 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6629 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6630 0, 0 6631 }, 6632 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6633 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6634 0, 0 6635 }, 6636 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6637 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6638 0, 0 6639 }, 6640 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6641 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6642 0, 0 6643 }, 6644 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6645 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6646 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6647 }, 6648 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6649 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6650 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6651 }, 6652 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6653 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6654 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6655 }, 6656 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6657 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6658 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6659 }, 6660 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6661 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6662 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6663 }, 6664 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6665 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6666 0, 0 6667 }, 6668 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6669 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6670 0, 0 6671 }, 6672 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6673 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6674 0, 0 6675 }, 6676 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6677 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6678 0, 0 6679 }, 6680 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6681 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6682 0, 0 6683 }, 6684 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6685 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6686 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6687 }, 6688 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6689 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6690 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6691 }, 6692 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6693 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6694 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6695 }, 6696 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6697 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6698 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6699 }, 6700 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6701 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6702 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6703 }, 6704 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6705 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6706 0, 0 6707 }, 6708 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6709 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6710 0, 0 6711 }, 6712 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6713 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6714 0, 0 6715 }, 6716 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6717 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6718 0, 0 6719 }, 6720 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6721 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6722 0, 0 6723 }, 6724 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6725 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6726 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6727 }, 6728 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6729 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6730 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6731 }, 6732 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6733 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6734 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6735 }, 6736 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6737 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6738 0, 0 6739 }, 6740 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6741 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6742 0, 0 6743 }, 6744 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6745 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6746 0, 0 6747 }, 6748 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6749 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6750 0, 0 6751 }, 6752 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6753 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6754 0, 0 6755 }, 6756 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6757 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6758 0, 0 6759 } 6760 }; 6761 6762 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6763 void *inject_if, uint32_t instance_mask) 6764 { 6765 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6766 int ret; 6767 struct ta_ras_trigger_error_input block_info = { 0 }; 6768 6769 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6770 return -EINVAL; 6771 6772 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6773 return -EINVAL; 6774 6775 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6776 return -EPERM; 6777 6778 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6779 info->head.type)) { 6780 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6781 ras_gfx_subblocks[info->head.sub_block_index].name, 6782 info->head.type); 6783 return -EPERM; 6784 } 6785 6786 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6787 info->head.type)) { 6788 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6789 ras_gfx_subblocks[info->head.sub_block_index].name, 6790 info->head.type); 6791 return -EPERM; 6792 } 6793 6794 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6795 block_info.sub_block_index = 6796 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6797 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6798 block_info.address = info->address; 6799 block_info.value = info->value; 6800 6801 mutex_lock(&adev->grbm_idx_mutex); 6802 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask); 6803 mutex_unlock(&adev->grbm_idx_mutex); 6804 6805 return ret; 6806 } 6807 6808 static const char * const vml2_mems[] = { 6809 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6810 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6811 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6812 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6813 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6814 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6815 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6816 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6817 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6818 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6819 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6820 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6821 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6822 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6823 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6824 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6825 }; 6826 6827 static const char * const vml2_walker_mems[] = { 6828 "UTC_VML2_CACHE_PDE0_MEM0", 6829 "UTC_VML2_CACHE_PDE0_MEM1", 6830 "UTC_VML2_CACHE_PDE1_MEM0", 6831 "UTC_VML2_CACHE_PDE1_MEM1", 6832 "UTC_VML2_CACHE_PDE2_MEM0", 6833 "UTC_VML2_CACHE_PDE2_MEM1", 6834 "UTC_VML2_RDIF_LOG_FIFO", 6835 }; 6836 6837 static const char * const atc_l2_cache_2m_mems[] = { 6838 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6839 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6840 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6841 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6842 }; 6843 6844 static const char *atc_l2_cache_4k_mems[] = { 6845 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6846 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6847 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6848 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6849 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6850 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6851 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6852 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6853 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6854 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6855 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6856 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6857 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6858 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6859 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6860 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6861 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6862 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6863 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6864 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6865 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6866 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6867 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6868 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6869 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6870 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6871 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6872 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6873 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6874 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6875 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6876 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6877 }; 6878 6879 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6880 struct ras_err_data *err_data) 6881 { 6882 uint32_t i, data; 6883 uint32_t sec_count, ded_count; 6884 6885 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6886 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6887 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6888 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6889 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6890 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6891 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6892 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6893 6894 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6895 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6896 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6897 6898 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6899 if (sec_count) { 6900 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6901 "SEC %d\n", i, vml2_mems[i], sec_count); 6902 err_data->ce_count += sec_count; 6903 } 6904 6905 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6906 if (ded_count) { 6907 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6908 "DED %d\n", i, vml2_mems[i], ded_count); 6909 err_data->ue_count += ded_count; 6910 } 6911 } 6912 6913 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6914 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6915 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6916 6917 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6918 SEC_COUNT); 6919 if (sec_count) { 6920 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6921 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6922 err_data->ce_count += sec_count; 6923 } 6924 6925 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6926 DED_COUNT); 6927 if (ded_count) { 6928 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6929 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6930 err_data->ue_count += ded_count; 6931 } 6932 } 6933 6934 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6935 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6936 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6937 6938 sec_count = (data & 0x00006000L) >> 0xd; 6939 if (sec_count) { 6940 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6941 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6942 sec_count); 6943 err_data->ce_count += sec_count; 6944 } 6945 } 6946 6947 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6948 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6949 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6950 6951 sec_count = (data & 0x00006000L) >> 0xd; 6952 if (sec_count) { 6953 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6954 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6955 sec_count); 6956 err_data->ce_count += sec_count; 6957 } 6958 6959 ded_count = (data & 0x00018000L) >> 0xf; 6960 if (ded_count) { 6961 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6962 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6963 ded_count); 6964 err_data->ue_count += ded_count; 6965 } 6966 } 6967 6968 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6969 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6970 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6971 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6972 6973 return 0; 6974 } 6975 6976 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6977 const struct soc15_reg_entry *reg, 6978 uint32_t se_id, uint32_t inst_id, uint32_t value, 6979 uint32_t *sec_count, uint32_t *ded_count) 6980 { 6981 uint32_t i; 6982 uint32_t sec_cnt, ded_cnt; 6983 6984 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6985 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6986 gfx_v9_0_ras_fields[i].seg != reg->seg || 6987 gfx_v9_0_ras_fields[i].inst != reg->inst) 6988 continue; 6989 6990 sec_cnt = (value & 6991 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6992 gfx_v9_0_ras_fields[i].sec_count_shift; 6993 if (sec_cnt) { 6994 dev_info(adev->dev, "GFX SubBlock %s, " 6995 "Instance[%d][%d], SEC %d\n", 6996 gfx_v9_0_ras_fields[i].name, 6997 se_id, inst_id, 6998 sec_cnt); 6999 *sec_count += sec_cnt; 7000 } 7001 7002 ded_cnt = (value & 7003 gfx_v9_0_ras_fields[i].ded_count_mask) >> 7004 gfx_v9_0_ras_fields[i].ded_count_shift; 7005 if (ded_cnt) { 7006 dev_info(adev->dev, "GFX SubBlock %s, " 7007 "Instance[%d][%d], DED %d\n", 7008 gfx_v9_0_ras_fields[i].name, 7009 se_id, inst_id, 7010 ded_cnt); 7011 *ded_count += ded_cnt; 7012 } 7013 } 7014 7015 return 0; 7016 } 7017 7018 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 7019 { 7020 int i, j, k; 7021 7022 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 7023 return; 7024 7025 /* read back registers to clear the counters */ 7026 mutex_lock(&adev->grbm_idx_mutex); 7027 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 7028 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 7029 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 7030 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0); 7031 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 7032 } 7033 } 7034 } 7035 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 7036 mutex_unlock(&adev->grbm_idx_mutex); 7037 7038 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 7039 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 7040 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 7041 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 7042 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 7043 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 7044 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 7045 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 7046 7047 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 7048 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 7049 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 7050 } 7051 7052 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 7053 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 7054 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 7055 } 7056 7057 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 7058 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 7059 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 7060 } 7061 7062 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 7063 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 7064 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 7065 } 7066 7067 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 7068 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 7069 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 7070 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 7071 } 7072 7073 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 7074 void *ras_error_status) 7075 { 7076 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 7077 uint32_t sec_count = 0, ded_count = 0; 7078 uint32_t i, j, k; 7079 uint32_t reg_value; 7080 7081 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 7082 return; 7083 7084 err_data->ue_count = 0; 7085 err_data->ce_count = 0; 7086 7087 mutex_lock(&adev->grbm_idx_mutex); 7088 7089 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 7090 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 7091 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 7092 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0); 7093 reg_value = 7094 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 7095 if (reg_value) 7096 gfx_v9_0_ras_error_count(adev, 7097 &gfx_v9_0_edc_counter_regs[i], 7098 j, k, reg_value, 7099 &sec_count, &ded_count); 7100 } 7101 } 7102 } 7103 7104 err_data->ce_count += sec_count; 7105 err_data->ue_count += ded_count; 7106 7107 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7108 mutex_unlock(&adev->grbm_idx_mutex); 7109 7110 gfx_v9_0_query_utc_edc_status(adev, err_data); 7111 } 7112 7113 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 7114 { 7115 const unsigned int cp_coher_cntl = 7116 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 7117 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 7118 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 7119 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 7120 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 7121 7122 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 7123 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 7124 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 7125 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 7126 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 7127 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 7128 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 7129 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 7130 } 7131 7132 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 7133 uint32_t pipe, bool enable) 7134 { 7135 struct amdgpu_device *adev = ring->adev; 7136 uint32_t val; 7137 uint32_t wcl_cs_reg; 7138 7139 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 7140 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 7141 7142 switch (pipe) { 7143 case 0: 7144 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 7145 break; 7146 case 1: 7147 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 7148 break; 7149 case 2: 7150 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 7151 break; 7152 case 3: 7153 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 7154 break; 7155 default: 7156 DRM_DEBUG("invalid pipe %d\n", pipe); 7157 return; 7158 } 7159 7160 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 7161 7162 } 7163 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 7164 { 7165 struct amdgpu_device *adev = ring->adev; 7166 uint32_t val; 7167 int i; 7168 7169 7170 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 7171 * number of gfx waves. Setting 5 bit will make sure gfx only gets 7172 * around 25% of gpu resources. 7173 */ 7174 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 7175 amdgpu_ring_emit_wreg(ring, 7176 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 7177 val); 7178 7179 /* Restrict waves for normal/low priority compute queues as well 7180 * to get best QoS for high priority compute jobs. 7181 * 7182 * amdgpu controls only 1st ME(0-3 CS pipes). 7183 */ 7184 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 7185 if (i != ring->pipe) 7186 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 7187 7188 } 7189 } 7190 7191 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 7192 { 7193 /* Header itself is a NOP packet */ 7194 if (num_nop == 1) { 7195 amdgpu_ring_write(ring, ring->funcs->nop); 7196 return; 7197 } 7198 7199 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 7200 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 7201 7202 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 7203 amdgpu_ring_insert_nop(ring, num_nop - 1); 7204 } 7205 7206 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid) 7207 { 7208 struct amdgpu_device *adev = ring->adev; 7209 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 7210 struct amdgpu_ring *kiq_ring = &kiq->ring; 7211 unsigned long flags; 7212 u32 tmp; 7213 int r; 7214 7215 if (amdgpu_sriov_vf(adev)) 7216 return -EINVAL; 7217 7218 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 7219 return -EINVAL; 7220 7221 spin_lock_irqsave(&kiq->ring_lock, flags); 7222 7223 if (amdgpu_ring_alloc(kiq_ring, 5)) { 7224 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7225 return -ENOMEM; 7226 } 7227 7228 tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid); 7229 gfx_v9_0_ring_emit_wreg(kiq_ring, 7230 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp); 7231 amdgpu_ring_commit(kiq_ring); 7232 7233 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7234 7235 r = amdgpu_ring_test_ring(kiq_ring); 7236 if (r) 7237 return r; 7238 7239 if (amdgpu_ring_alloc(ring, 7 + 7 + 5)) 7240 return -ENOMEM; 7241 gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr, 7242 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC); 7243 gfx_v9_0_ring_emit_reg_wait(ring, 7244 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0, 0xffff); 7245 gfx_v9_0_ring_emit_wreg(ring, 7246 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0); 7247 7248 return amdgpu_ring_test_ring(ring); 7249 } 7250 7251 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring, 7252 unsigned int vmid) 7253 { 7254 struct amdgpu_device *adev = ring->adev; 7255 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 7256 struct amdgpu_ring *kiq_ring = &kiq->ring; 7257 unsigned long flags; 7258 int i, r; 7259 7260 if (amdgpu_sriov_vf(adev)) 7261 return -EINVAL; 7262 7263 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 7264 return -EINVAL; 7265 7266 spin_lock_irqsave(&kiq->ring_lock, flags); 7267 7268 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { 7269 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7270 return -ENOMEM; 7271 } 7272 7273 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES, 7274 0, 0); 7275 amdgpu_ring_commit(kiq_ring); 7276 7277 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7278 7279 r = amdgpu_ring_test_ring(kiq_ring); 7280 if (r) 7281 return r; 7282 7283 /* make sure dequeue is complete*/ 7284 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 7285 mutex_lock(&adev->srbm_mutex); 7286 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 7287 for (i = 0; i < adev->usec_timeout; i++) { 7288 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 7289 break; 7290 udelay(1); 7291 } 7292 if (i >= adev->usec_timeout) 7293 r = -ETIMEDOUT; 7294 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 7295 mutex_unlock(&adev->srbm_mutex); 7296 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 7297 if (r) { 7298 dev_err(adev->dev, "fail to wait on hqd deactive\n"); 7299 return r; 7300 } 7301 7302 r = amdgpu_bo_reserve(ring->mqd_obj, false); 7303 if (unlikely(r != 0)){ 7304 dev_err(adev->dev, "fail to resv mqd_obj\n"); 7305 return r; 7306 } 7307 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 7308 if (!r) { 7309 r = gfx_v9_0_kcq_init_queue(ring, true); 7310 amdgpu_bo_kunmap(ring->mqd_obj); 7311 ring->mqd_ptr = NULL; 7312 } 7313 amdgpu_bo_unreserve(ring->mqd_obj); 7314 if (r) { 7315 dev_err(adev->dev, "fail to unresv mqd_obj\n"); 7316 return r; 7317 } 7318 spin_lock_irqsave(&kiq->ring_lock, flags); 7319 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size); 7320 if (r) { 7321 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7322 return -ENOMEM; 7323 } 7324 kiq->pmf->kiq_map_queues(kiq_ring, ring); 7325 amdgpu_ring_commit(kiq_ring); 7326 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7327 r = amdgpu_ring_test_ring(kiq_ring); 7328 if (r) { 7329 DRM_ERROR("fail to remap queue\n"); 7330 return r; 7331 } 7332 return amdgpu_ring_test_ring(ring); 7333 } 7334 7335 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 7336 { 7337 struct amdgpu_device *adev = ip_block->adev; 7338 uint32_t i, j, k, reg, index = 0; 7339 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9); 7340 7341 if (!adev->gfx.ip_dump_core) 7342 return; 7343 7344 for (i = 0; i < reg_count; i++) 7345 drm_printf(p, "%-50s \t 0x%08x\n", 7346 gc_reg_list_9[i].reg_name, 7347 adev->gfx.ip_dump_core[i]); 7348 7349 /* print compute queue registers for all instances */ 7350 if (!adev->gfx.ip_dump_compute_queues) 7351 return; 7352 7353 reg_count = ARRAY_SIZE(gc_cp_reg_list_9); 7354 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 7355 adev->gfx.mec.num_mec, 7356 adev->gfx.mec.num_pipe_per_mec, 7357 adev->gfx.mec.num_queue_per_pipe); 7358 7359 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 7360 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 7361 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 7362 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 7363 for (reg = 0; reg < reg_count; reg++) { 7364 drm_printf(p, "%-50s \t 0x%08x\n", 7365 gc_cp_reg_list_9[reg].reg_name, 7366 adev->gfx.ip_dump_compute_queues[index + reg]); 7367 } 7368 index += reg_count; 7369 } 7370 } 7371 } 7372 7373 } 7374 7375 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block) 7376 { 7377 struct amdgpu_device *adev = ip_block->adev; 7378 uint32_t i, j, k, reg, index = 0; 7379 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9); 7380 7381 if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings) 7382 return; 7383 7384 amdgpu_gfx_off_ctrl(adev, false); 7385 for (i = 0; i < reg_count; i++) 7386 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i])); 7387 amdgpu_gfx_off_ctrl(adev, true); 7388 7389 /* dump compute queue registers for all instances */ 7390 if (!adev->gfx.ip_dump_compute_queues) 7391 return; 7392 7393 reg_count = ARRAY_SIZE(gc_cp_reg_list_9); 7394 amdgpu_gfx_off_ctrl(adev, false); 7395 mutex_lock(&adev->srbm_mutex); 7396 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 7397 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 7398 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 7399 /* ME0 is for GFX so start from 1 for CP */ 7400 soc15_grbm_select(adev, 1 + i, j, k, 0, 0); 7401 7402 for (reg = 0; reg < reg_count; reg++) { 7403 adev->gfx.ip_dump_compute_queues[index + reg] = 7404 RREG32(SOC15_REG_ENTRY_OFFSET( 7405 gc_cp_reg_list_9[reg])); 7406 } 7407 index += reg_count; 7408 } 7409 } 7410 } 7411 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 7412 mutex_unlock(&adev->srbm_mutex); 7413 amdgpu_gfx_off_ctrl(adev, true); 7414 7415 } 7416 7417 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 7418 { 7419 /* Emit the cleaner shader */ 7420 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 7421 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 7422 } 7423 7424 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 7425 .name = "gfx_v9_0", 7426 .early_init = gfx_v9_0_early_init, 7427 .late_init = gfx_v9_0_late_init, 7428 .sw_init = gfx_v9_0_sw_init, 7429 .sw_fini = gfx_v9_0_sw_fini, 7430 .hw_init = gfx_v9_0_hw_init, 7431 .hw_fini = gfx_v9_0_hw_fini, 7432 .suspend = gfx_v9_0_suspend, 7433 .resume = gfx_v9_0_resume, 7434 .is_idle = gfx_v9_0_is_idle, 7435 .wait_for_idle = gfx_v9_0_wait_for_idle, 7436 .soft_reset = gfx_v9_0_soft_reset, 7437 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 7438 .set_powergating_state = gfx_v9_0_set_powergating_state, 7439 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 7440 .dump_ip_state = gfx_v9_ip_dump, 7441 .print_ip_state = gfx_v9_ip_print, 7442 }; 7443 7444 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 7445 .type = AMDGPU_RING_TYPE_GFX, 7446 .align_mask = 0xff, 7447 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7448 .support_64bit_ptrs = true, 7449 .secure_submission_supported = true, 7450 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 7451 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 7452 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 7453 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 7454 5 + /* COND_EXEC */ 7455 7 + /* PIPELINE_SYNC */ 7456 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7457 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7458 2 + /* VM_FLUSH */ 7459 8 + /* FENCE for VM_FLUSH */ 7460 20 + /* GDS switch */ 7461 4 + /* double SWITCH_BUFFER, 7462 the first COND_EXEC jump to the place just 7463 prior to this double SWITCH_BUFFER */ 7464 5 + /* COND_EXEC */ 7465 7 + /* HDP_flush */ 7466 4 + /* VGT_flush */ 7467 14 + /* CE_META */ 7468 31 + /* DE_META */ 7469 3 + /* CNTX_CTRL */ 7470 5 + /* HDP_INVL */ 7471 8 + 8 + /* FENCE x2 */ 7472 2 + /* SWITCH_BUFFER */ 7473 7 + /* gfx_v9_0_emit_mem_sync */ 7474 2, /* gfx_v9_0_ring_emit_cleaner_shader */ 7475 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 7476 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 7477 .emit_fence = gfx_v9_0_ring_emit_fence, 7478 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 7479 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 7480 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 7481 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7482 .test_ring = gfx_v9_0_ring_test_ring, 7483 .insert_nop = gfx_v9_ring_insert_nop, 7484 .pad_ib = amdgpu_ring_generic_pad_ib, 7485 .emit_switch_buffer = gfx_v9_ring_emit_sb, 7486 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 7487 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 7488 .preempt_ib = gfx_v9_0_ring_preempt_ib, 7489 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 7490 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7491 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7492 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7493 .soft_recovery = gfx_v9_0_ring_soft_recovery, 7494 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7495 .reset = gfx_v9_0_reset_kgq, 7496 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader, 7497 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use, 7498 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use, 7499 }; 7500 7501 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = { 7502 .type = AMDGPU_RING_TYPE_GFX, 7503 .align_mask = 0xff, 7504 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7505 .support_64bit_ptrs = true, 7506 .secure_submission_supported = true, 7507 .get_rptr = amdgpu_sw_ring_get_rptr_gfx, 7508 .get_wptr = amdgpu_sw_ring_get_wptr_gfx, 7509 .set_wptr = amdgpu_sw_ring_set_wptr_gfx, 7510 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 7511 5 + /* COND_EXEC */ 7512 7 + /* PIPELINE_SYNC */ 7513 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7514 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7515 2 + /* VM_FLUSH */ 7516 8 + /* FENCE for VM_FLUSH */ 7517 20 + /* GDS switch */ 7518 4 + /* double SWITCH_BUFFER, 7519 * the first COND_EXEC jump to the place just 7520 * prior to this double SWITCH_BUFFER 7521 */ 7522 5 + /* COND_EXEC */ 7523 7 + /* HDP_flush */ 7524 4 + /* VGT_flush */ 7525 14 + /* CE_META */ 7526 31 + /* DE_META */ 7527 3 + /* CNTX_CTRL */ 7528 5 + /* HDP_INVL */ 7529 8 + 8 + /* FENCE x2 */ 7530 2 + /* SWITCH_BUFFER */ 7531 7 + /* gfx_v9_0_emit_mem_sync */ 7532 2, /* gfx_v9_0_ring_emit_cleaner_shader */ 7533 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 7534 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 7535 .emit_fence = gfx_v9_0_ring_emit_fence, 7536 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 7537 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 7538 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 7539 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7540 .test_ring = gfx_v9_0_ring_test_ring, 7541 .test_ib = gfx_v9_0_ring_test_ib, 7542 .insert_nop = gfx_v9_ring_insert_nop, 7543 .pad_ib = amdgpu_ring_generic_pad_ib, 7544 .emit_switch_buffer = gfx_v9_ring_emit_sb, 7545 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 7546 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 7547 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 7548 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7549 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7550 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7551 .soft_recovery = gfx_v9_0_ring_soft_recovery, 7552 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7553 .patch_cntl = gfx_v9_0_ring_patch_cntl, 7554 .patch_de = gfx_v9_0_ring_patch_de_meta, 7555 .patch_ce = gfx_v9_0_ring_patch_ce_meta, 7556 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader, 7557 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use, 7558 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use, 7559 }; 7560 7561 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 7562 .type = AMDGPU_RING_TYPE_COMPUTE, 7563 .align_mask = 0xff, 7564 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7565 .support_64bit_ptrs = true, 7566 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 7567 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 7568 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 7569 .emit_frame_size = 7570 20 + /* gfx_v9_0_ring_emit_gds_switch */ 7571 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 7572 5 + /* hdp invalidate */ 7573 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 7574 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7575 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7576 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 7577 7 + /* gfx_v9_0_emit_mem_sync */ 7578 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 7579 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 7580 2, /* gfx_v9_0_ring_emit_cleaner_shader */ 7581 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 7582 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 7583 .emit_fence = gfx_v9_0_ring_emit_fence, 7584 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 7585 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 7586 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 7587 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7588 .test_ring = gfx_v9_0_ring_test_ring, 7589 .test_ib = gfx_v9_0_ring_test_ib, 7590 .insert_nop = gfx_v9_ring_insert_nop, 7591 .pad_ib = amdgpu_ring_generic_pad_ib, 7592 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7593 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7594 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7595 .soft_recovery = gfx_v9_0_ring_soft_recovery, 7596 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7597 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 7598 .reset = gfx_v9_0_reset_kcq, 7599 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader, 7600 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use, 7601 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use, 7602 }; 7603 7604 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 7605 .type = AMDGPU_RING_TYPE_KIQ, 7606 .align_mask = 0xff, 7607 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7608 .support_64bit_ptrs = true, 7609 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 7610 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 7611 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 7612 .emit_frame_size = 7613 20 + /* gfx_v9_0_ring_emit_gds_switch */ 7614 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 7615 5 + /* hdp invalidate */ 7616 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 7617 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7618 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7619 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7620 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 7621 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 7622 .test_ring = gfx_v9_0_ring_test_ring, 7623 .insert_nop = amdgpu_ring_insert_nop, 7624 .pad_ib = amdgpu_ring_generic_pad_ib, 7625 .emit_rreg = gfx_v9_0_ring_emit_rreg, 7626 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7627 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7628 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7629 }; 7630 7631 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 7632 { 7633 int i; 7634 7635 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq; 7636 7637 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7638 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 7639 7640 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { 7641 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 7642 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx; 7643 } 7644 7645 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7646 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 7647 } 7648 7649 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 7650 .set = gfx_v9_0_set_eop_interrupt_state, 7651 .process = gfx_v9_0_eop_irq, 7652 }; 7653 7654 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 7655 .set = gfx_v9_0_set_priv_reg_fault_state, 7656 .process = gfx_v9_0_priv_reg_irq, 7657 }; 7658 7659 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = { 7660 .set = gfx_v9_0_set_bad_op_fault_state, 7661 .process = gfx_v9_0_bad_op_irq, 7662 }; 7663 7664 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 7665 .set = gfx_v9_0_set_priv_inst_fault_state, 7666 .process = gfx_v9_0_priv_inst_irq, 7667 }; 7668 7669 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 7670 .set = gfx_v9_0_set_cp_ecc_error_state, 7671 .process = amdgpu_gfx_cp_ecc_error_irq, 7672 }; 7673 7674 7675 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 7676 { 7677 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7678 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 7679 7680 adev->gfx.priv_reg_irq.num_types = 1; 7681 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 7682 7683 adev->gfx.bad_op_irq.num_types = 1; 7684 adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs; 7685 7686 adev->gfx.priv_inst_irq.num_types = 1; 7687 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 7688 7689 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 7690 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7691 } 7692 7693 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7694 { 7695 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7696 case IP_VERSION(9, 0, 1): 7697 case IP_VERSION(9, 2, 1): 7698 case IP_VERSION(9, 4, 0): 7699 case IP_VERSION(9, 2, 2): 7700 case IP_VERSION(9, 1, 0): 7701 case IP_VERSION(9, 4, 1): 7702 case IP_VERSION(9, 3, 0): 7703 case IP_VERSION(9, 4, 2): 7704 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7705 break; 7706 default: 7707 break; 7708 } 7709 } 7710 7711 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7712 { 7713 /* init asci gds info */ 7714 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7715 case IP_VERSION(9, 0, 1): 7716 case IP_VERSION(9, 2, 1): 7717 case IP_VERSION(9, 4, 0): 7718 adev->gds.gds_size = 0x10000; 7719 break; 7720 case IP_VERSION(9, 2, 2): 7721 case IP_VERSION(9, 1, 0): 7722 case IP_VERSION(9, 4, 1): 7723 adev->gds.gds_size = 0x1000; 7724 break; 7725 case IP_VERSION(9, 4, 2): 7726 /* aldebaran removed all the GDS internal memory, 7727 * only support GWS opcode in kernel, like barrier 7728 * semaphore.etc */ 7729 adev->gds.gds_size = 0; 7730 break; 7731 default: 7732 adev->gds.gds_size = 0x10000; 7733 break; 7734 } 7735 7736 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7737 case IP_VERSION(9, 0, 1): 7738 case IP_VERSION(9, 4, 0): 7739 adev->gds.gds_compute_max_wave_id = 0x7ff; 7740 break; 7741 case IP_VERSION(9, 2, 1): 7742 adev->gds.gds_compute_max_wave_id = 0x27f; 7743 break; 7744 case IP_VERSION(9, 2, 2): 7745 case IP_VERSION(9, 1, 0): 7746 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7747 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7748 else 7749 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7750 break; 7751 case IP_VERSION(9, 4, 1): 7752 adev->gds.gds_compute_max_wave_id = 0xfff; 7753 break; 7754 case IP_VERSION(9, 4, 2): 7755 /* deprecated for Aldebaran, no usage at all */ 7756 adev->gds.gds_compute_max_wave_id = 0; 7757 break; 7758 default: 7759 /* this really depends on the chip */ 7760 adev->gds.gds_compute_max_wave_id = 0x7ff; 7761 break; 7762 } 7763 7764 adev->gds.gws_size = 64; 7765 adev->gds.oa_size = 16; 7766 } 7767 7768 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7769 u32 bitmap) 7770 { 7771 u32 data; 7772 7773 if (!bitmap) 7774 return; 7775 7776 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7777 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7778 7779 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7780 } 7781 7782 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7783 { 7784 u32 data, mask; 7785 7786 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7787 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7788 7789 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7790 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7791 7792 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7793 7794 return (~data) & mask; 7795 } 7796 7797 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7798 struct amdgpu_cu_info *cu_info) 7799 { 7800 int i, j, k, counter, active_cu_number = 0; 7801 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7802 unsigned disable_masks[4 * 4]; 7803 7804 if (!adev || !cu_info) 7805 return -EINVAL; 7806 7807 /* 7808 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7809 */ 7810 if (adev->gfx.config.max_shader_engines * 7811 adev->gfx.config.max_sh_per_se > 16) 7812 return -EINVAL; 7813 7814 amdgpu_gfx_parse_disable_cu(disable_masks, 7815 adev->gfx.config.max_shader_engines, 7816 adev->gfx.config.max_sh_per_se); 7817 7818 mutex_lock(&adev->grbm_idx_mutex); 7819 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7820 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7821 mask = 1; 7822 ao_bitmap = 0; 7823 counter = 0; 7824 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 7825 gfx_v9_0_set_user_cu_inactive_bitmap( 7826 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7827 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7828 7829 /* 7830 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7831 * 4x4 size array, and it's usually suitable for Vega 7832 * ASICs which has 4*2 SE/SH layout. 7833 * But for Arcturus, SE/SH layout is changed to 8*1. 7834 * To mostly reduce the impact, we make it compatible 7835 * with current bitmap array as below: 7836 * SE4,SH0 --> bitmap[0][1] 7837 * SE5,SH0 --> bitmap[1][1] 7838 * SE6,SH0 --> bitmap[2][1] 7839 * SE7,SH0 --> bitmap[3][1] 7840 */ 7841 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap; 7842 7843 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7844 if (bitmap & mask) { 7845 if (counter < adev->gfx.config.max_cu_per_sh) 7846 ao_bitmap |= mask; 7847 counter ++; 7848 } 7849 mask <<= 1; 7850 } 7851 active_cu_number += counter; 7852 if (i < 2 && j < 2) 7853 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7854 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7855 } 7856 } 7857 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7858 mutex_unlock(&adev->grbm_idx_mutex); 7859 7860 cu_info->number = active_cu_number; 7861 cu_info->ao_cu_mask = ao_cu_mask; 7862 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7863 7864 return 0; 7865 } 7866 7867 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7868 { 7869 .type = AMD_IP_BLOCK_TYPE_GFX, 7870 .major = 9, 7871 .minor = 0, 7872 .rev = 0, 7873 .funcs = &gfx_v9_0_ip_funcs, 7874 }; 7875