1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/delay.h> 24 #include <linux/kernel.h> 25 #include <linux/firmware.h> 26 #include <linux/module.h> 27 #include <linux/pci.h> 28 #include "amdgpu.h" 29 #include "amdgpu_gfx.h" 30 #include "amdgpu_psp.h" 31 #include "amdgpu_smu.h" 32 #include "imu_v11_0.h" 33 #include "soc21.h" 34 #include "nvd.h" 35 36 #include "gc/gc_11_0_0_offset.h" 37 #include "gc/gc_11_0_0_sh_mask.h" 38 #include "smuio/smuio_13_0_6_offset.h" 39 #include "smuio/smuio_13_0_6_sh_mask.h" 40 #include "navi10_enum.h" 41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" 42 43 #include "soc15.h" 44 #include "clearstate_gfx11.h" 45 #include "v11_structs.h" 46 #include "gfx_v11_0.h" 47 #include "gfx_v11_0_cleaner_shader.h" 48 #include "gfx_v11_0_3.h" 49 #include "nbio_v4_3.h" 50 #include "mes_v11_0.h" 51 52 #define GFX11_NUM_GFX_RINGS 1 53 #define GFX11_MEC_HPD_SIZE 2048 54 55 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 56 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388 57 58 #define regCGTT_WD_CLK_CTRL 0x5086 59 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1 60 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e 61 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1 62 #define regPC_CONFIG_CNTL_1 0x194d 63 #define regPC_CONFIG_CNTL_1_BASE_IDX 1 64 65 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100 66 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000 67 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000 68 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01 69 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000 70 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000 71 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000 72 73 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006 74 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 75 #define regCP_MQD_CONTROL_DEFAULT 0x00000100 76 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509 77 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 78 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000 79 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501 80 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000 81 82 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin"); 83 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin"); 84 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin"); 85 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin"); 86 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin"); 87 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin"); 88 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin"); 90 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin"); 92 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin"); 93 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin"); 94 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin"); 95 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin"); 96 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin"); 97 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin"); 98 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin"); 99 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin"); 101 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin"); 102 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin"); 103 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin"); 104 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin"); 105 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin"); 106 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin"); 107 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin"); 109 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin"); 110 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin"); 112 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin"); 113 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin"); 114 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin"); 115 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin"); 116 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin"); 117 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin"); 118 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin"); 119 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin"); 120 121 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { 122 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), 123 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2), 124 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3), 125 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1), 126 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2), 127 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3), 128 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1), 129 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1), 130 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT), 131 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT), 132 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT), 133 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2), 134 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2), 135 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS), 136 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR), 137 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0), 138 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE), 139 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 140 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR), 141 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE), 142 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR), 143 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR), 144 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE), 145 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR), 146 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR), 147 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 148 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ), 149 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 150 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 151 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 152 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO), 153 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI), 154 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ), 155 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS), 156 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS), 157 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS), 158 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT), 159 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT), 160 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS), 161 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2), 162 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS), 163 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS), 164 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS), 165 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES), 166 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS), 167 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS), 168 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL), 169 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS), 170 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG), 171 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL), 172 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL), 173 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR), 174 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR), 175 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR), 176 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR), 177 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR), 178 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS), 179 /* cp header registers */ 180 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 181 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 182 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 183 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 184 /* SE status registers */ 185 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0), 186 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1), 187 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2), 188 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3), 189 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4), 190 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5) 191 }; 192 193 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = { 194 /* compute registers */ 195 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID), 196 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE), 197 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY), 198 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY), 199 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM), 200 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE), 201 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI), 202 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR), 203 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 204 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 205 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), 206 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL), 207 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR), 208 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI), 209 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR), 210 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL), 211 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST), 212 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR), 213 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI), 214 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL), 215 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR), 216 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR), 217 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS), 218 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO), 219 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI), 220 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL), 221 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET), 222 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE), 223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET), 224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE), 225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE), 226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR), 227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM), 228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO), 229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI), 230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET), 231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT), 232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET), 233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS) 234 }; 235 236 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = { 237 /* gfx queue registers */ 238 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE), 239 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID), 240 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY), 241 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM), 242 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE), 243 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI), 244 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET), 245 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL), 246 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR), 247 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR), 248 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI), 249 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST), 250 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED), 251 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL), 252 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0), 253 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0), 254 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR), 255 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI), 256 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO), 257 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI), 258 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 259 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 260 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 261 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 262 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ) 263 }; 264 265 static const struct soc15_reg_golden golden_settings_gc_11_0[] = { 266 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000) 267 }; 268 269 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] = 270 { 271 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010), 272 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010), 273 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200), 274 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988), 275 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007), 276 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008), 277 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100), 278 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000), 279 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a) 280 }; 281 282 #define DEFAULT_SH_MEM_CONFIG \ 283 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \ 284 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \ 285 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT)) 286 287 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev); 288 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev); 289 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev); 290 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev); 291 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev); 292 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev); 293 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev); 294 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 295 struct amdgpu_cu_info *cu_info); 296 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev); 297 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 298 u32 sh_num, u32 instance, int xcc_id); 299 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); 300 301 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); 302 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); 303 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 304 uint32_t val); 305 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); 306 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 307 uint16_t pasid, uint32_t flush_type, 308 bool all_hub, uint8_t dst_sel); 309 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 310 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 311 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 312 bool enable); 313 314 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) 315 { 316 struct amdgpu_device *adev = kiq_ring->adev; 317 u64 shader_mc_addr; 318 319 /* Cleaner shader MC address */ 320 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 321 322 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 323 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 324 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */ 325 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 326 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 327 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 328 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 329 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 330 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 331 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 332 } 333 334 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring, 335 struct amdgpu_ring *ring) 336 { 337 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 338 uint64_t wptr_addr = ring->wptr_gpu_addr; 339 uint32_t me = 0, eng_sel = 0; 340 341 switch (ring->funcs->type) { 342 case AMDGPU_RING_TYPE_COMPUTE: 343 me = 1; 344 eng_sel = 0; 345 break; 346 case AMDGPU_RING_TYPE_GFX: 347 me = 0; 348 eng_sel = 4; 349 break; 350 case AMDGPU_RING_TYPE_MES: 351 me = 2; 352 eng_sel = 5; 353 break; 354 default: 355 WARN_ON(1); 356 } 357 358 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 359 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 360 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 361 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 362 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 363 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 364 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 365 PACKET3_MAP_QUEUES_ME((me)) | 366 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 367 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 368 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 369 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 370 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 371 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 372 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 373 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 374 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 375 } 376 377 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 378 struct amdgpu_ring *ring, 379 enum amdgpu_unmap_queues_action action, 380 u64 gpu_addr, u64 seq) 381 { 382 struct amdgpu_device *adev = kiq_ring->adev; 383 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 384 385 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) { 386 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq); 387 return; 388 } 389 390 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 391 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 392 PACKET3_UNMAP_QUEUES_ACTION(action) | 393 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 394 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 395 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 396 amdgpu_ring_write(kiq_ring, 397 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 398 399 if (action == PREEMPT_QUEUES_NO_UNMAP) { 400 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 401 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 402 amdgpu_ring_write(kiq_ring, seq); 403 } else { 404 amdgpu_ring_write(kiq_ring, 0); 405 amdgpu_ring_write(kiq_ring, 0); 406 amdgpu_ring_write(kiq_ring, 0); 407 } 408 } 409 410 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring, 411 struct amdgpu_ring *ring, 412 u64 addr, 413 u64 seq) 414 { 415 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 416 417 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 418 amdgpu_ring_write(kiq_ring, 419 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 420 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 421 PACKET3_QUERY_STATUS_COMMAND(2)); 422 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 423 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 424 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 425 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 426 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 427 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 428 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 429 } 430 431 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 432 uint16_t pasid, uint32_t flush_type, 433 bool all_hub) 434 { 435 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1); 436 } 437 438 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = { 439 .kiq_set_resources = gfx11_kiq_set_resources, 440 .kiq_map_queues = gfx11_kiq_map_queues, 441 .kiq_unmap_queues = gfx11_kiq_unmap_queues, 442 .kiq_query_status = gfx11_kiq_query_status, 443 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs, 444 .set_resources_size = 8, 445 .map_queues_size = 7, 446 .unmap_queues_size = 6, 447 .query_status_size = 7, 448 .invalidate_tlbs_size = 2, 449 }; 450 451 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 452 { 453 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs; 454 } 455 456 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev) 457 { 458 if (amdgpu_sriov_vf(adev)) 459 return; 460 461 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 462 case IP_VERSION(11, 0, 1): 463 case IP_VERSION(11, 0, 4): 464 soc15_program_register_sequence(adev, 465 golden_settings_gc_11_0_1, 466 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1)); 467 break; 468 default: 469 break; 470 } 471 soc15_program_register_sequence(adev, 472 golden_settings_gc_11_0, 473 (const u32)ARRAY_SIZE(golden_settings_gc_11_0)); 474 475 } 476 477 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 478 bool wc, uint32_t reg, uint32_t val) 479 { 480 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 481 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 482 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0)); 483 amdgpu_ring_write(ring, reg); 484 amdgpu_ring_write(ring, 0); 485 amdgpu_ring_write(ring, val); 486 } 487 488 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 489 int mem_space, int opt, uint32_t addr0, 490 uint32_t addr1, uint32_t ref, uint32_t mask, 491 uint32_t inv) 492 { 493 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 494 amdgpu_ring_write(ring, 495 /* memory (1) or register (0) */ 496 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 497 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 498 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 499 WAIT_REG_MEM_ENGINE(eng_sel))); 500 501 if (mem_space) 502 BUG_ON(addr0 & 0x3); /* Dword align */ 503 amdgpu_ring_write(ring, addr0); 504 amdgpu_ring_write(ring, addr1); 505 amdgpu_ring_write(ring, ref); 506 amdgpu_ring_write(ring, mask); 507 amdgpu_ring_write(ring, inv); /* poll interval */ 508 } 509 510 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 511 { 512 /* Header itself is a NOP packet */ 513 if (num_nop == 1) { 514 amdgpu_ring_write(ring, ring->funcs->nop); 515 return; 516 } 517 518 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 519 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 520 521 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 522 amdgpu_ring_insert_nop(ring, num_nop - 1); 523 } 524 525 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring) 526 { 527 struct amdgpu_device *adev = ring->adev; 528 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 529 uint32_t tmp = 0; 530 unsigned i; 531 int r; 532 533 WREG32(scratch, 0xCAFEDEAD); 534 r = amdgpu_ring_alloc(ring, 5); 535 if (r) { 536 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n", 537 ring->idx, r); 538 return r; 539 } 540 541 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) { 542 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF); 543 } else { 544 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 545 amdgpu_ring_write(ring, scratch - 546 PACKET3_SET_UCONFIG_REG_START); 547 amdgpu_ring_write(ring, 0xDEADBEEF); 548 } 549 amdgpu_ring_commit(ring); 550 551 for (i = 0; i < adev->usec_timeout; i++) { 552 tmp = RREG32(scratch); 553 if (tmp == 0xDEADBEEF) 554 break; 555 if (amdgpu_emu_mode == 1) 556 msleep(1); 557 else 558 udelay(1); 559 } 560 561 if (i >= adev->usec_timeout) 562 r = -ETIMEDOUT; 563 return r; 564 } 565 566 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 567 { 568 struct amdgpu_device *adev = ring->adev; 569 struct amdgpu_ib ib; 570 struct dma_fence *f = NULL; 571 unsigned index; 572 uint64_t gpu_addr; 573 volatile uint32_t *cpu_ptr; 574 long r; 575 576 /* MES KIQ fw hasn't indirect buffer support for now */ 577 if (adev->enable_mes_kiq && 578 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 579 return 0; 580 581 memset(&ib, 0, sizeof(ib)); 582 583 if (ring->is_mes_queue) { 584 uint32_t padding, offset; 585 586 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS); 587 padding = amdgpu_mes_ctx_get_offs(ring, 588 AMDGPU_MES_CTX_PADDING_OFFS); 589 590 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 591 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); 592 593 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding); 594 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding); 595 *cpu_ptr = cpu_to_le32(0xCAFEDEAD); 596 } else { 597 r = amdgpu_device_wb_get(adev, &index); 598 if (r) 599 return r; 600 601 gpu_addr = adev->wb.gpu_addr + (index * 4); 602 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 603 cpu_ptr = &adev->wb.wb[index]; 604 605 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 606 if (r) { 607 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); 608 goto err1; 609 } 610 } 611 612 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 613 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 614 ib.ptr[2] = lower_32_bits(gpu_addr); 615 ib.ptr[3] = upper_32_bits(gpu_addr); 616 ib.ptr[4] = 0xDEADBEEF; 617 ib.length_dw = 5; 618 619 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 620 if (r) 621 goto err2; 622 623 r = dma_fence_wait_timeout(f, false, timeout); 624 if (r == 0) { 625 r = -ETIMEDOUT; 626 goto err2; 627 } else if (r < 0) { 628 goto err2; 629 } 630 631 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) 632 r = 0; 633 else 634 r = -EINVAL; 635 err2: 636 if (!ring->is_mes_queue) 637 amdgpu_ib_free(&ib, NULL); 638 dma_fence_put(f); 639 err1: 640 if (!ring->is_mes_queue) 641 amdgpu_device_wb_free(adev, index); 642 return r; 643 } 644 645 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev) 646 { 647 amdgpu_ucode_release(&adev->gfx.pfp_fw); 648 amdgpu_ucode_release(&adev->gfx.me_fw); 649 amdgpu_ucode_release(&adev->gfx.rlc_fw); 650 amdgpu_ucode_release(&adev->gfx.mec_fw); 651 652 kfree(adev->gfx.rlc.register_list_format); 653 } 654 655 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) 656 { 657 const struct psp_firmware_header_v1_0 *toc_hdr; 658 int err = 0; 659 660 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, 661 AMDGPU_UCODE_REQUIRED, 662 "amdgpu/%s_toc.bin", ucode_prefix); 663 if (err) 664 goto out; 665 666 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data; 667 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version); 668 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version); 669 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes); 670 adev->psp.toc.start_addr = (uint8_t *)toc_hdr + 671 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes); 672 return 0; 673 out: 674 amdgpu_ucode_release(&adev->psp.toc_fw); 675 return err; 676 } 677 678 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev) 679 { 680 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 681 case IP_VERSION(11, 0, 0): 682 case IP_VERSION(11, 0, 2): 683 case IP_VERSION(11, 0, 3): 684 if ((adev->gfx.me_fw_version >= 1505) && 685 (adev->gfx.pfp_fw_version >= 1600) && 686 (adev->gfx.mec_fw_version >= 512)) { 687 if (amdgpu_sriov_vf(adev)) 688 adev->gfx.cp_gfx_shadow = true; 689 else 690 adev->gfx.cp_gfx_shadow = false; 691 } 692 break; 693 default: 694 adev->gfx.cp_gfx_shadow = false; 695 break; 696 } 697 } 698 699 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) 700 { 701 char ucode_prefix[25]; 702 int err; 703 const struct rlc_firmware_header_v2_0 *rlc_hdr; 704 uint16_t version_major; 705 uint16_t version_minor; 706 707 DRM_DEBUG("\n"); 708 709 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 710 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 711 AMDGPU_UCODE_REQUIRED, 712 "amdgpu/%s_pfp.bin", ucode_prefix); 713 if (err) 714 goto out; 715 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/ 716 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version( 717 (union amdgpu_firmware_header *) 718 adev->gfx.pfp_fw->data, 2, 0); 719 if (adev->gfx.rs64_enable) { 720 dev_info(adev->dev, "CP RS64 enable\n"); 721 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP); 722 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK); 723 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK); 724 } else { 725 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 726 } 727 728 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 729 AMDGPU_UCODE_REQUIRED, 730 "amdgpu/%s_me.bin", ucode_prefix); 731 if (err) 732 goto out; 733 if (adev->gfx.rs64_enable) { 734 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME); 735 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK); 736 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK); 737 } else { 738 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 739 } 740 741 if (!amdgpu_sriov_vf(adev)) { 742 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) && 743 adev->pdev->revision == 0xCE) 744 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 745 AMDGPU_UCODE_REQUIRED, 746 "amdgpu/gc_11_0_0_rlc_1.bin"); 747 else 748 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 749 AMDGPU_UCODE_REQUIRED, 750 "amdgpu/%s_rlc.bin", ucode_prefix); 751 if (err) 752 goto out; 753 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 754 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 755 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 756 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 757 if (err) 758 goto out; 759 } 760 761 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 762 AMDGPU_UCODE_REQUIRED, 763 "amdgpu/%s_mec.bin", ucode_prefix); 764 if (err) 765 goto out; 766 if (adev->gfx.rs64_enable) { 767 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); 768 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); 769 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); 770 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); 771 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); 772 } else { 773 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 774 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 775 } 776 777 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 778 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix); 779 780 /* only one MEC for gfx 11.0.0. */ 781 adev->gfx.mec2_fw = NULL; 782 783 gfx_v11_0_check_fw_cp_gfx_shadow(adev); 784 785 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) { 786 err = adev->gfx.imu.funcs->init_microcode(adev); 787 if (err) 788 DRM_ERROR("Failed to init imu firmware!\n"); 789 return err; 790 } 791 792 out: 793 if (err) { 794 amdgpu_ucode_release(&adev->gfx.pfp_fw); 795 amdgpu_ucode_release(&adev->gfx.me_fw); 796 amdgpu_ucode_release(&adev->gfx.rlc_fw); 797 amdgpu_ucode_release(&adev->gfx.mec_fw); 798 } 799 800 return err; 801 } 802 803 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev) 804 { 805 u32 count = 0; 806 const struct cs_section_def *sect = NULL; 807 const struct cs_extent_def *ext = NULL; 808 809 /* begin clear state */ 810 count += 2; 811 /* context control state */ 812 count += 3; 813 814 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 815 for (ext = sect->section; ext->extent != NULL; ++ext) { 816 if (sect->id == SECT_CONTEXT) 817 count += 2 + ext->reg_count; 818 else 819 return 0; 820 } 821 } 822 823 /* set PA_SC_TILE_STEERING_OVERRIDE */ 824 count += 3; 825 /* end clear state */ 826 count += 2; 827 /* clear state */ 828 count += 2; 829 830 return count; 831 } 832 833 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, 834 volatile u32 *buffer) 835 { 836 u32 count = 0, i; 837 const struct cs_section_def *sect = NULL; 838 const struct cs_extent_def *ext = NULL; 839 int ctx_reg_offset; 840 841 if (adev->gfx.rlc.cs_data == NULL) 842 return; 843 if (buffer == NULL) 844 return; 845 846 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 847 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 848 849 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 850 buffer[count++] = cpu_to_le32(0x80000000); 851 buffer[count++] = cpu_to_le32(0x80000000); 852 853 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 854 for (ext = sect->section; ext->extent != NULL; ++ext) { 855 if (sect->id == SECT_CONTEXT) { 856 buffer[count++] = 857 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 858 buffer[count++] = cpu_to_le32(ext->reg_index - 859 PACKET3_SET_CONTEXT_REG_START); 860 for (i = 0; i < ext->reg_count; i++) 861 buffer[count++] = cpu_to_le32(ext->extent[i]); 862 } else { 863 return; 864 } 865 } 866 } 867 868 ctx_reg_offset = 869 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 870 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 871 buffer[count++] = cpu_to_le32(ctx_reg_offset); 872 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override); 873 874 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 875 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 876 877 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 878 buffer[count++] = cpu_to_le32(0); 879 } 880 881 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev) 882 { 883 /* clear state block */ 884 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 885 &adev->gfx.rlc.clear_state_gpu_addr, 886 (void **)&adev->gfx.rlc.cs_ptr); 887 888 /* jump table block */ 889 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 890 &adev->gfx.rlc.cp_table_gpu_addr, 891 (void **)&adev->gfx.rlc.cp_table_ptr); 892 } 893 894 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 895 { 896 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 897 898 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 899 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 900 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1); 901 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2); 902 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3); 903 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL); 904 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX); 905 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0); 906 adev->gfx.rlc.rlcg_reg_access_supported = true; 907 } 908 909 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev) 910 { 911 const struct cs_section_def *cs_data; 912 int r; 913 914 adev->gfx.rlc.cs_data = gfx11_cs_data; 915 916 cs_data = adev->gfx.rlc.cs_data; 917 918 if (cs_data) { 919 /* init clear state block */ 920 r = amdgpu_gfx_rlc_init_csb(adev); 921 if (r) 922 return r; 923 } 924 925 /* init spm vmid with 0xf */ 926 if (adev->gfx.rlc.funcs->update_spm_vmid) 927 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf); 928 929 return 0; 930 } 931 932 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev) 933 { 934 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 935 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 936 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL); 937 } 938 939 static void gfx_v11_0_me_init(struct amdgpu_device *adev) 940 { 941 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 942 943 amdgpu_gfx_graphics_queue_acquire(adev); 944 } 945 946 static int gfx_v11_0_mec_init(struct amdgpu_device *adev) 947 { 948 int r; 949 u32 *hpd; 950 size_t mec_hpd_size; 951 952 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 953 954 /* take ownership of the relevant compute queues */ 955 amdgpu_gfx_compute_queue_acquire(adev); 956 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE; 957 958 if (mec_hpd_size) { 959 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 960 AMDGPU_GEM_DOMAIN_GTT, 961 &adev->gfx.mec.hpd_eop_obj, 962 &adev->gfx.mec.hpd_eop_gpu_addr, 963 (void **)&hpd); 964 if (r) { 965 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 966 gfx_v11_0_mec_fini(adev); 967 return r; 968 } 969 970 memset(hpd, 0, mec_hpd_size); 971 972 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 973 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 974 } 975 976 return 0; 977 } 978 979 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address) 980 { 981 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 982 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 983 (address << SQ_IND_INDEX__INDEX__SHIFT)); 984 return RREG32_SOC15(GC, 0, regSQ_IND_DATA); 985 } 986 987 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave, 988 uint32_t thread, uint32_t regno, 989 uint32_t num, uint32_t *out) 990 { 991 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 992 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 993 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 994 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) | 995 (SQ_IND_INDEX__AUTO_INCR_MASK)); 996 while (num--) 997 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA); 998 } 999 1000 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1001 { 1002 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE 1003 * field when performing a select_se_sh so it should be 1004 * zero here */ 1005 WARN_ON(simd != 0); 1006 1007 /* type 3 wave data */ 1008 dst[(*no_fields)++] = 3; 1009 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS); 1010 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO); 1011 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI); 1012 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO); 1013 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI); 1014 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1); 1015 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2); 1016 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC); 1017 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC); 1018 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS); 1019 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS); 1020 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2); 1021 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1); 1022 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0); 1023 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE); 1024 } 1025 1026 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1027 uint32_t wave, uint32_t start, 1028 uint32_t size, uint32_t *dst) 1029 { 1030 WARN_ON(simd != 0); 1031 1032 wave_read_regs( 1033 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size, 1034 dst); 1035 } 1036 1037 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1038 uint32_t wave, uint32_t thread, 1039 uint32_t start, uint32_t size, 1040 uint32_t *dst) 1041 { 1042 wave_read_regs( 1043 adev, wave, thread, 1044 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1045 } 1046 1047 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev, 1048 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1049 { 1050 soc21_grbm_select(adev, me, pipe, q, vm); 1051 } 1052 1053 /* all sizes are in bytes */ 1054 #define MQD_SHADOW_BASE_SIZE 73728 1055 #define MQD_SHADOW_BASE_ALIGNMENT 256 1056 #define MQD_FWWORKAREA_SIZE 484 1057 #define MQD_FWWORKAREA_ALIGNMENT 256 1058 1059 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev, 1060 struct amdgpu_gfx_shadow_info *shadow_info) 1061 { 1062 if (adev->gfx.cp_gfx_shadow) { 1063 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE; 1064 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT; 1065 shadow_info->csa_size = MQD_FWWORKAREA_SIZE; 1066 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT; 1067 return 0; 1068 } else { 1069 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info)); 1070 return -ENOTSUPP; 1071 } 1072 } 1073 1074 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = { 1075 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter, 1076 .select_se_sh = &gfx_v11_0_select_se_sh, 1077 .read_wave_data = &gfx_v11_0_read_wave_data, 1078 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs, 1079 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs, 1080 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q, 1081 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk, 1082 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info, 1083 }; 1084 1085 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) 1086 { 1087 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1088 case IP_VERSION(11, 0, 0): 1089 case IP_VERSION(11, 0, 2): 1090 adev->gfx.config.max_hw_contexts = 8; 1091 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1092 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1093 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1094 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1095 break; 1096 case IP_VERSION(11, 0, 3): 1097 adev->gfx.ras = &gfx_v11_0_3_ras; 1098 adev->gfx.config.max_hw_contexts = 8; 1099 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1100 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1101 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1102 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1103 break; 1104 case IP_VERSION(11, 0, 1): 1105 case IP_VERSION(11, 0, 4): 1106 case IP_VERSION(11, 5, 0): 1107 case IP_VERSION(11, 5, 1): 1108 case IP_VERSION(11, 5, 2): 1109 case IP_VERSION(11, 5, 3): 1110 adev->gfx.config.max_hw_contexts = 8; 1111 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1112 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1113 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1114 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300; 1115 break; 1116 default: 1117 BUG(); 1118 break; 1119 } 1120 1121 return 0; 1122 } 1123 1124 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id, 1125 int me, int pipe, int queue) 1126 { 1127 struct amdgpu_ring *ring; 1128 unsigned int irq_type; 1129 unsigned int hw_prio; 1130 1131 ring = &adev->gfx.gfx_ring[ring_id]; 1132 1133 ring->me = me; 1134 ring->pipe = pipe; 1135 ring->queue = queue; 1136 1137 ring->ring_obj = NULL; 1138 ring->use_doorbell = true; 1139 1140 if (!ring_id) 1141 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 1142 else 1143 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1; 1144 ring->vm_hub = AMDGPU_GFXHUB(0); 1145 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1146 1147 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe; 1148 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ? 1149 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1150 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1151 hw_prio, NULL); 1152 } 1153 1154 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1155 int mec, int pipe, int queue) 1156 { 1157 int r; 1158 unsigned irq_type; 1159 struct amdgpu_ring *ring; 1160 unsigned int hw_prio; 1161 1162 ring = &adev->gfx.compute_ring[ring_id]; 1163 1164 /* mec0 is me1 */ 1165 ring->me = mec + 1; 1166 ring->pipe = pipe; 1167 ring->queue = queue; 1168 1169 ring->ring_obj = NULL; 1170 ring->use_doorbell = true; 1171 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 1172 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1173 + (ring_id * GFX11_MEC_HPD_SIZE); 1174 ring->vm_hub = AMDGPU_GFXHUB(0); 1175 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1176 1177 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1178 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1179 + ring->pipe; 1180 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1181 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1182 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1183 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1184 hw_prio, NULL); 1185 if (r) 1186 return r; 1187 1188 return 0; 1189 } 1190 1191 static struct { 1192 SOC21_FIRMWARE_ID id; 1193 unsigned int offset; 1194 unsigned int size; 1195 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX]; 1196 1197 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) 1198 { 1199 RLC_TABLE_OF_CONTENT *ucode = rlc_toc; 1200 1201 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) && 1202 (ucode->id < SOC21_FIRMWARE_ID_MAX)) { 1203 rlc_autoload_info[ucode->id].id = ucode->id; 1204 rlc_autoload_info[ucode->id].offset = ucode->offset * 4; 1205 rlc_autoload_info[ucode->id].size = ucode->size * 4; 1206 1207 ucode++; 1208 } 1209 } 1210 1211 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev) 1212 { 1213 uint32_t total_size = 0; 1214 SOC21_FIRMWARE_ID id; 1215 1216 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr); 1217 1218 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++) 1219 total_size += rlc_autoload_info[id].size; 1220 1221 /* In case the offset in rlc toc ucode is aligned */ 1222 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset) 1223 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset + 1224 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size; 1225 1226 return total_size; 1227 } 1228 1229 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev) 1230 { 1231 int r; 1232 uint32_t total_size; 1233 1234 total_size = gfx_v11_0_calc_toc_total_size(adev); 1235 1236 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024, 1237 AMDGPU_GEM_DOMAIN_VRAM | 1238 AMDGPU_GEM_DOMAIN_GTT, 1239 &adev->gfx.rlc.rlc_autoload_bo, 1240 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1241 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1242 1243 if (r) { 1244 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r); 1245 return r; 1246 } 1247 1248 return 0; 1249 } 1250 1251 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, 1252 SOC21_FIRMWARE_ID id, 1253 const void *fw_data, 1254 uint32_t fw_size, 1255 uint32_t *fw_autoload_mask) 1256 { 1257 uint32_t toc_offset; 1258 uint32_t toc_fw_size; 1259 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; 1260 1261 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX) 1262 return; 1263 1264 toc_offset = rlc_autoload_info[id].offset; 1265 toc_fw_size = rlc_autoload_info[id].size; 1266 1267 if (fw_size == 0) 1268 fw_size = toc_fw_size; 1269 1270 if (fw_size > toc_fw_size) 1271 fw_size = toc_fw_size; 1272 1273 memcpy(ptr + toc_offset, fw_data, fw_size); 1274 1275 if (fw_size < toc_fw_size) 1276 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); 1277 1278 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME)) 1279 *(uint64_t *)fw_autoload_mask |= 1ULL << id; 1280 } 1281 1282 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev, 1283 uint32_t *fw_autoload_mask) 1284 { 1285 void *data; 1286 uint32_t size; 1287 uint64_t *toc_ptr; 1288 1289 *(uint64_t *)fw_autoload_mask |= 0x1; 1290 1291 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask); 1292 1293 data = adev->psp.toc.start_addr; 1294 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size; 1295 1296 toc_ptr = (uint64_t *)data + size / 8 - 1; 1297 *toc_ptr = *(uint64_t *)fw_autoload_mask; 1298 1299 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC, 1300 data, size, fw_autoload_mask); 1301 } 1302 1303 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev, 1304 uint32_t *fw_autoload_mask) 1305 { 1306 const __le32 *fw_data; 1307 uint32_t fw_size; 1308 const struct gfx_firmware_header_v1_0 *cp_hdr; 1309 const struct gfx_firmware_header_v2_0 *cpv2_hdr; 1310 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1311 const struct rlc_firmware_header_v2_2 *rlcv22_hdr; 1312 uint16_t version_major, version_minor; 1313 1314 if (adev->gfx.rs64_enable) { 1315 /* pfp ucode */ 1316 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1317 adev->gfx.pfp_fw->data; 1318 /* instruction */ 1319 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1320 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1321 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1322 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP, 1323 fw_data, fw_size, fw_autoload_mask); 1324 /* data */ 1325 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1326 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1327 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1328 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK, 1329 fw_data, fw_size, fw_autoload_mask); 1330 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK, 1331 fw_data, fw_size, fw_autoload_mask); 1332 /* me ucode */ 1333 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1334 adev->gfx.me_fw->data; 1335 /* instruction */ 1336 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1337 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1338 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1339 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME, 1340 fw_data, fw_size, fw_autoload_mask); 1341 /* data */ 1342 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1343 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1344 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1345 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK, 1346 fw_data, fw_size, fw_autoload_mask); 1347 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK, 1348 fw_data, fw_size, fw_autoload_mask); 1349 /* mec ucode */ 1350 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1351 adev->gfx.mec_fw->data; 1352 /* instruction */ 1353 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1354 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1355 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1356 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC, 1357 fw_data, fw_size, fw_autoload_mask); 1358 /* data */ 1359 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1360 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1361 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1362 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK, 1363 fw_data, fw_size, fw_autoload_mask); 1364 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK, 1365 fw_data, fw_size, fw_autoload_mask); 1366 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK, 1367 fw_data, fw_size, fw_autoload_mask); 1368 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK, 1369 fw_data, fw_size, fw_autoload_mask); 1370 } else { 1371 /* pfp ucode */ 1372 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1373 adev->gfx.pfp_fw->data; 1374 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1375 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1376 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1377 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP, 1378 fw_data, fw_size, fw_autoload_mask); 1379 1380 /* me ucode */ 1381 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1382 adev->gfx.me_fw->data; 1383 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1384 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1385 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1386 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME, 1387 fw_data, fw_size, fw_autoload_mask); 1388 1389 /* mec ucode */ 1390 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1391 adev->gfx.mec_fw->data; 1392 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1393 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1394 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1395 cp_hdr->jt_size * 4; 1396 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC, 1397 fw_data, fw_size, fw_autoload_mask); 1398 } 1399 1400 /* rlc ucode */ 1401 rlc_hdr = (const struct rlc_firmware_header_v2_0 *) 1402 adev->gfx.rlc_fw->data; 1403 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1404 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes)); 1405 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes); 1406 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE, 1407 fw_data, fw_size, fw_autoload_mask); 1408 1409 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1410 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1411 if (version_major == 2) { 1412 if (version_minor >= 2) { 1413 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1414 1415 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1416 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes)); 1417 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes); 1418 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE, 1419 fw_data, fw_size, fw_autoload_mask); 1420 1421 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1422 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes)); 1423 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes); 1424 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT, 1425 fw_data, fw_size, fw_autoload_mask); 1426 } 1427 } 1428 } 1429 1430 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev, 1431 uint32_t *fw_autoload_mask) 1432 { 1433 const __le32 *fw_data; 1434 uint32_t fw_size; 1435 const struct sdma_firmware_header_v2_0 *sdma_hdr; 1436 1437 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 1438 adev->sdma.instance[0].fw->data; 1439 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1440 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes)); 1441 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes); 1442 1443 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1444 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask); 1445 1446 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1447 le32_to_cpu(sdma_hdr->ctl_ucode_offset)); 1448 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes); 1449 1450 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1451 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask); 1452 } 1453 1454 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev, 1455 uint32_t *fw_autoload_mask) 1456 { 1457 const __le32 *fw_data; 1458 unsigned fw_size; 1459 const struct mes_firmware_header_v1_0 *mes_hdr; 1460 int pipe, ucode_id, data_id; 1461 1462 for (pipe = 0; pipe < 2; pipe++) { 1463 if (pipe==0) { 1464 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0; 1465 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK; 1466 } else { 1467 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1; 1468 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK; 1469 } 1470 1471 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1472 adev->mes.fw[pipe]->data; 1473 1474 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1475 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); 1476 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); 1477 1478 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1479 ucode_id, fw_data, fw_size, fw_autoload_mask); 1480 1481 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1482 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); 1483 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); 1484 1485 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1486 data_id, fw_data, fw_size, fw_autoload_mask); 1487 } 1488 } 1489 1490 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) 1491 { 1492 uint32_t rlc_g_offset, rlc_g_size; 1493 uint64_t gpu_addr; 1494 uint32_t autoload_fw_id[2]; 1495 1496 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2); 1497 1498 /* RLC autoload sequence 2: copy ucode */ 1499 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id); 1500 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id); 1501 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id); 1502 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id); 1503 1504 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset; 1505 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size; 1506 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset; 1507 1508 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr)); 1509 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr)); 1510 1511 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size); 1512 1513 /* RLC autoload sequence 3: load IMU fw */ 1514 if (adev->gfx.imu.funcs->load_microcode) 1515 adev->gfx.imu.funcs->load_microcode(adev); 1516 /* RLC autoload sequence 4 init IMU fw */ 1517 if (adev->gfx.imu.funcs->setup_imu) 1518 adev->gfx.imu.funcs->setup_imu(adev); 1519 if (adev->gfx.imu.funcs->start_imu) 1520 adev->gfx.imu.funcs->start_imu(adev); 1521 1522 /* RLC autoload sequence 5 disable gpa mode */ 1523 gfx_v11_0_disable_gpa_mode(adev); 1524 1525 return 0; 1526 } 1527 1528 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev) 1529 { 1530 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 1531 uint32_t *ptr; 1532 uint32_t inst; 1533 1534 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 1535 if (!ptr) { 1536 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 1537 adev->gfx.ip_dump_core = NULL; 1538 } else { 1539 adev->gfx.ip_dump_core = ptr; 1540 } 1541 1542 /* Allocate memory for compute queue registers for all the instances */ 1543 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 1544 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1545 adev->gfx.mec.num_queue_per_pipe; 1546 1547 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1548 if (!ptr) { 1549 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 1550 adev->gfx.ip_dump_compute_queues = NULL; 1551 } else { 1552 adev->gfx.ip_dump_compute_queues = ptr; 1553 } 1554 1555 /* Allocate memory for gfx queue registers for all the instances */ 1556 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 1557 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me * 1558 adev->gfx.me.num_queue_per_pipe; 1559 1560 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1561 if (!ptr) { 1562 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n"); 1563 adev->gfx.ip_dump_gfx_queues = NULL; 1564 } else { 1565 adev->gfx.ip_dump_gfx_queues = ptr; 1566 } 1567 } 1568 1569 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) 1570 { 1571 int i, j, k, r, ring_id = 0; 1572 int xcc_id = 0; 1573 struct amdgpu_device *adev = ip_block->adev; 1574 1575 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler); 1576 1577 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1578 case IP_VERSION(11, 0, 0): 1579 case IP_VERSION(11, 0, 2): 1580 case IP_VERSION(11, 0, 3): 1581 adev->gfx.me.num_me = 1; 1582 adev->gfx.me.num_pipe_per_me = 1; 1583 adev->gfx.me.num_queue_per_pipe = 1; 1584 adev->gfx.mec.num_mec = 2; 1585 adev->gfx.mec.num_pipe_per_mec = 4; 1586 adev->gfx.mec.num_queue_per_pipe = 4; 1587 break; 1588 case IP_VERSION(11, 0, 1): 1589 case IP_VERSION(11, 0, 4): 1590 case IP_VERSION(11, 5, 0): 1591 case IP_VERSION(11, 5, 1): 1592 case IP_VERSION(11, 5, 2): 1593 case IP_VERSION(11, 5, 3): 1594 adev->gfx.me.num_me = 1; 1595 adev->gfx.me.num_pipe_per_me = 1; 1596 adev->gfx.me.num_queue_per_pipe = 1; 1597 adev->gfx.mec.num_mec = 1; 1598 adev->gfx.mec.num_pipe_per_mec = 4; 1599 adev->gfx.mec.num_queue_per_pipe = 4; 1600 break; 1601 default: 1602 adev->gfx.me.num_me = 1; 1603 adev->gfx.me.num_pipe_per_me = 1; 1604 adev->gfx.me.num_queue_per_pipe = 1; 1605 adev->gfx.mec.num_mec = 1; 1606 adev->gfx.mec.num_pipe_per_mec = 4; 1607 adev->gfx.mec.num_queue_per_pipe = 8; 1608 break; 1609 } 1610 1611 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1612 case IP_VERSION(11, 0, 0): 1613 case IP_VERSION(11, 0, 2): 1614 case IP_VERSION(11, 0, 3): 1615 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1616 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1617 if (adev->gfx.me_fw_version >= 2280 && 1618 adev->gfx.pfp_fw_version >= 2370 && 1619 adev->gfx.mec_fw_version >= 2450 && 1620 adev->mes.fw_version[0] >= 99) { 1621 adev->gfx.enable_cleaner_shader = true; 1622 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1623 if (r) { 1624 adev->gfx.enable_cleaner_shader = false; 1625 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1626 } 1627 } 1628 break; 1629 default: 1630 adev->gfx.enable_cleaner_shader = false; 1631 break; 1632 } 1633 1634 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */ 1635 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) && 1636 amdgpu_sriov_is_pp_one_vf(adev)) 1637 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG; 1638 1639 /* EOP Event */ 1640 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1641 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT, 1642 &adev->gfx.eop_irq); 1643 if (r) 1644 return r; 1645 1646 /* Bad opcode Event */ 1647 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1648 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR, 1649 &adev->gfx.bad_op_irq); 1650 if (r) 1651 return r; 1652 1653 /* Privileged reg */ 1654 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1655 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT, 1656 &adev->gfx.priv_reg_irq); 1657 if (r) 1658 return r; 1659 1660 /* Privileged inst */ 1661 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1662 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT, 1663 &adev->gfx.priv_inst_irq); 1664 if (r) 1665 return r; 1666 1667 /* FED error */ 1668 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, 1669 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT, 1670 &adev->gfx.rlc_gc_fed_irq); 1671 if (r) 1672 return r; 1673 1674 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1675 1676 gfx_v11_0_me_init(adev); 1677 1678 r = gfx_v11_0_rlc_init(adev); 1679 if (r) { 1680 DRM_ERROR("Failed to init rlc BOs!\n"); 1681 return r; 1682 } 1683 1684 r = gfx_v11_0_mec_init(adev); 1685 if (r) { 1686 DRM_ERROR("Failed to init MEC BOs!\n"); 1687 return r; 1688 } 1689 1690 /* set up the gfx ring */ 1691 for (i = 0; i < adev->gfx.me.num_me; i++) { 1692 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { 1693 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 1694 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j)) 1695 continue; 1696 1697 r = gfx_v11_0_gfx_ring_init(adev, ring_id, 1698 i, k, j); 1699 if (r) 1700 return r; 1701 ring_id++; 1702 } 1703 } 1704 } 1705 1706 ring_id = 0; 1707 /* set up the compute queues - allocate horizontally across pipes */ 1708 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1709 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1710 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1711 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1712 k, j)) 1713 continue; 1714 1715 r = gfx_v11_0_compute_ring_init(adev, ring_id, 1716 i, k, j); 1717 if (r) 1718 return r; 1719 1720 ring_id++; 1721 } 1722 } 1723 } 1724 1725 adev->gfx.gfx_supported_reset = 1726 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 1727 adev->gfx.compute_supported_reset = 1728 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 1729 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1730 case IP_VERSION(11, 0, 0): 1731 case IP_VERSION(11, 0, 2): 1732 case IP_VERSION(11, 0, 3): 1733 if ((adev->gfx.me_fw_version >= 2280) && 1734 (adev->gfx.mec_fw_version >= 2410)) { 1735 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1736 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1737 } 1738 break; 1739 default: 1740 break; 1741 } 1742 1743 if (!adev->enable_mes_kiq) { 1744 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0); 1745 if (r) { 1746 DRM_ERROR("Failed to init KIQ BOs!\n"); 1747 return r; 1748 } 1749 1750 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 1751 if (r) 1752 return r; 1753 } 1754 1755 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0); 1756 if (r) 1757 return r; 1758 1759 /* allocate visible FB for rlc auto-loading fw */ 1760 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 1761 r = gfx_v11_0_rlc_autoload_buffer_init(adev); 1762 if (r) 1763 return r; 1764 } 1765 1766 r = gfx_v11_0_gpu_early_init(adev); 1767 if (r) 1768 return r; 1769 1770 if (amdgpu_gfx_ras_sw_init(adev)) { 1771 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 1772 return -EINVAL; 1773 } 1774 1775 gfx_v11_0_alloc_ip_dump(adev); 1776 1777 r = amdgpu_gfx_sysfs_init(adev); 1778 if (r) 1779 return r; 1780 1781 return 0; 1782 } 1783 1784 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev) 1785 { 1786 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj, 1787 &adev->gfx.pfp.pfp_fw_gpu_addr, 1788 (void **)&adev->gfx.pfp.pfp_fw_ptr); 1789 1790 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj, 1791 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 1792 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 1793 } 1794 1795 static void gfx_v11_0_me_fini(struct amdgpu_device *adev) 1796 { 1797 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, 1798 &adev->gfx.me.me_fw_gpu_addr, 1799 (void **)&adev->gfx.me.me_fw_ptr); 1800 1801 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj, 1802 &adev->gfx.me.me_fw_data_gpu_addr, 1803 (void **)&adev->gfx.me.me_fw_data_ptr); 1804 } 1805 1806 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev) 1807 { 1808 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, 1809 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1810 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1811 } 1812 1813 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block) 1814 { 1815 int i; 1816 struct amdgpu_device *adev = ip_block->adev; 1817 1818 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1819 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 1820 for (i = 0; i < adev->gfx.num_compute_rings; i++) 1821 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 1822 1823 amdgpu_gfx_mqd_sw_fini(adev, 0); 1824 1825 if (!adev->enable_mes_kiq) { 1826 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 1827 amdgpu_gfx_kiq_fini(adev, 0); 1828 } 1829 1830 amdgpu_gfx_cleaner_shader_sw_fini(adev); 1831 1832 gfx_v11_0_pfp_fini(adev); 1833 gfx_v11_0_me_fini(adev); 1834 gfx_v11_0_rlc_fini(adev); 1835 gfx_v11_0_mec_fini(adev); 1836 1837 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 1838 gfx_v11_0_rlc_autoload_buffer_fini(adev); 1839 1840 gfx_v11_0_free_microcode(adev); 1841 1842 amdgpu_gfx_sysfs_fini(adev); 1843 1844 kfree(adev->gfx.ip_dump_core); 1845 kfree(adev->gfx.ip_dump_compute_queues); 1846 kfree(adev->gfx.ip_dump_gfx_queues); 1847 1848 return 0; 1849 } 1850 1851 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 1852 u32 sh_num, u32 instance, int xcc_id) 1853 { 1854 u32 data; 1855 1856 if (instance == 0xffffffff) 1857 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, 1858 INSTANCE_BROADCAST_WRITES, 1); 1859 else 1860 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, 1861 instance); 1862 1863 if (se_num == 0xffffffff) 1864 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1865 1); 1866 else 1867 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 1868 1869 if (sh_num == 0xffffffff) 1870 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES, 1871 1); 1872 else 1873 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num); 1874 1875 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data); 1876 } 1877 1878 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev) 1879 { 1880 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask; 1881 1882 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE); 1883 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask, 1884 CC_GC_SA_UNIT_DISABLE, 1885 SA_DISABLE); 1886 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE); 1887 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask, 1888 GC_USER_SA_UNIT_DISABLE, 1889 SA_DISABLE); 1890 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * 1891 adev->gfx.config.max_shader_engines); 1892 1893 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask)); 1894 } 1895 1896 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev) 1897 { 1898 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask; 1899 u32 rb_mask; 1900 1901 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE); 1902 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask, 1903 CC_RB_BACKEND_DISABLE, 1904 BACKEND_DISABLE); 1905 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE); 1906 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask, 1907 GC_USER_RB_BACKEND_DISABLE, 1908 BACKEND_DISABLE); 1909 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se * 1910 adev->gfx.config.max_shader_engines); 1911 1912 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask)); 1913 } 1914 1915 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev) 1916 { 1917 u32 rb_bitmap_per_sa; 1918 u32 rb_bitmap_width_per_sa; 1919 u32 max_sa; 1920 u32 active_sa_bitmap; 1921 u32 global_active_rb_bitmap; 1922 u32 active_rb_bitmap = 0; 1923 u32 i; 1924 1925 /* query sa bitmap from SA_UNIT_DISABLE registers */ 1926 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev); 1927 /* query rb bitmap from RB_BACKEND_DISABLE registers */ 1928 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev); 1929 1930 /* generate active rb bitmap according to active sa bitmap */ 1931 max_sa = adev->gfx.config.max_shader_engines * 1932 adev->gfx.config.max_sh_per_se; 1933 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se / 1934 adev->gfx.config.max_sh_per_se; 1935 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa); 1936 1937 for (i = 0; i < max_sa; i++) { 1938 if (active_sa_bitmap & (1 << i)) 1939 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa)); 1940 } 1941 1942 active_rb_bitmap &= global_active_rb_bitmap; 1943 adev->gfx.config.backend_enable_mask = active_rb_bitmap; 1944 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap); 1945 } 1946 1947 #define DEFAULT_SH_MEM_BASES (0x6000) 1948 #define LDS_APP_BASE 0x1 1949 #define SCRATCH_APP_BASE 0x2 1950 1951 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev) 1952 { 1953 int i; 1954 uint32_t sh_mem_bases; 1955 uint32_t data; 1956 1957 /* 1958 * Configure apertures: 1959 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 1960 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 1961 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 1962 */ 1963 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) | 1964 SCRATCH_APP_BASE; 1965 1966 mutex_lock(&adev->srbm_mutex); 1967 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 1968 soc21_grbm_select(adev, 0, 0, 0, i); 1969 /* CP and shaders */ 1970 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 1971 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); 1972 1973 /* Enable trap for each kfd vmid. */ 1974 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); 1975 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 1976 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data); 1977 } 1978 soc21_grbm_select(adev, 0, 0, 0, 0); 1979 mutex_unlock(&adev->srbm_mutex); 1980 1981 /* 1982 * Initialize all compute VMIDs to have no GDS, GWS, or OA 1983 * access. These should be enabled by FW for target VMIDs. 1984 */ 1985 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 1986 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0); 1987 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0); 1988 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0); 1989 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0); 1990 } 1991 } 1992 1993 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev) 1994 { 1995 int vmid; 1996 1997 /* 1998 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 1999 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2000 * the driver can enable them for graphics. VMID0 should maintain 2001 * access so that HWS firmware can save/restore entries. 2002 */ 2003 for (vmid = 1; vmid < 16; vmid++) { 2004 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0); 2005 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0); 2006 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0); 2007 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0); 2008 } 2009 } 2010 2011 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev) 2012 { 2013 /* TODO: harvest feature to be added later. */ 2014 } 2015 2016 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev) 2017 { 2018 /* TCCs are global (not instanced). */ 2019 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) | 2020 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE); 2021 2022 adev->gfx.config.tcc_disabled_mask = 2023 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) | 2024 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16); 2025 } 2026 2027 static void gfx_v11_0_constants_init(struct amdgpu_device *adev) 2028 { 2029 u32 tmp; 2030 int i; 2031 2032 if (!amdgpu_sriov_vf(adev)) 2033 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2034 2035 gfx_v11_0_setup_rb(adev); 2036 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info); 2037 gfx_v11_0_get_tcc_info(adev); 2038 adev->gfx.config.pa_sc_tile_steering_override = 0; 2039 2040 /* Set whether texture coordinate truncation is conformant. */ 2041 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); 2042 adev->gfx.config.ta_cntl2_truncate_coord_mode = 2043 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); 2044 2045 /* XXX SH_MEM regs */ 2046 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2047 mutex_lock(&adev->srbm_mutex); 2048 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2049 soc21_grbm_select(adev, 0, 0, 0, i); 2050 /* CP and shaders */ 2051 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2052 if (i != 0) { 2053 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2054 (adev->gmc.private_aperture_start >> 48)); 2055 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2056 (adev->gmc.shared_aperture_start >> 48)); 2057 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp); 2058 } 2059 } 2060 soc21_grbm_select(adev, 0, 0, 0, 0); 2061 2062 mutex_unlock(&adev->srbm_mutex); 2063 2064 gfx_v11_0_init_compute_vmid(adev); 2065 gfx_v11_0_init_gds_vmid(adev); 2066 } 2067 2068 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev, 2069 int me, int pipe) 2070 { 2071 if (me != 0) 2072 return 0; 2073 2074 switch (pipe) { 2075 case 0: 2076 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 2077 case 1: 2078 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 2079 default: 2080 return 0; 2081 } 2082 } 2083 2084 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev, 2085 int me, int pipe) 2086 { 2087 /* 2088 * amdgpu controls only the first MEC. That's why this function only 2089 * handles the setting of interrupts for this specific MEC. All other 2090 * pipes' interrupts are set by amdkfd. 2091 */ 2092 if (me != 1) 2093 return 0; 2094 2095 switch (pipe) { 2096 case 0: 2097 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 2098 case 1: 2099 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 2100 case 2: 2101 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 2102 case 3: 2103 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 2104 default: 2105 return 0; 2106 } 2107 } 2108 2109 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2110 bool enable) 2111 { 2112 u32 tmp, cp_int_cntl_reg; 2113 int i, j; 2114 2115 if (amdgpu_sriov_vf(adev)) 2116 return; 2117 2118 for (i = 0; i < adev->gfx.me.num_me; i++) { 2119 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 2120 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 2121 2122 if (cp_int_cntl_reg) { 2123 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 2124 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 2125 enable ? 1 : 0); 2126 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 2127 enable ? 1 : 0); 2128 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 2129 enable ? 1 : 0); 2130 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 2131 enable ? 1 : 0); 2132 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp); 2133 } 2134 } 2135 } 2136 } 2137 2138 static int gfx_v11_0_init_csb(struct amdgpu_device *adev) 2139 { 2140 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2141 2142 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI, 2143 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2144 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO, 2145 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2146 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); 2147 2148 return 0; 2149 } 2150 2151 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev) 2152 { 2153 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL); 2154 2155 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 2156 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp); 2157 } 2158 2159 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev) 2160 { 2161 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2162 udelay(50); 2163 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2164 udelay(50); 2165 } 2166 2167 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev, 2168 bool enable) 2169 { 2170 uint32_t rlc_pg_cntl; 2171 2172 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 2173 2174 if (!enable) { 2175 /* RLC_PG_CNTL[23] = 0 (default) 2176 * RLC will wait for handshake acks with SMU 2177 * GFXOFF will be enabled 2178 * RLC_PG_CNTL[23] = 1 2179 * RLC will not issue any message to SMU 2180 * hence no handshake between SMU & RLC 2181 * GFXOFF will be disabled 2182 */ 2183 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2184 } else 2185 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2186 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl); 2187 } 2188 2189 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev) 2190 { 2191 /* TODO: enable rlc & smu handshake until smu 2192 * and gfxoff feature works as expected */ 2193 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK)) 2194 gfx_v11_0_rlc_smu_handshake_cntl(adev, false); 2195 2196 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2197 udelay(50); 2198 } 2199 2200 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev) 2201 { 2202 uint32_t tmp; 2203 2204 /* enable Save Restore Machine */ 2205 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL)); 2206 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2207 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK; 2208 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp); 2209 } 2210 2211 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev) 2212 { 2213 const struct rlc_firmware_header_v2_0 *hdr; 2214 const __le32 *fw_data; 2215 unsigned i, fw_size; 2216 2217 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2218 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2219 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2220 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2221 2222 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, 2223 RLCG_UCODE_LOADING_START_ADDRESS); 2224 2225 for (i = 0; i < fw_size; i++) 2226 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, 2227 le32_to_cpup(fw_data++)); 2228 2229 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2230 } 2231 2232 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev) 2233 { 2234 const struct rlc_firmware_header_v2_2 *hdr; 2235 const __le32 *fw_data; 2236 unsigned i, fw_size; 2237 u32 tmp; 2238 2239 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 2240 2241 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2242 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes)); 2243 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4; 2244 2245 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0); 2246 2247 for (i = 0; i < fw_size; i++) { 2248 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2249 msleep(1); 2250 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA, 2251 le32_to_cpup(fw_data++)); 2252 } 2253 2254 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2255 2256 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2257 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes)); 2258 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4; 2259 2260 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0); 2261 for (i = 0; i < fw_size; i++) { 2262 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2263 msleep(1); 2264 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA, 2265 le32_to_cpup(fw_data++)); 2266 } 2267 2268 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2269 2270 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL); 2271 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1); 2272 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); 2273 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp); 2274 } 2275 2276 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev) 2277 { 2278 const struct rlc_firmware_header_v2_3 *hdr; 2279 const __le32 *fw_data; 2280 unsigned i, fw_size; 2281 u32 tmp; 2282 2283 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; 2284 2285 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2286 le32_to_cpu(hdr->rlcp_ucode_offset_bytes)); 2287 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4; 2288 2289 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0); 2290 2291 for (i = 0; i < fw_size; i++) { 2292 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2293 msleep(1); 2294 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA, 2295 le32_to_cpup(fw_data++)); 2296 } 2297 2298 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version); 2299 2300 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE); 2301 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1); 2302 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp); 2303 2304 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2305 le32_to_cpu(hdr->rlcv_ucode_offset_bytes)); 2306 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4; 2307 2308 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0); 2309 2310 for (i = 0; i < fw_size; i++) { 2311 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2312 msleep(1); 2313 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA, 2314 le32_to_cpup(fw_data++)); 2315 } 2316 2317 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version); 2318 2319 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL); 2320 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1); 2321 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp); 2322 } 2323 2324 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev) 2325 { 2326 const struct rlc_firmware_header_v2_0 *hdr; 2327 uint16_t version_major; 2328 uint16_t version_minor; 2329 2330 if (!adev->gfx.rlc_fw) 2331 return -EINVAL; 2332 2333 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2334 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2335 2336 version_major = le16_to_cpu(hdr->header.header_version_major); 2337 version_minor = le16_to_cpu(hdr->header.header_version_minor); 2338 2339 if (version_major == 2) { 2340 gfx_v11_0_load_rlcg_microcode(adev); 2341 if (amdgpu_dpm == 1) { 2342 if (version_minor >= 2) 2343 gfx_v11_0_load_rlc_iram_dram_microcode(adev); 2344 if (version_minor == 3) 2345 gfx_v11_0_load_rlcp_rlcv_microcode(adev); 2346 } 2347 2348 return 0; 2349 } 2350 2351 return -EINVAL; 2352 } 2353 2354 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev) 2355 { 2356 int r; 2357 2358 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 2359 gfx_v11_0_init_csb(adev); 2360 2361 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ 2362 gfx_v11_0_rlc_enable_srm(adev); 2363 } else { 2364 if (amdgpu_sriov_vf(adev)) { 2365 gfx_v11_0_init_csb(adev); 2366 return 0; 2367 } 2368 2369 adev->gfx.rlc.funcs->stop(adev); 2370 2371 /* disable CG */ 2372 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0); 2373 2374 /* disable PG */ 2375 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0); 2376 2377 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2378 /* legacy rlc firmware loading */ 2379 r = gfx_v11_0_rlc_load_microcode(adev); 2380 if (r) 2381 return r; 2382 } 2383 2384 gfx_v11_0_init_csb(adev); 2385 2386 adev->gfx.rlc.funcs->start(adev); 2387 } 2388 return 0; 2389 } 2390 2391 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr) 2392 { 2393 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2394 uint32_t tmp; 2395 int i; 2396 2397 /* Trigger an invalidation of the L1 instruction caches */ 2398 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2399 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2400 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2401 2402 /* Wait for invalidation complete */ 2403 for (i = 0; i < usec_timeout; i++) { 2404 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2405 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2406 INVALIDATE_CACHE_COMPLETE)) 2407 break; 2408 udelay(1); 2409 } 2410 2411 if (i >= usec_timeout) { 2412 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2413 return -EINVAL; 2414 } 2415 2416 if (amdgpu_emu_mode == 1) 2417 adev->hdp.funcs->flush_hdp(adev, NULL); 2418 2419 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2420 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2421 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2422 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2423 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2424 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2425 2426 /* Program me ucode address into intruction cache address register */ 2427 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2428 lower_32_bits(addr) & 0xFFFFF000); 2429 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2430 upper_32_bits(addr)); 2431 2432 return 0; 2433 } 2434 2435 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr) 2436 { 2437 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2438 uint32_t tmp; 2439 int i; 2440 2441 /* Trigger an invalidation of the L1 instruction caches */ 2442 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2443 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2444 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2445 2446 /* Wait for invalidation complete */ 2447 for (i = 0; i < usec_timeout; i++) { 2448 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2449 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2450 INVALIDATE_CACHE_COMPLETE)) 2451 break; 2452 udelay(1); 2453 } 2454 2455 if (i >= usec_timeout) { 2456 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2457 return -EINVAL; 2458 } 2459 2460 if (amdgpu_emu_mode == 1) 2461 adev->hdp.funcs->flush_hdp(adev, NULL); 2462 2463 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2464 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2465 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2466 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2467 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2468 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2469 2470 /* Program pfp ucode address into intruction cache address register */ 2471 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2472 lower_32_bits(addr) & 0xFFFFF000); 2473 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2474 upper_32_bits(addr)); 2475 2476 return 0; 2477 } 2478 2479 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr) 2480 { 2481 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2482 uint32_t tmp; 2483 int i; 2484 2485 /* Trigger an invalidation of the L1 instruction caches */ 2486 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2487 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2488 2489 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2490 2491 /* Wait for invalidation complete */ 2492 for (i = 0; i < usec_timeout; i++) { 2493 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2494 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2495 INVALIDATE_CACHE_COMPLETE)) 2496 break; 2497 udelay(1); 2498 } 2499 2500 if (i >= usec_timeout) { 2501 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2502 return -EINVAL; 2503 } 2504 2505 if (amdgpu_emu_mode == 1) 2506 adev->hdp.funcs->flush_hdp(adev, NULL); 2507 2508 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2509 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2510 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2511 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2512 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2513 2514 /* Program mec1 ucode address into intruction cache address register */ 2515 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, 2516 lower_32_bits(addr) & 0xFFFFF000); 2517 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2518 upper_32_bits(addr)); 2519 2520 return 0; 2521 } 2522 2523 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2524 { 2525 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2526 uint32_t tmp; 2527 unsigned i, pipe_id; 2528 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2529 2530 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 2531 adev->gfx.pfp_fw->data; 2532 2533 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2534 lower_32_bits(addr)); 2535 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2536 upper_32_bits(addr)); 2537 2538 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2539 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2540 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2541 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2542 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2543 2544 /* 2545 * Programming any of the CP_PFP_IC_BASE registers 2546 * forces invalidation of the ME L1 I$. Wait for the 2547 * invalidation complete 2548 */ 2549 for (i = 0; i < usec_timeout; i++) { 2550 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2551 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2552 INVALIDATE_CACHE_COMPLETE)) 2553 break; 2554 udelay(1); 2555 } 2556 2557 if (i >= usec_timeout) { 2558 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2559 return -EINVAL; 2560 } 2561 2562 /* Prime the L1 instruction caches */ 2563 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2564 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 2565 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2566 /* Waiting for cache primed*/ 2567 for (i = 0; i < usec_timeout; i++) { 2568 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2569 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2570 ICACHE_PRIMED)) 2571 break; 2572 udelay(1); 2573 } 2574 2575 if (i >= usec_timeout) { 2576 dev_err(adev->dev, "failed to prime instruction cache\n"); 2577 return -EINVAL; 2578 } 2579 2580 mutex_lock(&adev->srbm_mutex); 2581 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2582 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2583 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 2584 (pfp_hdr->ucode_start_addr_hi << 30) | 2585 (pfp_hdr->ucode_start_addr_lo >> 2)); 2586 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 2587 pfp_hdr->ucode_start_addr_hi >> 2); 2588 2589 /* 2590 * Program CP_ME_CNTL to reset given PIPE to take 2591 * effect of CP_PFP_PRGRM_CNTR_START. 2592 */ 2593 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2594 if (pipe_id == 0) 2595 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2596 PFP_PIPE0_RESET, 1); 2597 else 2598 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2599 PFP_PIPE1_RESET, 1); 2600 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2601 2602 /* Clear pfp pipe0 reset bit. */ 2603 if (pipe_id == 0) 2604 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2605 PFP_PIPE0_RESET, 0); 2606 else 2607 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2608 PFP_PIPE1_RESET, 0); 2609 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2610 2611 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 2612 lower_32_bits(addr2)); 2613 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 2614 upper_32_bits(addr2)); 2615 } 2616 soc21_grbm_select(adev, 0, 0, 0, 0); 2617 mutex_unlock(&adev->srbm_mutex); 2618 2619 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2620 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2621 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2622 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2623 2624 /* Invalidate the data caches */ 2625 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2626 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2627 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2628 2629 for (i = 0; i < usec_timeout; i++) { 2630 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2631 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2632 INVALIDATE_DCACHE_COMPLETE)) 2633 break; 2634 udelay(1); 2635 } 2636 2637 if (i >= usec_timeout) { 2638 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2639 return -EINVAL; 2640 } 2641 2642 return 0; 2643 } 2644 2645 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2646 { 2647 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2648 uint32_t tmp; 2649 unsigned i, pipe_id; 2650 const struct gfx_firmware_header_v2_0 *me_hdr; 2651 2652 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2653 adev->gfx.me_fw->data; 2654 2655 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2656 lower_32_bits(addr)); 2657 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2658 upper_32_bits(addr)); 2659 2660 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2661 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2662 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2663 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2664 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2665 2666 /* 2667 * Programming any of the CP_ME_IC_BASE registers 2668 * forces invalidation of the ME L1 I$. Wait for the 2669 * invalidation complete 2670 */ 2671 for (i = 0; i < usec_timeout; i++) { 2672 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2673 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2674 INVALIDATE_CACHE_COMPLETE)) 2675 break; 2676 udelay(1); 2677 } 2678 2679 if (i >= usec_timeout) { 2680 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2681 return -EINVAL; 2682 } 2683 2684 /* Prime the instruction caches */ 2685 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2686 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 2687 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2688 2689 /* Waiting for instruction cache primed*/ 2690 for (i = 0; i < usec_timeout; i++) { 2691 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2692 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2693 ICACHE_PRIMED)) 2694 break; 2695 udelay(1); 2696 } 2697 2698 if (i >= usec_timeout) { 2699 dev_err(adev->dev, "failed to prime instruction cache\n"); 2700 return -EINVAL; 2701 } 2702 2703 mutex_lock(&adev->srbm_mutex); 2704 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2705 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2706 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 2707 (me_hdr->ucode_start_addr_hi << 30) | 2708 (me_hdr->ucode_start_addr_lo >> 2) ); 2709 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 2710 me_hdr->ucode_start_addr_hi>>2); 2711 2712 /* 2713 * Program CP_ME_CNTL to reset given PIPE to take 2714 * effect of CP_PFP_PRGRM_CNTR_START. 2715 */ 2716 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2717 if (pipe_id == 0) 2718 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2719 ME_PIPE0_RESET, 1); 2720 else 2721 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2722 ME_PIPE1_RESET, 1); 2723 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2724 2725 /* Clear pfp pipe0 reset bit. */ 2726 if (pipe_id == 0) 2727 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2728 ME_PIPE0_RESET, 0); 2729 else 2730 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2731 ME_PIPE1_RESET, 0); 2732 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2733 2734 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 2735 lower_32_bits(addr2)); 2736 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 2737 upper_32_bits(addr2)); 2738 } 2739 soc21_grbm_select(adev, 0, 0, 0, 0); 2740 mutex_unlock(&adev->srbm_mutex); 2741 2742 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2743 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2744 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2745 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2746 2747 /* Invalidate the data caches */ 2748 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2749 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2750 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2751 2752 for (i = 0; i < usec_timeout; i++) { 2753 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2754 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2755 INVALIDATE_DCACHE_COMPLETE)) 2756 break; 2757 udelay(1); 2758 } 2759 2760 if (i >= usec_timeout) { 2761 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2762 return -EINVAL; 2763 } 2764 2765 return 0; 2766 } 2767 2768 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2769 { 2770 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2771 uint32_t tmp; 2772 unsigned i; 2773 const struct gfx_firmware_header_v2_0 *mec_hdr; 2774 2775 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2776 adev->gfx.mec_fw->data; 2777 2778 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2779 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 2780 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2781 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2782 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2783 2784 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 2785 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 2786 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 2787 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 2788 2789 mutex_lock(&adev->srbm_mutex); 2790 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2791 soc21_grbm_select(adev, 1, i, 0, 0); 2792 2793 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2); 2794 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 2795 upper_32_bits(addr2)); 2796 2797 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 2798 mec_hdr->ucode_start_addr_lo >> 2 | 2799 mec_hdr->ucode_start_addr_hi << 30); 2800 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2801 mec_hdr->ucode_start_addr_hi >> 2); 2802 2803 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr); 2804 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2805 upper_32_bits(addr)); 2806 } 2807 mutex_unlock(&adev->srbm_mutex); 2808 soc21_grbm_select(adev, 0, 0, 0, 0); 2809 2810 /* Trigger an invalidation of the L1 instruction caches */ 2811 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2812 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2813 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 2814 2815 /* Wait for invalidation complete */ 2816 for (i = 0; i < usec_timeout; i++) { 2817 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2818 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 2819 INVALIDATE_DCACHE_COMPLETE)) 2820 break; 2821 udelay(1); 2822 } 2823 2824 if (i >= usec_timeout) { 2825 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2826 return -EINVAL; 2827 } 2828 2829 /* Trigger an invalidation of the L1 instruction caches */ 2830 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2831 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2832 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2833 2834 /* Wait for invalidation complete */ 2835 for (i = 0; i < usec_timeout; i++) { 2836 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2837 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2838 INVALIDATE_CACHE_COMPLETE)) 2839 break; 2840 udelay(1); 2841 } 2842 2843 if (i >= usec_timeout) { 2844 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2845 return -EINVAL; 2846 } 2847 2848 return 0; 2849 } 2850 2851 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev) 2852 { 2853 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2854 const struct gfx_firmware_header_v2_0 *me_hdr; 2855 const struct gfx_firmware_header_v2_0 *mec_hdr; 2856 uint32_t pipe_id, tmp; 2857 2858 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2859 adev->gfx.mec_fw->data; 2860 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2861 adev->gfx.me_fw->data; 2862 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 2863 adev->gfx.pfp_fw->data; 2864 2865 /* config pfp program start addr */ 2866 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 2867 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2868 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 2869 (pfp_hdr->ucode_start_addr_hi << 30) | 2870 (pfp_hdr->ucode_start_addr_lo >> 2)); 2871 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 2872 pfp_hdr->ucode_start_addr_hi >> 2); 2873 } 2874 soc21_grbm_select(adev, 0, 0, 0, 0); 2875 2876 /* reset pfp pipe */ 2877 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2878 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1); 2879 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1); 2880 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2881 2882 /* clear pfp pipe reset */ 2883 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0); 2884 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0); 2885 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2886 2887 /* config me program start addr */ 2888 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 2889 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2890 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 2891 (me_hdr->ucode_start_addr_hi << 30) | 2892 (me_hdr->ucode_start_addr_lo >> 2) ); 2893 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 2894 me_hdr->ucode_start_addr_hi>>2); 2895 } 2896 soc21_grbm_select(adev, 0, 0, 0, 0); 2897 2898 /* reset me pipe */ 2899 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2900 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1); 2901 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1); 2902 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2903 2904 /* clear me pipe reset */ 2905 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0); 2906 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0); 2907 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2908 2909 /* config mec program start addr */ 2910 for (pipe_id = 0; pipe_id < 4; pipe_id++) { 2911 soc21_grbm_select(adev, 1, pipe_id, 0, 0); 2912 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 2913 mec_hdr->ucode_start_addr_lo >> 2 | 2914 mec_hdr->ucode_start_addr_hi << 30); 2915 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2916 mec_hdr->ucode_start_addr_hi >> 2); 2917 } 2918 soc21_grbm_select(adev, 0, 0, 0, 0); 2919 2920 /* reset mec pipe */ 2921 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 2922 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1); 2923 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1); 2924 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1); 2925 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1); 2926 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 2927 2928 /* clear mec pipe reset */ 2929 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); 2930 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); 2931 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); 2932 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); 2933 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 2934 } 2935 2936 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) 2937 { 2938 uint32_t cp_status; 2939 uint32_t bootload_status; 2940 int i, r; 2941 uint64_t addr, addr2; 2942 2943 for (i = 0; i < adev->usec_timeout; i++) { 2944 cp_status = RREG32_SOC15(GC, 0, regCP_STAT); 2945 2946 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 2947 IP_VERSION(11, 0, 1) || 2948 amdgpu_ip_version(adev, GC_HWIP, 0) == 2949 IP_VERSION(11, 0, 4) || 2950 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) || 2951 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) || 2952 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || 2953 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3)) 2954 bootload_status = RREG32_SOC15(GC, 0, 2955 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); 2956 else 2957 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS); 2958 2959 if ((cp_status == 0) && 2960 (REG_GET_FIELD(bootload_status, 2961 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) { 2962 break; 2963 } 2964 udelay(1); 2965 } 2966 2967 if (i >= adev->usec_timeout) { 2968 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n"); 2969 return -ETIMEDOUT; 2970 } 2971 2972 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 2973 if (adev->gfx.rs64_enable) { 2974 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 2975 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset; 2976 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 2977 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset; 2978 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2); 2979 if (r) 2980 return r; 2981 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 2982 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset; 2983 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 2984 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset; 2985 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2); 2986 if (r) 2987 return r; 2988 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 2989 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset; 2990 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 2991 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset; 2992 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2); 2993 if (r) 2994 return r; 2995 } else { 2996 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 2997 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset; 2998 r = gfx_v11_0_config_me_cache(adev, addr); 2999 if (r) 3000 return r; 3001 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3002 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset; 3003 r = gfx_v11_0_config_pfp_cache(adev, addr); 3004 if (r) 3005 return r; 3006 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3007 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset; 3008 r = gfx_v11_0_config_mec_cache(adev, addr); 3009 if (r) 3010 return r; 3011 } 3012 } 3013 3014 return 0; 3015 } 3016 3017 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3018 { 3019 int i; 3020 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3021 3022 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3023 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3024 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3025 3026 for (i = 0; i < adev->usec_timeout; i++) { 3027 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0) 3028 break; 3029 udelay(1); 3030 } 3031 3032 if (i >= adev->usec_timeout) 3033 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt"); 3034 3035 return 0; 3036 } 3037 3038 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) 3039 { 3040 int r; 3041 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3042 const __le32 *fw_data; 3043 unsigned i, fw_size; 3044 3045 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3046 adev->gfx.pfp_fw->data; 3047 3048 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3049 3050 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3051 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3052 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes); 3053 3054 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes, 3055 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3056 &adev->gfx.pfp.pfp_fw_obj, 3057 &adev->gfx.pfp.pfp_fw_gpu_addr, 3058 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3059 if (r) { 3060 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r); 3061 gfx_v11_0_pfp_fini(adev); 3062 return r; 3063 } 3064 3065 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size); 3066 3067 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3068 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3069 3070 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr); 3071 3072 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0); 3073 3074 for (i = 0; i < pfp_hdr->jt_size; i++) 3075 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA, 3076 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i)); 3077 3078 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3079 3080 return 0; 3081 } 3082 3083 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) 3084 { 3085 int r; 3086 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3087 const __le32 *fw_ucode, *fw_data; 3088 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3089 uint32_t tmp; 3090 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3091 3092 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3093 adev->gfx.pfp_fw->data; 3094 3095 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3096 3097 /* instruction */ 3098 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data + 3099 le32_to_cpu(pfp_hdr->ucode_offset_bytes)); 3100 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes); 3101 /* data */ 3102 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3103 le32_to_cpu(pfp_hdr->data_offset_bytes)); 3104 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes); 3105 3106 /* 64kb align */ 3107 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3108 64 * 1024, 3109 AMDGPU_GEM_DOMAIN_VRAM | 3110 AMDGPU_GEM_DOMAIN_GTT, 3111 &adev->gfx.pfp.pfp_fw_obj, 3112 &adev->gfx.pfp.pfp_fw_gpu_addr, 3113 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3114 if (r) { 3115 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r); 3116 gfx_v11_0_pfp_fini(adev); 3117 return r; 3118 } 3119 3120 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3121 64 * 1024, 3122 AMDGPU_GEM_DOMAIN_VRAM | 3123 AMDGPU_GEM_DOMAIN_GTT, 3124 &adev->gfx.pfp.pfp_fw_data_obj, 3125 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 3126 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 3127 if (r) { 3128 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r); 3129 gfx_v11_0_pfp_fini(adev); 3130 return r; 3131 } 3132 3133 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); 3134 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); 3135 3136 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3137 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); 3138 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3139 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj); 3140 3141 if (amdgpu_emu_mode == 1) 3142 adev->hdp.funcs->flush_hdp(adev, NULL); 3143 3144 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 3145 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3146 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 3147 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3148 3149 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 3150 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 3151 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 3152 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 3153 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 3154 3155 /* 3156 * Programming any of the CP_PFP_IC_BASE registers 3157 * forces invalidation of the ME L1 I$. Wait for the 3158 * invalidation complete 3159 */ 3160 for (i = 0; i < usec_timeout; i++) { 3161 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3162 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3163 INVALIDATE_CACHE_COMPLETE)) 3164 break; 3165 udelay(1); 3166 } 3167 3168 if (i >= usec_timeout) { 3169 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3170 return -EINVAL; 3171 } 3172 3173 /* Prime the L1 instruction caches */ 3174 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3175 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 3176 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 3177 /* Waiting for cache primed*/ 3178 for (i = 0; i < usec_timeout; i++) { 3179 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3180 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3181 ICACHE_PRIMED)) 3182 break; 3183 udelay(1); 3184 } 3185 3186 if (i >= usec_timeout) { 3187 dev_err(adev->dev, "failed to prime instruction cache\n"); 3188 return -EINVAL; 3189 } 3190 3191 mutex_lock(&adev->srbm_mutex); 3192 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3193 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3194 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3195 (pfp_hdr->ucode_start_addr_hi << 30) | 3196 (pfp_hdr->ucode_start_addr_lo >> 2) ); 3197 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3198 pfp_hdr->ucode_start_addr_hi>>2); 3199 3200 /* 3201 * Program CP_ME_CNTL to reset given PIPE to take 3202 * effect of CP_PFP_PRGRM_CNTR_START. 3203 */ 3204 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3205 if (pipe_id == 0) 3206 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3207 PFP_PIPE0_RESET, 1); 3208 else 3209 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3210 PFP_PIPE1_RESET, 1); 3211 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3212 3213 /* Clear pfp pipe0 reset bit. */ 3214 if (pipe_id == 0) 3215 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3216 PFP_PIPE0_RESET, 0); 3217 else 3218 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3219 PFP_PIPE1_RESET, 0); 3220 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3221 3222 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 3223 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3224 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 3225 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3226 } 3227 soc21_grbm_select(adev, 0, 0, 0, 0); 3228 mutex_unlock(&adev->srbm_mutex); 3229 3230 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3231 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3232 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3233 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3234 3235 /* Invalidate the data caches */ 3236 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3237 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3238 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3239 3240 for (i = 0; i < usec_timeout; i++) { 3241 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3242 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3243 INVALIDATE_DCACHE_COMPLETE)) 3244 break; 3245 udelay(1); 3246 } 3247 3248 if (i >= usec_timeout) { 3249 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3250 return -EINVAL; 3251 } 3252 3253 return 0; 3254 } 3255 3256 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) 3257 { 3258 int r; 3259 const struct gfx_firmware_header_v1_0 *me_hdr; 3260 const __le32 *fw_data; 3261 unsigned i, fw_size; 3262 3263 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3264 adev->gfx.me_fw->data; 3265 3266 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3267 3268 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3269 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3270 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes); 3271 3272 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes, 3273 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3274 &adev->gfx.me.me_fw_obj, 3275 &adev->gfx.me.me_fw_gpu_addr, 3276 (void **)&adev->gfx.me.me_fw_ptr); 3277 if (r) { 3278 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r); 3279 gfx_v11_0_me_fini(adev); 3280 return r; 3281 } 3282 3283 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size); 3284 3285 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3286 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3287 3288 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr); 3289 3290 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0); 3291 3292 for (i = 0; i < me_hdr->jt_size; i++) 3293 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA, 3294 le32_to_cpup(fw_data + me_hdr->jt_offset + i)); 3295 3296 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); 3297 3298 return 0; 3299 } 3300 3301 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) 3302 { 3303 int r; 3304 const struct gfx_firmware_header_v2_0 *me_hdr; 3305 const __le32 *fw_ucode, *fw_data; 3306 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3307 uint32_t tmp; 3308 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3309 3310 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3311 adev->gfx.me_fw->data; 3312 3313 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3314 3315 /* instruction */ 3316 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data + 3317 le32_to_cpu(me_hdr->ucode_offset_bytes)); 3318 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes); 3319 /* data */ 3320 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3321 le32_to_cpu(me_hdr->data_offset_bytes)); 3322 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes); 3323 3324 /* 64kb align*/ 3325 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3326 64 * 1024, 3327 AMDGPU_GEM_DOMAIN_VRAM | 3328 AMDGPU_GEM_DOMAIN_GTT, 3329 &adev->gfx.me.me_fw_obj, 3330 &adev->gfx.me.me_fw_gpu_addr, 3331 (void **)&adev->gfx.me.me_fw_ptr); 3332 if (r) { 3333 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r); 3334 gfx_v11_0_me_fini(adev); 3335 return r; 3336 } 3337 3338 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3339 64 * 1024, 3340 AMDGPU_GEM_DOMAIN_VRAM | 3341 AMDGPU_GEM_DOMAIN_GTT, 3342 &adev->gfx.me.me_fw_data_obj, 3343 &adev->gfx.me.me_fw_data_gpu_addr, 3344 (void **)&adev->gfx.me.me_fw_data_ptr); 3345 if (r) { 3346 dev_err(adev->dev, "(%d) failed to create me data bo\n", r); 3347 gfx_v11_0_pfp_fini(adev); 3348 return r; 3349 } 3350 3351 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); 3352 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); 3353 3354 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3355 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); 3356 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3357 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj); 3358 3359 if (amdgpu_emu_mode == 1) 3360 adev->hdp.funcs->flush_hdp(adev, NULL); 3361 3362 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 3363 lower_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3364 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 3365 upper_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3366 3367 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 3368 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 3369 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 3370 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 3371 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 3372 3373 /* 3374 * Programming any of the CP_ME_IC_BASE registers 3375 * forces invalidation of the ME L1 I$. Wait for the 3376 * invalidation complete 3377 */ 3378 for (i = 0; i < usec_timeout; i++) { 3379 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3380 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3381 INVALIDATE_CACHE_COMPLETE)) 3382 break; 3383 udelay(1); 3384 } 3385 3386 if (i >= usec_timeout) { 3387 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3388 return -EINVAL; 3389 } 3390 3391 /* Prime the instruction caches */ 3392 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3393 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 3394 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 3395 3396 /* Waiting for instruction cache primed*/ 3397 for (i = 0; i < usec_timeout; i++) { 3398 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3399 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3400 ICACHE_PRIMED)) 3401 break; 3402 udelay(1); 3403 } 3404 3405 if (i >= usec_timeout) { 3406 dev_err(adev->dev, "failed to prime instruction cache\n"); 3407 return -EINVAL; 3408 } 3409 3410 mutex_lock(&adev->srbm_mutex); 3411 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3412 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3413 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3414 (me_hdr->ucode_start_addr_hi << 30) | 3415 (me_hdr->ucode_start_addr_lo >> 2) ); 3416 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3417 me_hdr->ucode_start_addr_hi>>2); 3418 3419 /* 3420 * Program CP_ME_CNTL to reset given PIPE to take 3421 * effect of CP_PFP_PRGRM_CNTR_START. 3422 */ 3423 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3424 if (pipe_id == 0) 3425 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3426 ME_PIPE0_RESET, 1); 3427 else 3428 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3429 ME_PIPE1_RESET, 1); 3430 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3431 3432 /* Clear pfp pipe0 reset bit. */ 3433 if (pipe_id == 0) 3434 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3435 ME_PIPE0_RESET, 0); 3436 else 3437 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3438 ME_PIPE1_RESET, 0); 3439 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3440 3441 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 3442 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3443 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 3444 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3445 } 3446 soc21_grbm_select(adev, 0, 0, 0, 0); 3447 mutex_unlock(&adev->srbm_mutex); 3448 3449 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3450 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3451 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3452 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3453 3454 /* Invalidate the data caches */ 3455 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3456 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3457 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3458 3459 for (i = 0; i < usec_timeout; i++) { 3460 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3461 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3462 INVALIDATE_DCACHE_COMPLETE)) 3463 break; 3464 udelay(1); 3465 } 3466 3467 if (i >= usec_timeout) { 3468 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3469 return -EINVAL; 3470 } 3471 3472 return 0; 3473 } 3474 3475 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3476 { 3477 int r; 3478 3479 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw) 3480 return -EINVAL; 3481 3482 gfx_v11_0_cp_gfx_enable(adev, false); 3483 3484 if (adev->gfx.rs64_enable) 3485 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev); 3486 else 3487 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev); 3488 if (r) { 3489 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r); 3490 return r; 3491 } 3492 3493 if (adev->gfx.rs64_enable) 3494 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev); 3495 else 3496 r = gfx_v11_0_cp_gfx_load_me_microcode(adev); 3497 if (r) { 3498 dev_err(adev->dev, "(%d) failed to load me fw\n", r); 3499 return r; 3500 } 3501 3502 return 0; 3503 } 3504 3505 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev) 3506 { 3507 struct amdgpu_ring *ring; 3508 const struct cs_section_def *sect = NULL; 3509 const struct cs_extent_def *ext = NULL; 3510 int r, i; 3511 int ctx_reg_offset; 3512 3513 /* init the CP */ 3514 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT, 3515 adev->gfx.config.max_hw_contexts - 1); 3516 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1); 3517 3518 if (!amdgpu_async_gfx_ring) 3519 gfx_v11_0_cp_gfx_enable(adev, true); 3520 3521 ring = &adev->gfx.gfx_ring[0]; 3522 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev)); 3523 if (r) { 3524 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3525 return r; 3526 } 3527 3528 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3529 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3530 3531 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3532 amdgpu_ring_write(ring, 0x80000000); 3533 amdgpu_ring_write(ring, 0x80000000); 3534 3535 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 3536 for (ext = sect->section; ext->extent != NULL; ++ext) { 3537 if (sect->id == SECT_CONTEXT) { 3538 amdgpu_ring_write(ring, 3539 PACKET3(PACKET3_SET_CONTEXT_REG, 3540 ext->reg_count)); 3541 amdgpu_ring_write(ring, ext->reg_index - 3542 PACKET3_SET_CONTEXT_REG_START); 3543 for (i = 0; i < ext->reg_count; i++) 3544 amdgpu_ring_write(ring, ext->extent[i]); 3545 } 3546 } 3547 } 3548 3549 ctx_reg_offset = 3550 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 3551 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 3552 amdgpu_ring_write(ring, ctx_reg_offset); 3553 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override); 3554 3555 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3556 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3557 3558 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3559 amdgpu_ring_write(ring, 0); 3560 3561 amdgpu_ring_commit(ring); 3562 3563 /* submit cs packet to copy state 0 to next available state */ 3564 if (adev->gfx.num_gfx_rings > 1) { 3565 /* maximum supported gfx ring is 2 */ 3566 ring = &adev->gfx.gfx_ring[1]; 3567 r = amdgpu_ring_alloc(ring, 2); 3568 if (r) { 3569 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3570 return r; 3571 } 3572 3573 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3574 amdgpu_ring_write(ring, 0); 3575 3576 amdgpu_ring_commit(ring); 3577 } 3578 return 0; 3579 } 3580 3581 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev, 3582 CP_PIPE_ID pipe) 3583 { 3584 u32 tmp; 3585 3586 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); 3587 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe); 3588 3589 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); 3590 } 3591 3592 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev, 3593 struct amdgpu_ring *ring) 3594 { 3595 u32 tmp; 3596 3597 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); 3598 if (ring->use_doorbell) { 3599 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3600 DOORBELL_OFFSET, ring->doorbell_index); 3601 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3602 DOORBELL_EN, 1); 3603 } else { 3604 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3605 DOORBELL_EN, 0); 3606 } 3607 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp); 3608 3609 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3610 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3611 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp); 3612 3613 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 3614 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3615 } 3616 3617 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev) 3618 { 3619 struct amdgpu_ring *ring; 3620 u32 tmp; 3621 u32 rb_bufsz; 3622 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3623 3624 /* Set the write pointer delay */ 3625 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0); 3626 3627 /* set the RB to use vmid 0 */ 3628 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0); 3629 3630 /* Init gfx ring 0 for pipe 0 */ 3631 mutex_lock(&adev->srbm_mutex); 3632 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3633 3634 /* Set ring buffer size */ 3635 ring = &adev->gfx.gfx_ring[0]; 3636 rb_bufsz = order_base_2(ring->ring_size / 8); 3637 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3638 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3639 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3640 3641 /* Initialize the ring buffer's write pointers */ 3642 ring->wptr = 0; 3643 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3644 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3645 3646 /* set the wb address whether it's enabled or not */ 3647 rptr_addr = ring->rptr_gpu_addr; 3648 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3649 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3650 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3651 3652 wptr_gpu_addr = ring->wptr_gpu_addr; 3653 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3654 lower_32_bits(wptr_gpu_addr)); 3655 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3656 upper_32_bits(wptr_gpu_addr)); 3657 3658 mdelay(1); 3659 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3660 3661 rb_addr = ring->gpu_addr >> 8; 3662 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr); 3663 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3664 3665 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1); 3666 3667 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3668 mutex_unlock(&adev->srbm_mutex); 3669 3670 /* Init gfx ring 1 for pipe 1 */ 3671 if (adev->gfx.num_gfx_rings > 1) { 3672 mutex_lock(&adev->srbm_mutex); 3673 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1); 3674 /* maximum supported gfx ring is 2 */ 3675 ring = &adev->gfx.gfx_ring[1]; 3676 rb_bufsz = order_base_2(ring->ring_size / 8); 3677 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz); 3678 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2); 3679 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3680 /* Initialize the ring buffer's write pointers */ 3681 ring->wptr = 0; 3682 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr)); 3683 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr)); 3684 /* Set the wb address whether it's enabled or not */ 3685 rptr_addr = ring->rptr_gpu_addr; 3686 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr)); 3687 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3688 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3689 wptr_gpu_addr = ring->wptr_gpu_addr; 3690 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3691 lower_32_bits(wptr_gpu_addr)); 3692 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3693 upper_32_bits(wptr_gpu_addr)); 3694 3695 mdelay(1); 3696 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3697 3698 rb_addr = ring->gpu_addr >> 8; 3699 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr); 3700 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr)); 3701 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1); 3702 3703 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3704 mutex_unlock(&adev->srbm_mutex); 3705 } 3706 /* Switch to pipe 0 */ 3707 mutex_lock(&adev->srbm_mutex); 3708 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3709 mutex_unlock(&adev->srbm_mutex); 3710 3711 /* start the ring */ 3712 gfx_v11_0_cp_gfx_start(adev); 3713 3714 return 0; 3715 } 3716 3717 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3718 { 3719 u32 data; 3720 3721 if (adev->gfx.rs64_enable) { 3722 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3723 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE, 3724 enable ? 0 : 1); 3725 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 3726 enable ? 0 : 1); 3727 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 3728 enable ? 0 : 1); 3729 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 3730 enable ? 0 : 1); 3731 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 3732 enable ? 0 : 1); 3733 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE, 3734 enable ? 1 : 0); 3735 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE, 3736 enable ? 1 : 0); 3737 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE, 3738 enable ? 1 : 0); 3739 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE, 3740 enable ? 1 : 0); 3741 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT, 3742 enable ? 0 : 1); 3743 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data); 3744 } else { 3745 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); 3746 3747 if (enable) { 3748 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0); 3749 if (!adev->enable_mes_kiq) 3750 data = REG_SET_FIELD(data, CP_MEC_CNTL, 3751 MEC_ME2_HALT, 0); 3752 } else { 3753 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1); 3754 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1); 3755 } 3756 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data); 3757 } 3758 3759 udelay(50); 3760 } 3761 3762 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3763 { 3764 const struct gfx_firmware_header_v1_0 *mec_hdr; 3765 const __le32 *fw_data; 3766 unsigned i, fw_size; 3767 u32 *fw = NULL; 3768 int r; 3769 3770 if (!adev->gfx.mec_fw) 3771 return -EINVAL; 3772 3773 gfx_v11_0_cp_compute_enable(adev, false); 3774 3775 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3776 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3777 3778 fw_data = (const __le32 *) 3779 (adev->gfx.mec_fw->data + 3780 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3781 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 3782 3783 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 3784 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3785 &adev->gfx.mec.mec_fw_obj, 3786 &adev->gfx.mec.mec_fw_gpu_addr, 3787 (void **)&fw); 3788 if (r) { 3789 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r); 3790 gfx_v11_0_mec_fini(adev); 3791 return r; 3792 } 3793 3794 memcpy(fw, fw_data, fw_size); 3795 3796 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 3797 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 3798 3799 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr); 3800 3801 /* MEC1 */ 3802 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0); 3803 3804 for (i = 0; i < mec_hdr->jt_size; i++) 3805 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA, 3806 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3807 3808 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); 3809 3810 return 0; 3811 } 3812 3813 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) 3814 { 3815 const struct gfx_firmware_header_v2_0 *mec_hdr; 3816 const __le32 *fw_ucode, *fw_data; 3817 u32 tmp, fw_ucode_size, fw_data_size; 3818 u32 i, usec_timeout = 50000; /* Wait for 50 ms */ 3819 u32 *fw_ucode_ptr, *fw_data_ptr; 3820 int r; 3821 3822 if (!adev->gfx.mec_fw) 3823 return -EINVAL; 3824 3825 gfx_v11_0_cp_compute_enable(adev, false); 3826 3827 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 3828 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3829 3830 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data + 3831 le32_to_cpu(mec_hdr->ucode_offset_bytes)); 3832 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes); 3833 3834 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 3835 le32_to_cpu(mec_hdr->data_offset_bytes)); 3836 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 3837 3838 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3839 64 * 1024, 3840 AMDGPU_GEM_DOMAIN_VRAM | 3841 AMDGPU_GEM_DOMAIN_GTT, 3842 &adev->gfx.mec.mec_fw_obj, 3843 &adev->gfx.mec.mec_fw_gpu_addr, 3844 (void **)&fw_ucode_ptr); 3845 if (r) { 3846 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 3847 gfx_v11_0_mec_fini(adev); 3848 return r; 3849 } 3850 3851 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3852 64 * 1024, 3853 AMDGPU_GEM_DOMAIN_VRAM | 3854 AMDGPU_GEM_DOMAIN_GTT, 3855 &adev->gfx.mec.mec_fw_data_obj, 3856 &adev->gfx.mec.mec_fw_data_gpu_addr, 3857 (void **)&fw_data_ptr); 3858 if (r) { 3859 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 3860 gfx_v11_0_mec_fini(adev); 3861 return r; 3862 } 3863 3864 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); 3865 memcpy(fw_data_ptr, fw_data, fw_data_size); 3866 3867 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 3868 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); 3869 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 3870 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj); 3871 3872 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 3873 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3874 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 3875 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3876 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 3877 3878 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 3879 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 3880 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 3881 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 3882 3883 mutex_lock(&adev->srbm_mutex); 3884 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 3885 soc21_grbm_select(adev, 1, i, 0, 0); 3886 3887 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr); 3888 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 3889 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr)); 3890 3891 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 3892 mec_hdr->ucode_start_addr_lo >> 2 | 3893 mec_hdr->ucode_start_addr_hi << 30); 3894 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 3895 mec_hdr->ucode_start_addr_hi >> 2); 3896 3897 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr); 3898 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 3899 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3900 } 3901 mutex_unlock(&adev->srbm_mutex); 3902 soc21_grbm_select(adev, 0, 0, 0, 0); 3903 3904 /* Trigger an invalidation of the L1 instruction caches */ 3905 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 3906 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3907 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 3908 3909 /* Wait for invalidation complete */ 3910 for (i = 0; i < usec_timeout; i++) { 3911 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 3912 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 3913 INVALIDATE_DCACHE_COMPLETE)) 3914 break; 3915 udelay(1); 3916 } 3917 3918 if (i >= usec_timeout) { 3919 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3920 return -EINVAL; 3921 } 3922 3923 /* Trigger an invalidation of the L1 instruction caches */ 3924 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 3925 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 3926 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 3927 3928 /* Wait for invalidation complete */ 3929 for (i = 0; i < usec_timeout; i++) { 3930 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 3931 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 3932 INVALIDATE_CACHE_COMPLETE)) 3933 break; 3934 udelay(1); 3935 } 3936 3937 if (i >= usec_timeout) { 3938 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3939 return -EINVAL; 3940 } 3941 3942 return 0; 3943 } 3944 3945 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring) 3946 { 3947 uint32_t tmp; 3948 struct amdgpu_device *adev = ring->adev; 3949 3950 /* tell RLC which is KIQ queue */ 3951 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); 3952 tmp &= 0xffffff00; 3953 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3954 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80); 3955 } 3956 3957 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev) 3958 { 3959 /* set graphics engine doorbell range */ 3960 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, 3961 (adev->doorbell_index.gfx_ring0 * 2) << 2); 3962 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 3963 (adev->doorbell_index.gfx_userqueue_end * 2) << 2); 3964 3965 /* set compute engine doorbell range */ 3966 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 3967 (adev->doorbell_index.kiq * 2) << 2); 3968 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 3969 (adev->doorbell_index.userqueue_end * 2) << 2); 3970 } 3971 3972 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev, 3973 struct v11_gfx_mqd *mqd, 3974 struct amdgpu_mqd_prop *prop) 3975 { 3976 bool priority = 0; 3977 u32 tmp; 3978 3979 /* set up default queue priority level 3980 * 0x0 = low priority, 0x1 = high priority 3981 */ 3982 if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH) 3983 priority = 1; 3984 3985 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT; 3986 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority); 3987 mqd->cp_gfx_hqd_queue_priority = tmp; 3988 } 3989 3990 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, 3991 struct amdgpu_mqd_prop *prop) 3992 { 3993 struct v11_gfx_mqd *mqd = m; 3994 uint64_t hqd_gpu_addr, wb_gpu_addr; 3995 uint32_t tmp; 3996 uint32_t rb_bufsz; 3997 3998 /* set up gfx hqd wptr */ 3999 mqd->cp_gfx_hqd_wptr = 0; 4000 mqd->cp_gfx_hqd_wptr_hi = 0; 4001 4002 /* set the pointer to the MQD */ 4003 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc; 4004 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4005 4006 /* set up mqd control */ 4007 tmp = regCP_GFX_MQD_CONTROL_DEFAULT; 4008 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0); 4009 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1); 4010 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0); 4011 mqd->cp_gfx_mqd_control = tmp; 4012 4013 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ 4014 tmp = regCP_GFX_HQD_VMID_DEFAULT; 4015 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0); 4016 mqd->cp_gfx_hqd_vmid = 0; 4017 4018 /* set up gfx queue priority */ 4019 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop); 4020 4021 /* set up time quantum */ 4022 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT; 4023 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1); 4024 mqd->cp_gfx_hqd_quantum = tmp; 4025 4026 /* set up gfx hqd base. this is similar as CP_RB_BASE */ 4027 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4028 mqd->cp_gfx_hqd_base = hqd_gpu_addr; 4029 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr); 4030 4031 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */ 4032 wb_gpu_addr = prop->rptr_gpu_addr; 4033 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc; 4034 mqd->cp_gfx_hqd_rptr_addr_hi = 4035 upper_32_bits(wb_gpu_addr) & 0xffff; 4036 4037 /* set up rb_wptr_poll addr */ 4038 wb_gpu_addr = prop->wptr_gpu_addr; 4039 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4040 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4041 4042 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */ 4043 rb_bufsz = order_base_2(prop->queue_size / 4) - 1; 4044 tmp = regCP_GFX_HQD_CNTL_DEFAULT; 4045 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); 4046 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); 4047 #ifdef __BIG_ENDIAN 4048 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); 4049 #endif 4050 mqd->cp_gfx_hqd_cntl = tmp; 4051 4052 /* set up cp_doorbell_control */ 4053 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT; 4054 if (prop->use_doorbell) { 4055 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4056 DOORBELL_OFFSET, prop->doorbell_index); 4057 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4058 DOORBELL_EN, 1); 4059 } else 4060 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4061 DOORBELL_EN, 0); 4062 mqd->cp_rb_doorbell_control = tmp; 4063 4064 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4065 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT; 4066 4067 /* active the queue */ 4068 mqd->cp_gfx_hqd_active = 1; 4069 4070 return 0; 4071 } 4072 4073 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) 4074 { 4075 struct amdgpu_device *adev = ring->adev; 4076 struct v11_gfx_mqd *mqd = ring->mqd_ptr; 4077 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; 4078 4079 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { 4080 memset((void *)mqd, 0, sizeof(*mqd)); 4081 mutex_lock(&adev->srbm_mutex); 4082 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4083 amdgpu_ring_init_mqd(ring); 4084 soc21_grbm_select(adev, 0, 0, 0, 0); 4085 mutex_unlock(&adev->srbm_mutex); 4086 if (adev->gfx.me.mqd_backup[mqd_idx]) 4087 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4088 } else { 4089 /* restore mqd with the backup copy */ 4090 if (adev->gfx.me.mqd_backup[mqd_idx]) 4091 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); 4092 /* reset the ring */ 4093 ring->wptr = 0; 4094 *ring->wptr_cpu_addr = 0; 4095 amdgpu_ring_clear_ring(ring); 4096 } 4097 4098 return 0; 4099 } 4100 4101 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) 4102 { 4103 int r, i; 4104 struct amdgpu_ring *ring; 4105 4106 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4107 ring = &adev->gfx.gfx_ring[i]; 4108 4109 r = amdgpu_bo_reserve(ring->mqd_obj, false); 4110 if (unlikely(r != 0)) 4111 return r; 4112 4113 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 4114 if (!r) { 4115 r = gfx_v11_0_kgq_init_queue(ring, false); 4116 amdgpu_bo_kunmap(ring->mqd_obj); 4117 ring->mqd_ptr = NULL; 4118 } 4119 amdgpu_bo_unreserve(ring->mqd_obj); 4120 if (r) 4121 return r; 4122 } 4123 4124 r = amdgpu_gfx_enable_kgq(adev, 0); 4125 if (r) 4126 return r; 4127 4128 return gfx_v11_0_cp_gfx_start(adev); 4129 } 4130 4131 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m, 4132 struct amdgpu_mqd_prop *prop) 4133 { 4134 struct v11_compute_mqd *mqd = m; 4135 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4136 uint32_t tmp; 4137 4138 mqd->header = 0xC0310800; 4139 mqd->compute_pipelinestat_enable = 0x00000001; 4140 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4141 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4142 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4143 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4144 mqd->compute_misc_reserved = 0x00000007; 4145 4146 eop_base_addr = prop->eop_gpu_addr >> 8; 4147 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4148 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4149 4150 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4151 tmp = regCP_HQD_EOP_CONTROL_DEFAULT; 4152 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4153 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1)); 4154 4155 mqd->cp_hqd_eop_control = tmp; 4156 4157 /* enable doorbell? */ 4158 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4159 4160 if (prop->use_doorbell) { 4161 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4162 DOORBELL_OFFSET, prop->doorbell_index); 4163 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4164 DOORBELL_EN, 1); 4165 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4166 DOORBELL_SOURCE, 0); 4167 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4168 DOORBELL_HIT, 0); 4169 } else { 4170 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4171 DOORBELL_EN, 0); 4172 } 4173 4174 mqd->cp_hqd_pq_doorbell_control = tmp; 4175 4176 /* disable the queue if it's active */ 4177 mqd->cp_hqd_dequeue_request = 0; 4178 mqd->cp_hqd_pq_rptr = 0; 4179 mqd->cp_hqd_pq_wptr_lo = 0; 4180 mqd->cp_hqd_pq_wptr_hi = 0; 4181 4182 /* set the pointer to the MQD */ 4183 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; 4184 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4185 4186 /* set MQD vmid to 0 */ 4187 tmp = regCP_MQD_CONTROL_DEFAULT; 4188 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4189 mqd->cp_mqd_control = tmp; 4190 4191 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4192 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4193 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4194 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4195 4196 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4197 tmp = regCP_HQD_PQ_CONTROL_DEFAULT; 4198 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4199 (order_base_2(prop->queue_size / 4) - 1)); 4200 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4201 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4202 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1); 4203 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 4204 prop->allow_tunneling); 4205 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4206 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4207 mqd->cp_hqd_pq_control = tmp; 4208 4209 /* set the wb address whether it's enabled or not */ 4210 wb_gpu_addr = prop->rptr_gpu_addr; 4211 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4212 mqd->cp_hqd_pq_rptr_report_addr_hi = 4213 upper_32_bits(wb_gpu_addr) & 0xffff; 4214 4215 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4216 wb_gpu_addr = prop->wptr_gpu_addr; 4217 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4218 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4219 4220 tmp = 0; 4221 /* enable the doorbell if requested */ 4222 if (prop->use_doorbell) { 4223 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4224 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4225 DOORBELL_OFFSET, prop->doorbell_index); 4226 4227 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4228 DOORBELL_EN, 1); 4229 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4230 DOORBELL_SOURCE, 0); 4231 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4232 DOORBELL_HIT, 0); 4233 } 4234 4235 mqd->cp_hqd_pq_doorbell_control = tmp; 4236 4237 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4238 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT; 4239 4240 /* set the vmid for the queue */ 4241 mqd->cp_hqd_vmid = 0; 4242 4243 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; 4244 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55); 4245 mqd->cp_hqd_persistent_state = tmp; 4246 4247 /* set MIN_IB_AVAIL_SIZE */ 4248 tmp = regCP_HQD_IB_CONTROL_DEFAULT; 4249 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4250 mqd->cp_hqd_ib_control = tmp; 4251 4252 /* set static priority for a compute queue/ring */ 4253 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; 4254 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; 4255 4256 mqd->cp_hqd_active = prop->hqd_active; 4257 4258 return 0; 4259 } 4260 4261 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring) 4262 { 4263 struct amdgpu_device *adev = ring->adev; 4264 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4265 int j; 4266 4267 /* inactivate the queue */ 4268 if (amdgpu_sriov_vf(adev)) 4269 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); 4270 4271 /* disable wptr polling */ 4272 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4273 4274 /* write the EOP addr */ 4275 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR, 4276 mqd->cp_hqd_eop_base_addr_lo); 4277 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI, 4278 mqd->cp_hqd_eop_base_addr_hi); 4279 4280 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4281 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL, 4282 mqd->cp_hqd_eop_control); 4283 4284 /* enable doorbell? */ 4285 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4286 mqd->cp_hqd_pq_doorbell_control); 4287 4288 /* disable the queue if it's active */ 4289 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { 4290 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); 4291 for (j = 0; j < adev->usec_timeout; j++) { 4292 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) 4293 break; 4294 udelay(1); 4295 } 4296 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 4297 mqd->cp_hqd_dequeue_request); 4298 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 4299 mqd->cp_hqd_pq_rptr); 4300 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4301 mqd->cp_hqd_pq_wptr_lo); 4302 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4303 mqd->cp_hqd_pq_wptr_hi); 4304 } 4305 4306 /* set the pointer to the MQD */ 4307 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, 4308 mqd->cp_mqd_base_addr_lo); 4309 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, 4310 mqd->cp_mqd_base_addr_hi); 4311 4312 /* set MQD vmid to 0 */ 4313 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 4314 mqd->cp_mqd_control); 4315 4316 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4317 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, 4318 mqd->cp_hqd_pq_base_lo); 4319 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, 4320 mqd->cp_hqd_pq_base_hi); 4321 4322 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4323 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, 4324 mqd->cp_hqd_pq_control); 4325 4326 /* set the wb address whether it's enabled or not */ 4327 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, 4328 mqd->cp_hqd_pq_rptr_report_addr_lo); 4329 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4330 mqd->cp_hqd_pq_rptr_report_addr_hi); 4331 4332 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4333 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, 4334 mqd->cp_hqd_pq_wptr_poll_addr_lo); 4335 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, 4336 mqd->cp_hqd_pq_wptr_poll_addr_hi); 4337 4338 /* enable the doorbell if requested */ 4339 if (ring->use_doorbell) { 4340 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4341 (adev->doorbell_index.kiq * 2) << 2); 4342 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4343 (adev->doorbell_index.userqueue_end * 2) << 2); 4344 } 4345 4346 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4347 mqd->cp_hqd_pq_doorbell_control); 4348 4349 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4350 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4351 mqd->cp_hqd_pq_wptr_lo); 4352 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4353 mqd->cp_hqd_pq_wptr_hi); 4354 4355 /* set the vmid for the queue */ 4356 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid); 4357 4358 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, 4359 mqd->cp_hqd_persistent_state); 4360 4361 /* activate the queue */ 4362 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 4363 mqd->cp_hqd_active); 4364 4365 if (ring->use_doorbell) 4366 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4367 4368 return 0; 4369 } 4370 4371 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) 4372 { 4373 struct amdgpu_device *adev = ring->adev; 4374 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4375 4376 gfx_v11_0_kiq_setting(ring); 4377 4378 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4379 /* reset MQD to a clean status */ 4380 if (adev->gfx.kiq[0].mqd_backup) 4381 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd)); 4382 4383 /* reset ring buffer */ 4384 ring->wptr = 0; 4385 amdgpu_ring_clear_ring(ring); 4386 4387 mutex_lock(&adev->srbm_mutex); 4388 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4389 gfx_v11_0_kiq_init_register(ring); 4390 soc21_grbm_select(adev, 0, 0, 0, 0); 4391 mutex_unlock(&adev->srbm_mutex); 4392 } else { 4393 memset((void *)mqd, 0, sizeof(*mqd)); 4394 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4395 amdgpu_ring_clear_ring(ring); 4396 mutex_lock(&adev->srbm_mutex); 4397 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4398 amdgpu_ring_init_mqd(ring); 4399 gfx_v11_0_kiq_init_register(ring); 4400 soc21_grbm_select(adev, 0, 0, 0, 0); 4401 mutex_unlock(&adev->srbm_mutex); 4402 4403 if (adev->gfx.kiq[0].mqd_backup) 4404 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd)); 4405 } 4406 4407 return 0; 4408 } 4409 4410 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset) 4411 { 4412 struct amdgpu_device *adev = ring->adev; 4413 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4414 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4415 4416 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { 4417 memset((void *)mqd, 0, sizeof(*mqd)); 4418 mutex_lock(&adev->srbm_mutex); 4419 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4420 amdgpu_ring_init_mqd(ring); 4421 soc21_grbm_select(adev, 0, 0, 0, 0); 4422 mutex_unlock(&adev->srbm_mutex); 4423 4424 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4425 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4426 } else { 4427 /* restore MQD to a clean status */ 4428 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4429 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); 4430 /* reset ring buffer */ 4431 ring->wptr = 0; 4432 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4433 amdgpu_ring_clear_ring(ring); 4434 } 4435 4436 return 0; 4437 } 4438 4439 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev) 4440 { 4441 struct amdgpu_ring *ring; 4442 int r; 4443 4444 ring = &adev->gfx.kiq[0].ring; 4445 4446 r = amdgpu_bo_reserve(ring->mqd_obj, false); 4447 if (unlikely(r != 0)) 4448 return r; 4449 4450 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 4451 if (unlikely(r != 0)) { 4452 amdgpu_bo_unreserve(ring->mqd_obj); 4453 return r; 4454 } 4455 4456 gfx_v11_0_kiq_init_queue(ring); 4457 amdgpu_bo_kunmap(ring->mqd_obj); 4458 ring->mqd_ptr = NULL; 4459 amdgpu_bo_unreserve(ring->mqd_obj); 4460 ring->sched.ready = true; 4461 return 0; 4462 } 4463 4464 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev) 4465 { 4466 struct amdgpu_ring *ring = NULL; 4467 int r = 0, i; 4468 4469 if (!amdgpu_async_gfx_ring) 4470 gfx_v11_0_cp_compute_enable(adev, true); 4471 4472 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4473 ring = &adev->gfx.compute_ring[i]; 4474 4475 r = amdgpu_bo_reserve(ring->mqd_obj, false); 4476 if (unlikely(r != 0)) 4477 goto done; 4478 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 4479 if (!r) { 4480 r = gfx_v11_0_kcq_init_queue(ring, false); 4481 amdgpu_bo_kunmap(ring->mqd_obj); 4482 ring->mqd_ptr = NULL; 4483 } 4484 amdgpu_bo_unreserve(ring->mqd_obj); 4485 if (r) 4486 goto done; 4487 } 4488 4489 r = amdgpu_gfx_enable_kcq(adev, 0); 4490 done: 4491 return r; 4492 } 4493 4494 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev) 4495 { 4496 int r, i; 4497 struct amdgpu_ring *ring; 4498 4499 if (!(adev->flags & AMD_IS_APU)) 4500 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 4501 4502 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4503 /* legacy firmware loading */ 4504 r = gfx_v11_0_cp_gfx_load_microcode(adev); 4505 if (r) 4506 return r; 4507 4508 if (adev->gfx.rs64_enable) 4509 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev); 4510 else 4511 r = gfx_v11_0_cp_compute_load_microcode(adev); 4512 if (r) 4513 return r; 4514 } 4515 4516 gfx_v11_0_cp_set_doorbell_range(adev); 4517 4518 if (amdgpu_async_gfx_ring) { 4519 gfx_v11_0_cp_compute_enable(adev, true); 4520 gfx_v11_0_cp_gfx_enable(adev, true); 4521 } 4522 4523 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) 4524 r = amdgpu_mes_kiq_hw_init(adev); 4525 else 4526 r = gfx_v11_0_kiq_resume(adev); 4527 if (r) 4528 return r; 4529 4530 r = gfx_v11_0_kcq_resume(adev); 4531 if (r) 4532 return r; 4533 4534 if (!amdgpu_async_gfx_ring) { 4535 r = gfx_v11_0_cp_gfx_resume(adev); 4536 if (r) 4537 return r; 4538 } else { 4539 r = gfx_v11_0_cp_async_gfx_ring_resume(adev); 4540 if (r) 4541 return r; 4542 } 4543 4544 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4545 ring = &adev->gfx.gfx_ring[i]; 4546 r = amdgpu_ring_test_helper(ring); 4547 if (r) 4548 return r; 4549 } 4550 4551 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4552 ring = &adev->gfx.compute_ring[i]; 4553 r = amdgpu_ring_test_helper(ring); 4554 if (r) 4555 return r; 4556 } 4557 4558 return 0; 4559 } 4560 4561 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable) 4562 { 4563 gfx_v11_0_cp_gfx_enable(adev, enable); 4564 gfx_v11_0_cp_compute_enable(adev, enable); 4565 } 4566 4567 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev) 4568 { 4569 int r; 4570 bool value; 4571 4572 r = adev->gfxhub.funcs->gart_enable(adev); 4573 if (r) 4574 return r; 4575 4576 adev->hdp.funcs->flush_hdp(adev, NULL); 4577 4578 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ? 4579 false : true; 4580 4581 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 4582 /* TODO investigate why this and the hdp flush above is needed, 4583 * are we missing a flush somewhere else? */ 4584 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); 4585 4586 return 0; 4587 } 4588 4589 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev) 4590 { 4591 u32 tmp; 4592 4593 /* select RS64 */ 4594 if (adev->gfx.rs64_enable) { 4595 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL); 4596 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1); 4597 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp); 4598 4599 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL); 4600 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1); 4601 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp); 4602 } 4603 4604 if (amdgpu_emu_mode == 1) 4605 msleep(100); 4606 } 4607 4608 static int get_gb_addr_config(struct amdgpu_device * adev) 4609 { 4610 u32 gb_addr_config; 4611 4612 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG); 4613 if (gb_addr_config == 0) 4614 return -EINVAL; 4615 4616 adev->gfx.config.gb_addr_config_fields.num_pkrs = 4617 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS); 4618 4619 adev->gfx.config.gb_addr_config = gb_addr_config; 4620 4621 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 4622 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4623 GB_ADDR_CONFIG, NUM_PIPES); 4624 4625 adev->gfx.config.max_tile_pipes = 4626 adev->gfx.config.gb_addr_config_fields.num_pipes; 4627 4628 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 4629 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4630 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS); 4631 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 4632 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4633 GB_ADDR_CONFIG, NUM_RB_PER_SE); 4634 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 4635 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4636 GB_ADDR_CONFIG, NUM_SHADER_ENGINES); 4637 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 4638 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4639 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE)); 4640 4641 return 0; 4642 } 4643 4644 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) 4645 { 4646 uint32_t data; 4647 4648 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); 4649 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; 4650 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); 4651 4652 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); 4653 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; 4654 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); 4655 } 4656 4657 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) 4658 { 4659 int r; 4660 struct amdgpu_device *adev = ip_block->adev; 4661 4662 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4663 adev->gfx.cleaner_shader_ptr); 4664 4665 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 4666 if (adev->gfx.imu.funcs) { 4667 /* RLC autoload sequence 1: Program rlc ram */ 4668 if (adev->gfx.imu.funcs->program_rlc_ram) 4669 adev->gfx.imu.funcs->program_rlc_ram(adev); 4670 /* rlc autoload firmware */ 4671 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev); 4672 if (r) 4673 return r; 4674 } 4675 } else { 4676 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4677 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { 4678 if (adev->gfx.imu.funcs->load_microcode) 4679 adev->gfx.imu.funcs->load_microcode(adev); 4680 if (adev->gfx.imu.funcs->setup_imu) 4681 adev->gfx.imu.funcs->setup_imu(adev); 4682 if (adev->gfx.imu.funcs->start_imu) 4683 adev->gfx.imu.funcs->start_imu(adev); 4684 } 4685 4686 /* disable gpa mode in backdoor loading */ 4687 gfx_v11_0_disable_gpa_mode(adev); 4688 } 4689 } 4690 4691 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) || 4692 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 4693 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev); 4694 if (r) { 4695 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r); 4696 return r; 4697 } 4698 } 4699 4700 adev->gfx.is_poweron = true; 4701 4702 if(get_gb_addr_config(adev)) 4703 DRM_WARN("Invalid gb_addr_config !\n"); 4704 4705 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP && 4706 adev->gfx.rs64_enable) 4707 gfx_v11_0_config_gfx_rs64(adev); 4708 4709 r = gfx_v11_0_gfxhub_enable(adev); 4710 if (r) 4711 return r; 4712 4713 if (!amdgpu_emu_mode) 4714 gfx_v11_0_init_golden_registers(adev); 4715 4716 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || 4717 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) { 4718 /** 4719 * For gfx 11, rlc firmware loading relies on smu firmware is 4720 * loaded firstly, so in direct type, it has to load smc ucode 4721 * here before rlc. 4722 */ 4723 r = amdgpu_pm_load_smu_firmware(adev, NULL); 4724 if (r) 4725 return r; 4726 } 4727 4728 gfx_v11_0_constants_init(adev); 4729 4730 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 4731 gfx_v11_0_select_cp_fw_arch(adev); 4732 4733 if (adev->nbio.funcs->gc_doorbell_init) 4734 adev->nbio.funcs->gc_doorbell_init(adev); 4735 4736 r = gfx_v11_0_rlc_resume(adev); 4737 if (r) 4738 return r; 4739 4740 /* 4741 * init golden registers and rlc resume may override some registers, 4742 * reconfig them here 4743 */ 4744 gfx_v11_0_tcp_harvest(adev); 4745 4746 r = gfx_v11_0_cp_resume(adev); 4747 if (r) 4748 return r; 4749 4750 /* get IMU version from HW if it's not set */ 4751 if (!adev->gfx.imu_fw_version) 4752 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); 4753 4754 return r; 4755 } 4756 4757 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) 4758 { 4759 struct amdgpu_device *adev = ip_block->adev; 4760 4761 cancel_delayed_work_sync(&adev->gfx.idle_work); 4762 4763 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4764 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4765 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 4766 4767 if (!adev->no_hw_access) { 4768 if (amdgpu_async_gfx_ring) { 4769 if (amdgpu_gfx_disable_kgq(adev, 0)) 4770 DRM_ERROR("KGQ disable failed\n"); 4771 } 4772 4773 if (amdgpu_gfx_disable_kcq(adev, 0)) 4774 DRM_ERROR("KCQ disable failed\n"); 4775 4776 amdgpu_mes_kiq_hw_fini(adev); 4777 } 4778 4779 if (amdgpu_sriov_vf(adev)) 4780 /* Remove the steps disabling CPG and clearing KIQ position, 4781 * so that CP could perform IDLE-SAVE during switch. Those 4782 * steps are necessary to avoid a DMAR error in gfx9 but it is 4783 * not reproduced on gfx11. 4784 */ 4785 return 0; 4786 4787 gfx_v11_0_cp_enable(adev, false); 4788 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 4789 4790 adev->gfxhub.funcs->gart_disable(adev); 4791 4792 adev->gfx.is_poweron = false; 4793 4794 return 0; 4795 } 4796 4797 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block) 4798 { 4799 return gfx_v11_0_hw_fini(ip_block); 4800 } 4801 4802 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block) 4803 { 4804 return gfx_v11_0_hw_init(ip_block); 4805 } 4806 4807 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block) 4808 { 4809 struct amdgpu_device *adev = ip_block->adev; 4810 4811 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS), 4812 GRBM_STATUS, GUI_ACTIVE)) 4813 return false; 4814 else 4815 return true; 4816 } 4817 4818 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 4819 { 4820 unsigned i; 4821 u32 tmp; 4822 struct amdgpu_device *adev = ip_block->adev; 4823 4824 for (i = 0; i < adev->usec_timeout; i++) { 4825 /* read MC_STATUS */ 4826 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) & 4827 GRBM_STATUS__GUI_ACTIVE_MASK; 4828 4829 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE)) 4830 return 0; 4831 udelay(1); 4832 } 4833 return -ETIMEDOUT; 4834 } 4835 4836 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev, 4837 bool req) 4838 { 4839 u32 i, tmp, val; 4840 4841 for (i = 0; i < adev->usec_timeout; i++) { 4842 /* Request with MeId=2, PipeId=0 */ 4843 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req); 4844 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4); 4845 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp); 4846 4847 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX); 4848 if (req) { 4849 if (val == tmp) 4850 break; 4851 } else { 4852 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, 4853 REQUEST, 1); 4854 4855 /* unlocked or locked by firmware */ 4856 if (val != tmp) 4857 break; 4858 } 4859 udelay(1); 4860 } 4861 4862 if (i >= adev->usec_timeout) 4863 return -EINVAL; 4864 4865 return 0; 4866 } 4867 4868 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block) 4869 { 4870 u32 grbm_soft_reset = 0; 4871 u32 tmp; 4872 int r, i, j, k; 4873 struct amdgpu_device *adev = ip_block->adev; 4874 4875 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4876 4877 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 4878 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0); 4879 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0); 4880 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0); 4881 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0); 4882 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 4883 4884 mutex_lock(&adev->srbm_mutex); 4885 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 4886 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 4887 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 4888 soc21_grbm_select(adev, i, k, j, 0); 4889 4890 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); 4891 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); 4892 } 4893 } 4894 } 4895 for (i = 0; i < adev->gfx.me.num_me; ++i) { 4896 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { 4897 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 4898 soc21_grbm_select(adev, i, k, j, 0); 4899 4900 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1); 4901 } 4902 } 4903 } 4904 soc21_grbm_select(adev, 0, 0, 0, 0); 4905 mutex_unlock(&adev->srbm_mutex); 4906 4907 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */ 4908 mutex_lock(&adev->gfx.reset_sem_mutex); 4909 r = gfx_v11_0_request_gfx_index_mutex(adev, true); 4910 if (r) { 4911 mutex_unlock(&adev->gfx.reset_sem_mutex); 4912 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n"); 4913 return r; 4914 } 4915 4916 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe); 4917 4918 // Read CP_VMID_RESET register three times. 4919 // to get sufficient time for GFX_HQD_ACTIVE reach 0 4920 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 4921 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 4922 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 4923 4924 /* release the gfx mutex */ 4925 r = gfx_v11_0_request_gfx_index_mutex(adev, false); 4926 mutex_unlock(&adev->gfx.reset_sem_mutex); 4927 if (r) { 4928 DRM_ERROR("Failed to release the gfx mutex during soft reset\n"); 4929 return r; 4930 } 4931 4932 for (i = 0; i < adev->usec_timeout; i++) { 4933 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) && 4934 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE)) 4935 break; 4936 udelay(1); 4937 } 4938 if (i >= adev->usec_timeout) { 4939 printk("Failed to wait all pipes clean\n"); 4940 return -EINVAL; 4941 } 4942 4943 /********** trigger soft reset ***********/ 4944 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 4945 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4946 SOFT_RESET_CP, 1); 4947 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4948 SOFT_RESET_GFX, 1); 4949 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4950 SOFT_RESET_CPF, 1); 4951 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4952 SOFT_RESET_CPC, 1); 4953 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4954 SOFT_RESET_CPG, 1); 4955 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 4956 /********** exit soft reset ***********/ 4957 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 4958 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4959 SOFT_RESET_CP, 0); 4960 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4961 SOFT_RESET_GFX, 0); 4962 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4963 SOFT_RESET_CPF, 0); 4964 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4965 SOFT_RESET_CPC, 0); 4966 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 4967 SOFT_RESET_CPG, 0); 4968 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 4969 4970 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL); 4971 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1); 4972 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp); 4973 4974 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0); 4975 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0); 4976 4977 for (i = 0; i < adev->usec_timeout; i++) { 4978 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET)) 4979 break; 4980 udelay(1); 4981 } 4982 if (i >= adev->usec_timeout) { 4983 printk("Failed to wait CP_VMID_RESET to 0\n"); 4984 return -EINVAL; 4985 } 4986 4987 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 4988 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 4989 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 4990 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 4991 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 4992 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 4993 4994 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4995 4996 return gfx_v11_0_cp_resume(adev); 4997 } 4998 4999 static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block) 5000 { 5001 int i, r; 5002 struct amdgpu_device *adev = ip_block->adev; 5003 struct amdgpu_ring *ring; 5004 long tmo = msecs_to_jiffies(1000); 5005 5006 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 5007 ring = &adev->gfx.gfx_ring[i]; 5008 r = amdgpu_ring_test_ib(ring, tmo); 5009 if (r) 5010 return true; 5011 } 5012 5013 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5014 ring = &adev->gfx.compute_ring[i]; 5015 r = amdgpu_ring_test_ib(ring, tmo); 5016 if (r) 5017 return true; 5018 } 5019 5020 return false; 5021 } 5022 5023 static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block) 5024 { 5025 struct amdgpu_device *adev = ip_block->adev; 5026 /** 5027 * GFX soft reset will impact MES, need resume MES when do GFX soft reset 5028 */ 5029 return amdgpu_mes_resume(adev); 5030 } 5031 5032 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5033 { 5034 uint64_t clock; 5035 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after; 5036 5037 if (amdgpu_sriov_vf(adev)) { 5038 amdgpu_gfx_off_ctrl(adev, false); 5039 mutex_lock(&adev->gfx.gpu_clock_mutex); 5040 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5041 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5042 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5043 if (clock_counter_hi_pre != clock_counter_hi_after) 5044 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5045 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5046 amdgpu_gfx_off_ctrl(adev, true); 5047 } else { 5048 preempt_disable(); 5049 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER); 5050 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER); 5051 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER); 5052 if (clock_counter_hi_pre != clock_counter_hi_after) 5053 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER); 5054 preempt_enable(); 5055 } 5056 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL); 5057 5058 return clock; 5059 } 5060 5061 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5062 uint32_t vmid, 5063 uint32_t gds_base, uint32_t gds_size, 5064 uint32_t gws_base, uint32_t gws_size, 5065 uint32_t oa_base, uint32_t oa_size) 5066 { 5067 struct amdgpu_device *adev = ring->adev; 5068 5069 /* GDS Base */ 5070 gfx_v11_0_write_data_to_reg(ring, 0, false, 5071 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid, 5072 gds_base); 5073 5074 /* GDS Size */ 5075 gfx_v11_0_write_data_to_reg(ring, 0, false, 5076 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid, 5077 gds_size); 5078 5079 /* GWS */ 5080 gfx_v11_0_write_data_to_reg(ring, 0, false, 5081 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid, 5082 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5083 5084 /* OA */ 5085 gfx_v11_0_write_data_to_reg(ring, 0, false, 5086 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid, 5087 (1 << (oa_size + oa_base)) - (1 << oa_base)); 5088 } 5089 5090 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) 5091 { 5092 struct amdgpu_device *adev = ip_block->adev; 5093 5094 adev->gfx.funcs = &gfx_v11_0_gfx_funcs; 5095 5096 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS; 5097 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5098 AMDGPU_MAX_COMPUTE_RINGS); 5099 5100 gfx_v11_0_set_kiq_pm4_funcs(adev); 5101 gfx_v11_0_set_ring_funcs(adev); 5102 gfx_v11_0_set_irq_funcs(adev); 5103 gfx_v11_0_set_gds_init(adev); 5104 gfx_v11_0_set_rlc_funcs(adev); 5105 gfx_v11_0_set_mqd_funcs(adev); 5106 gfx_v11_0_set_imu_funcs(adev); 5107 5108 gfx_v11_0_init_rlcg_reg_access_ctrl(adev); 5109 5110 return gfx_v11_0_init_microcode(adev); 5111 } 5112 5113 static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block) 5114 { 5115 struct amdgpu_device *adev = ip_block->adev; 5116 int r; 5117 5118 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5119 if (r) 5120 return r; 5121 5122 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5123 if (r) 5124 return r; 5125 5126 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 5127 if (r) 5128 return r; 5129 return 0; 5130 } 5131 5132 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) 5133 { 5134 uint32_t rlc_cntl; 5135 5136 /* if RLC is not enabled, do nothing */ 5137 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); 5138 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; 5139 } 5140 5141 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5142 { 5143 uint32_t data; 5144 unsigned i; 5145 5146 data = RLC_SAFE_MODE__CMD_MASK; 5147 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5148 5149 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data); 5150 5151 /* wait for RLC_SAFE_MODE */ 5152 for (i = 0; i < adev->usec_timeout; i++) { 5153 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), 5154 RLC_SAFE_MODE, CMD)) 5155 break; 5156 udelay(1); 5157 } 5158 } 5159 5160 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5161 { 5162 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); 5163 } 5164 5165 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 5166 bool enable) 5167 { 5168 uint32_t def, data; 5169 5170 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK)) 5171 return; 5172 5173 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5174 5175 if (enable) 5176 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5177 else 5178 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5179 5180 if (def != data) 5181 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5182 } 5183 5184 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev, 5185 bool enable) 5186 { 5187 uint32_t def, data; 5188 5189 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG)) 5190 return; 5191 5192 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5193 5194 if (enable) 5195 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5196 else 5197 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5198 5199 if (def != data) 5200 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5201 } 5202 5203 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev, 5204 bool enable) 5205 { 5206 uint32_t def, data; 5207 5208 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG)) 5209 return; 5210 5211 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5212 5213 if (enable) 5214 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5215 else 5216 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5217 5218 if (def != data) 5219 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5220 } 5221 5222 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5223 bool enable) 5224 { 5225 uint32_t data, def; 5226 5227 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS))) 5228 return; 5229 5230 /* It is disabled by HW by default */ 5231 if (enable) { 5232 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5233 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 5234 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5235 5236 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5237 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5238 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5239 5240 if (def != data) 5241 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5242 } 5243 } else { 5244 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5245 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5246 5247 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5248 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5249 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5250 5251 if (def != data) 5252 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5253 } 5254 } 5255 } 5256 5257 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5258 bool enable) 5259 { 5260 uint32_t def, data; 5261 5262 if (!(adev->cg_flags & 5263 (AMD_CG_SUPPORT_GFX_CGCG | 5264 AMD_CG_SUPPORT_GFX_CGLS | 5265 AMD_CG_SUPPORT_GFX_3D_CGCG | 5266 AMD_CG_SUPPORT_GFX_3D_CGLS))) 5267 return; 5268 5269 if (enable) { 5270 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5271 5272 /* unset CGCG override */ 5273 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5274 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5275 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5276 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5277 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG || 5278 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5279 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5280 5281 /* update CGCG override bits */ 5282 if (def != data) 5283 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5284 5285 /* enable cgcg FSM(0x0000363F) */ 5286 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5287 5288 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5289 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK; 5290 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5291 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5292 } 5293 5294 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5295 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK; 5296 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5297 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5298 } 5299 5300 if (def != data) 5301 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5302 5303 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5304 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5305 5306 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5307 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK; 5308 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5309 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5310 } 5311 5312 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5313 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK; 5314 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5315 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5316 } 5317 5318 if (def != data) 5319 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5320 5321 /* set IDLE_POLL_COUNT(0x00900100) */ 5322 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL); 5323 5324 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK); 5325 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5326 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5327 5328 if (def != data) 5329 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data); 5330 5331 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5332 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5333 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5334 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5335 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5336 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data); 5337 5338 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5339 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5340 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5341 5342 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5343 if (adev->sdma.num_instances > 1) { 5344 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5345 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5346 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5347 } 5348 } else { 5349 /* Program RLC_CGCG_CGLS_CTRL */ 5350 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5351 5352 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5353 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5354 5355 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5356 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5357 5358 if (def != data) 5359 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5360 5361 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5362 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5363 5364 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5365 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5366 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5367 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5368 5369 if (def != data) 5370 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5371 5372 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5373 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5374 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5375 5376 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5377 if (adev->sdma.num_instances > 1) { 5378 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5379 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5380 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5381 } 5382 } 5383 } 5384 5385 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5386 bool enable) 5387 { 5388 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5389 5390 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable); 5391 5392 gfx_v11_0_update_medium_grain_clock_gating(adev, enable); 5393 5394 gfx_v11_0_update_repeater_fgcg(adev, enable); 5395 5396 gfx_v11_0_update_sram_fgcg(adev, enable); 5397 5398 gfx_v11_0_update_perf_clk(adev, enable); 5399 5400 if (adev->cg_flags & 5401 (AMD_CG_SUPPORT_GFX_MGCG | 5402 AMD_CG_SUPPORT_GFX_CGLS | 5403 AMD_CG_SUPPORT_GFX_CGCG | 5404 AMD_CG_SUPPORT_GFX_3D_CGCG | 5405 AMD_CG_SUPPORT_GFX_3D_CGLS)) 5406 gfx_v11_0_enable_gui_idle_interrupt(adev, enable); 5407 5408 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5409 5410 return 0; 5411 } 5412 5413 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid) 5414 { 5415 u32 reg, pre_data, data; 5416 5417 amdgpu_gfx_off_ctrl(adev, false); 5418 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); 5419 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) 5420 pre_data = RREG32_NO_KIQ(reg); 5421 else 5422 pre_data = RREG32(reg); 5423 5424 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK); 5425 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5426 5427 if (pre_data != data) { 5428 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) { 5429 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); 5430 } else 5431 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); 5432 } 5433 amdgpu_gfx_off_ctrl(adev, true); 5434 5435 if (ring 5436 && amdgpu_sriov_is_pp_one_vf(adev) 5437 && (pre_data != data) 5438 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) 5439 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) { 5440 amdgpu_ring_emit_wreg(ring, reg, data); 5441 } 5442 } 5443 5444 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = { 5445 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled, 5446 .set_safe_mode = gfx_v11_0_set_safe_mode, 5447 .unset_safe_mode = gfx_v11_0_unset_safe_mode, 5448 .init = gfx_v11_0_rlc_init, 5449 .get_csb_size = gfx_v11_0_get_csb_size, 5450 .get_csb_buffer = gfx_v11_0_get_csb_buffer, 5451 .resume = gfx_v11_0_rlc_resume, 5452 .stop = gfx_v11_0_rlc_stop, 5453 .reset = gfx_v11_0_rlc_reset, 5454 .start = gfx_v11_0_rlc_start, 5455 .update_spm_vmid = gfx_v11_0_update_spm_vmid, 5456 }; 5457 5458 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) 5459 { 5460 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 5461 5462 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) 5463 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5464 else 5465 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5466 5467 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data); 5468 5469 // Program RLC_PG_DELAY3 for CGPG hysteresis 5470 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { 5471 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5472 case IP_VERSION(11, 0, 1): 5473 case IP_VERSION(11, 0, 4): 5474 case IP_VERSION(11, 5, 0): 5475 case IP_VERSION(11, 5, 1): 5476 case IP_VERSION(11, 5, 2): 5477 case IP_VERSION(11, 5, 3): 5478 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); 5479 break; 5480 default: 5481 break; 5482 } 5483 } 5484 } 5485 5486 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable) 5487 { 5488 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5489 5490 gfx_v11_cntl_power_gating(adev, enable); 5491 5492 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5493 } 5494 5495 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5496 enum amd_powergating_state state) 5497 { 5498 struct amdgpu_device *adev = ip_block->adev; 5499 bool enable = (state == AMD_PG_STATE_GATE); 5500 5501 if (amdgpu_sriov_vf(adev)) 5502 return 0; 5503 5504 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5505 case IP_VERSION(11, 0, 0): 5506 case IP_VERSION(11, 0, 2): 5507 case IP_VERSION(11, 0, 3): 5508 amdgpu_gfx_off_ctrl(adev, enable); 5509 break; 5510 case IP_VERSION(11, 0, 1): 5511 case IP_VERSION(11, 0, 4): 5512 case IP_VERSION(11, 5, 0): 5513 case IP_VERSION(11, 5, 1): 5514 case IP_VERSION(11, 5, 2): 5515 case IP_VERSION(11, 5, 3): 5516 if (!enable) 5517 amdgpu_gfx_off_ctrl(adev, false); 5518 5519 gfx_v11_cntl_pg(adev, enable); 5520 5521 if (enable) 5522 amdgpu_gfx_off_ctrl(adev, true); 5523 5524 break; 5525 default: 5526 break; 5527 } 5528 5529 return 0; 5530 } 5531 5532 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5533 enum amd_clockgating_state state) 5534 { 5535 struct amdgpu_device *adev = ip_block->adev; 5536 5537 if (amdgpu_sriov_vf(adev)) 5538 return 0; 5539 5540 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5541 case IP_VERSION(11, 0, 0): 5542 case IP_VERSION(11, 0, 1): 5543 case IP_VERSION(11, 0, 2): 5544 case IP_VERSION(11, 0, 3): 5545 case IP_VERSION(11, 0, 4): 5546 case IP_VERSION(11, 5, 0): 5547 case IP_VERSION(11, 5, 1): 5548 case IP_VERSION(11, 5, 2): 5549 case IP_VERSION(11, 5, 3): 5550 gfx_v11_0_update_gfx_clock_gating(adev, 5551 state == AMD_CG_STATE_GATE); 5552 break; 5553 default: 5554 break; 5555 } 5556 5557 return 0; 5558 } 5559 5560 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5561 { 5562 struct amdgpu_device *adev = ip_block->adev; 5563 int data; 5564 5565 /* AMD_CG_SUPPORT_GFX_MGCG */ 5566 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5567 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5568 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5569 5570 /* AMD_CG_SUPPORT_REPEATER_FGCG */ 5571 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK)) 5572 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG; 5573 5574 /* AMD_CG_SUPPORT_GFX_FGCG */ 5575 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK)) 5576 *flags |= AMD_CG_SUPPORT_GFX_FGCG; 5577 5578 /* AMD_CG_SUPPORT_GFX_PERF_CLK */ 5579 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK)) 5580 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK; 5581 5582 /* AMD_CG_SUPPORT_GFX_CGCG */ 5583 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5584 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5585 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5586 5587 /* AMD_CG_SUPPORT_GFX_CGLS */ 5588 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5589 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5590 5591 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5592 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5593 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5594 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5595 5596 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5597 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5598 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5599 } 5600 5601 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5602 { 5603 /* gfx11 is 32bit rptr*/ 5604 return *(uint32_t *)ring->rptr_cpu_addr; 5605 } 5606 5607 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5608 { 5609 struct amdgpu_device *adev = ring->adev; 5610 u64 wptr; 5611 5612 /* XXX check if swapping is necessary on BE */ 5613 if (ring->use_doorbell) { 5614 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5615 } else { 5616 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR); 5617 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32; 5618 } 5619 5620 return wptr; 5621 } 5622 5623 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5624 { 5625 struct amdgpu_device *adev = ring->adev; 5626 5627 if (ring->use_doorbell) { 5628 /* XXX check if swapping is necessary on BE */ 5629 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5630 ring->wptr); 5631 WDOORBELL64(ring->doorbell_index, ring->wptr); 5632 } else { 5633 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, 5634 lower_32_bits(ring->wptr)); 5635 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, 5636 upper_32_bits(ring->wptr)); 5637 } 5638 } 5639 5640 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5641 { 5642 /* gfx11 hardware is 32bit rptr */ 5643 return *(uint32_t *)ring->rptr_cpu_addr; 5644 } 5645 5646 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5647 { 5648 u64 wptr; 5649 5650 /* XXX check if swapping is necessary on BE */ 5651 if (ring->use_doorbell) 5652 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5653 else 5654 BUG(); 5655 return wptr; 5656 } 5657 5658 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5659 { 5660 struct amdgpu_device *adev = ring->adev; 5661 5662 /* XXX check if swapping is necessary on BE */ 5663 if (ring->use_doorbell) { 5664 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5665 ring->wptr); 5666 WDOORBELL64(ring->doorbell_index, ring->wptr); 5667 } else { 5668 BUG(); /* only DOORBELL method supported on gfx11 now */ 5669 } 5670 } 5671 5672 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5673 { 5674 struct amdgpu_device *adev = ring->adev; 5675 u32 ref_and_mask, reg_mem_engine; 5676 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5677 5678 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5679 switch (ring->me) { 5680 case 1: 5681 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5682 break; 5683 case 2: 5684 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5685 break; 5686 default: 5687 return; 5688 } 5689 reg_mem_engine = 0; 5690 } else { 5691 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe; 5692 reg_mem_engine = 1; /* pfp */ 5693 } 5694 5695 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5696 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5697 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5698 ref_and_mask, ref_and_mask, 0x20); 5699 } 5700 5701 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5702 struct amdgpu_job *job, 5703 struct amdgpu_ib *ib, 5704 uint32_t flags) 5705 { 5706 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5707 u32 header, control = 0; 5708 5709 BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE); 5710 5711 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5712 5713 control |= ib->length_dw | (vmid << 24); 5714 5715 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5716 control |= INDIRECT_BUFFER_PRE_ENB(1); 5717 5718 if (flags & AMDGPU_IB_PREEMPTED) 5719 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5720 5721 if (vmid) 5722 gfx_v11_0_ring_emit_de_meta(ring, 5723 (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false); 5724 } 5725 5726 if (ring->is_mes_queue) 5727 /* inherit vmid from mqd */ 5728 control |= 0x400000; 5729 5730 amdgpu_ring_write(ring, header); 5731 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5732 amdgpu_ring_write(ring, 5733 #ifdef __BIG_ENDIAN 5734 (2 << 0) | 5735 #endif 5736 lower_32_bits(ib->gpu_addr)); 5737 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5738 amdgpu_ring_write(ring, control); 5739 } 5740 5741 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5742 struct amdgpu_job *job, 5743 struct amdgpu_ib *ib, 5744 uint32_t flags) 5745 { 5746 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5747 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5748 5749 if (ring->is_mes_queue) 5750 /* inherit vmid from mqd */ 5751 control |= 0x40000000; 5752 5753 /* Currently, there is a high possibility to get wave ID mismatch 5754 * between ME and GDS, leading to a hw deadlock, because ME generates 5755 * different wave IDs than the GDS expects. This situation happens 5756 * randomly when at least 5 compute pipes use GDS ordered append. 5757 * The wave IDs generated by ME are also wrong after suspend/resume. 5758 * Those are probably bugs somewhere else in the kernel driver. 5759 * 5760 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5761 * GDS to 0 for this ring (me/pipe). 5762 */ 5763 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5764 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5765 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID); 5766 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5767 } 5768 5769 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5770 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5771 amdgpu_ring_write(ring, 5772 #ifdef __BIG_ENDIAN 5773 (2 << 0) | 5774 #endif 5775 lower_32_bits(ib->gpu_addr)); 5776 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5777 amdgpu_ring_write(ring, control); 5778 } 5779 5780 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5781 u64 seq, unsigned flags) 5782 { 5783 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5784 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5785 5786 /* RELEASE_MEM - flush caches, send int */ 5787 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5788 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ | 5789 PACKET3_RELEASE_MEM_GCR_GL2_WB | 5790 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */ 5791 PACKET3_RELEASE_MEM_GCR_GLM_WB | 5792 PACKET3_RELEASE_MEM_CACHE_POLICY(3) | 5793 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5794 PACKET3_RELEASE_MEM_EVENT_INDEX(5))); 5795 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) | 5796 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); 5797 5798 /* 5799 * the address should be Qword aligned if 64bit write, Dword 5800 * aligned if only send 32bit data low (discard data high) 5801 */ 5802 if (write64bit) 5803 BUG_ON(addr & 0x7); 5804 else 5805 BUG_ON(addr & 0x3); 5806 amdgpu_ring_write(ring, lower_32_bits(addr)); 5807 amdgpu_ring_write(ring, upper_32_bits(addr)); 5808 amdgpu_ring_write(ring, lower_32_bits(seq)); 5809 amdgpu_ring_write(ring, upper_32_bits(seq)); 5810 amdgpu_ring_write(ring, ring->is_mes_queue ? 5811 (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0); 5812 } 5813 5814 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5815 { 5816 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5817 uint32_t seq = ring->fence_drv.sync_seq; 5818 uint64_t addr = ring->fence_drv.gpu_addr; 5819 5820 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr), 5821 upper_32_bits(addr), seq, 0xffffffff, 4); 5822 } 5823 5824 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 5825 uint16_t pasid, uint32_t flush_type, 5826 bool all_hub, uint8_t dst_sel) 5827 { 5828 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 5829 amdgpu_ring_write(ring, 5830 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) | 5831 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 5832 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 5833 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 5834 } 5835 5836 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5837 unsigned vmid, uint64_t pd_addr) 5838 { 5839 if (ring->is_mes_queue) 5840 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0); 5841 else 5842 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5843 5844 /* compute doesn't have PFP */ 5845 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5846 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5847 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5848 amdgpu_ring_write(ring, 0x0); 5849 } 5850 5851 /* Make sure that we can't skip the SET_Q_MODE packets when the VM 5852 * changed in any way. 5853 */ 5854 ring->set_q_mode_offs = 0; 5855 ring->set_q_mode_ptr = NULL; 5856 } 5857 5858 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5859 u64 seq, unsigned int flags) 5860 { 5861 struct amdgpu_device *adev = ring->adev; 5862 5863 /* we only allocate 32bit for each seq wb address */ 5864 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5865 5866 /* write fence seq to the "addr" */ 5867 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5868 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5869 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5870 amdgpu_ring_write(ring, lower_32_bits(addr)); 5871 amdgpu_ring_write(ring, upper_32_bits(addr)); 5872 amdgpu_ring_write(ring, lower_32_bits(seq)); 5873 5874 if (flags & AMDGPU_FENCE_FLAG_INT) { 5875 /* set register to trigger INT */ 5876 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5877 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5878 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5879 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS)); 5880 amdgpu_ring_write(ring, 0); 5881 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5882 } 5883 } 5884 5885 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, 5886 uint32_t flags) 5887 { 5888 uint32_t dw2 = 0; 5889 5890 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5891 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5892 /* set load_global_config & load_global_uconfig */ 5893 dw2 |= 0x8001; 5894 /* set load_cs_sh_regs */ 5895 dw2 |= 0x01000000; 5896 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5897 dw2 |= 0x10002; 5898 } 5899 5900 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5901 amdgpu_ring_write(ring, dw2); 5902 amdgpu_ring_write(ring, 0); 5903 } 5904 5905 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 5906 uint64_t addr) 5907 { 5908 unsigned ret; 5909 5910 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5911 amdgpu_ring_write(ring, lower_32_bits(addr)); 5912 amdgpu_ring_write(ring, upper_32_bits(addr)); 5913 /* discard following DWs if *cond_exec_gpu_addr==0 */ 5914 amdgpu_ring_write(ring, 0); 5915 ret = ring->wptr & ring->buf_mask; 5916 /* patch dummy value later */ 5917 amdgpu_ring_write(ring, 0); 5918 5919 return ret; 5920 } 5921 5922 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring, 5923 u64 shadow_va, u64 csa_va, 5924 u64 gds_va, bool init_shadow, 5925 int vmid) 5926 { 5927 struct amdgpu_device *adev = ring->adev; 5928 unsigned int offs, end; 5929 5930 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj) 5931 return; 5932 5933 /* 5934 * The logic here isn't easy to understand because we need to keep state 5935 * accross multiple executions of the function as well as between the 5936 * CPU and GPU. The general idea is that the newly written GPU command 5937 * has a condition on the previous one and only executed if really 5938 * necessary. 5939 */ 5940 5941 /* 5942 * The dw in the NOP controls if the next SET_Q_MODE packet should be 5943 * executed or not. Reserve 64bits just to be on the save side. 5944 */ 5945 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1)); 5946 offs = ring->wptr & ring->buf_mask; 5947 5948 /* 5949 * We start with skipping the prefix SET_Q_MODE and always executing 5950 * the postfix SET_Q_MODE packet. This is changed below with a 5951 * WRITE_DATA command when the postfix executed. 5952 */ 5953 amdgpu_ring_write(ring, shadow_va ? 1 : 0); 5954 amdgpu_ring_write(ring, 0); 5955 5956 if (ring->set_q_mode_offs) { 5957 uint64_t addr; 5958 5959 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 5960 addr += ring->set_q_mode_offs << 2; 5961 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr); 5962 } 5963 5964 /* 5965 * When the postfix SET_Q_MODE packet executes we need to make sure that the 5966 * next prefix SET_Q_MODE packet executes as well. 5967 */ 5968 if (!shadow_va) { 5969 uint64_t addr; 5970 5971 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 5972 addr += offs << 2; 5973 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5974 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); 5975 amdgpu_ring_write(ring, lower_32_bits(addr)); 5976 amdgpu_ring_write(ring, upper_32_bits(addr)); 5977 amdgpu_ring_write(ring, 0x1); 5978 } 5979 5980 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7)); 5981 amdgpu_ring_write(ring, lower_32_bits(shadow_va)); 5982 amdgpu_ring_write(ring, upper_32_bits(shadow_va)); 5983 amdgpu_ring_write(ring, lower_32_bits(gds_va)); 5984 amdgpu_ring_write(ring, upper_32_bits(gds_va)); 5985 amdgpu_ring_write(ring, lower_32_bits(csa_va)); 5986 amdgpu_ring_write(ring, upper_32_bits(csa_va)); 5987 amdgpu_ring_write(ring, shadow_va ? 5988 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0); 5989 amdgpu_ring_write(ring, init_shadow ? 5990 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0); 5991 5992 if (ring->set_q_mode_offs) 5993 amdgpu_ring_patch_cond_exec(ring, end); 5994 5995 if (shadow_va) { 5996 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid; 5997 5998 /* 5999 * If the tokens match try to skip the last postfix SET_Q_MODE 6000 * packet to avoid saving/restoring the state all the time. 6001 */ 6002 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token) 6003 *ring->set_q_mode_ptr = 0; 6004 6005 ring->set_q_mode_token = token; 6006 } else { 6007 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs]; 6008 } 6009 6010 ring->set_q_mode_offs = offs; 6011 } 6012 6013 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring) 6014 { 6015 int i, r = 0; 6016 struct amdgpu_device *adev = ring->adev; 6017 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 6018 struct amdgpu_ring *kiq_ring = &kiq->ring; 6019 unsigned long flags; 6020 6021 if (adev->enable_mes) 6022 return -EINVAL; 6023 6024 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 6025 return -EINVAL; 6026 6027 spin_lock_irqsave(&kiq->ring_lock, flags); 6028 6029 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { 6030 spin_unlock_irqrestore(&kiq->ring_lock, flags); 6031 return -ENOMEM; 6032 } 6033 6034 /* assert preemption condition */ 6035 amdgpu_ring_set_preempt_cond_exec(ring, false); 6036 6037 /* assert IB preemption, emit the trailing fence */ 6038 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, 6039 ring->trail_fence_gpu_addr, 6040 ++ring->trail_seq); 6041 amdgpu_ring_commit(kiq_ring); 6042 6043 spin_unlock_irqrestore(&kiq->ring_lock, flags); 6044 6045 /* poll the trailing fence */ 6046 for (i = 0; i < adev->usec_timeout; i++) { 6047 if (ring->trail_seq == 6048 le32_to_cpu(*(ring->trail_fence_cpu_addr))) 6049 break; 6050 udelay(1); 6051 } 6052 6053 if (i >= adev->usec_timeout) { 6054 r = -EINVAL; 6055 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx); 6056 } 6057 6058 /* deassert preemption condition */ 6059 amdgpu_ring_set_preempt_cond_exec(ring, true); 6060 return r; 6061 } 6062 6063 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) 6064 { 6065 struct amdgpu_device *adev = ring->adev; 6066 struct v10_de_ib_state de_payload = {0}; 6067 uint64_t offset, gds_addr, de_payload_gpu_addr; 6068 void *de_payload_cpu_addr; 6069 int cnt; 6070 6071 if (ring->is_mes_queue) { 6072 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 6073 gfx[0].gfx_meta_data) + 6074 offsetof(struct v10_gfx_meta_data, de_payload); 6075 de_payload_gpu_addr = 6076 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 6077 de_payload_cpu_addr = 6078 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); 6079 6080 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 6081 gfx[0].gds_backup) + 6082 offsetof(struct v10_gfx_meta_data, de_payload); 6083 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 6084 } else { 6085 offset = offsetof(struct v10_gfx_meta_data, de_payload); 6086 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 6087 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 6088 6089 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 6090 AMDGPU_CSA_SIZE - adev->gds.gds_size, 6091 PAGE_SIZE); 6092 } 6093 6094 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 6095 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 6096 6097 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 6098 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 6099 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 6100 WRITE_DATA_DST_SEL(8) | 6101 WR_CONFIRM) | 6102 WRITE_DATA_CACHE_POLICY(0)); 6103 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 6104 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 6105 6106 if (resume) 6107 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 6108 sizeof(de_payload) >> 2); 6109 else 6110 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 6111 sizeof(de_payload) >> 2); 6112 } 6113 6114 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 6115 bool secure) 6116 { 6117 uint32_t v = secure ? FRAME_TMZ : 0; 6118 6119 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 6120 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 6121 } 6122 6123 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6124 uint32_t reg_val_offs) 6125 { 6126 struct amdgpu_device *adev = ring->adev; 6127 6128 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6129 amdgpu_ring_write(ring, 0 | /* src: register*/ 6130 (5 << 8) | /* dst: memory */ 6131 (1 << 20)); /* write confirm */ 6132 amdgpu_ring_write(ring, reg); 6133 amdgpu_ring_write(ring, 0); 6134 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6135 reg_val_offs * 4)); 6136 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6137 reg_val_offs * 4)); 6138 } 6139 6140 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6141 uint32_t val) 6142 { 6143 uint32_t cmd = 0; 6144 6145 switch (ring->funcs->type) { 6146 case AMDGPU_RING_TYPE_GFX: 6147 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6148 break; 6149 case AMDGPU_RING_TYPE_KIQ: 6150 cmd = (1 << 16); /* no inc addr */ 6151 break; 6152 default: 6153 cmd = WR_CONFIRM; 6154 break; 6155 } 6156 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6157 amdgpu_ring_write(ring, cmd); 6158 amdgpu_ring_write(ring, reg); 6159 amdgpu_ring_write(ring, 0); 6160 amdgpu_ring_write(ring, val); 6161 } 6162 6163 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 6164 uint32_t val, uint32_t mask) 6165 { 6166 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 6167 } 6168 6169 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 6170 uint32_t reg0, uint32_t reg1, 6171 uint32_t ref, uint32_t mask) 6172 { 6173 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6174 6175 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 6176 ref, mask, 0x20); 6177 } 6178 6179 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring, 6180 unsigned vmid) 6181 { 6182 struct amdgpu_device *adev = ring->adev; 6183 uint32_t value = 0; 6184 6185 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 6186 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 6187 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 6188 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 6189 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 6190 WREG32_SOC15(GC, 0, regSQ_CMD, value); 6191 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 6192 } 6193 6194 static void 6195 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6196 uint32_t me, uint32_t pipe, 6197 enum amdgpu_interrupt_state state) 6198 { 6199 uint32_t cp_int_cntl, cp_int_cntl_reg; 6200 6201 if (!me) { 6202 switch (pipe) { 6203 case 0: 6204 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 6205 break; 6206 case 1: 6207 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 6208 break; 6209 default: 6210 DRM_DEBUG("invalid pipe %d\n", pipe); 6211 return; 6212 } 6213 } else { 6214 DRM_DEBUG("invalid me %d\n", me); 6215 return; 6216 } 6217 6218 switch (state) { 6219 case AMDGPU_IRQ_STATE_DISABLE: 6220 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6221 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6222 TIME_STAMP_INT_ENABLE, 0); 6223 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6224 GENERIC0_INT_ENABLE, 0); 6225 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6226 break; 6227 case AMDGPU_IRQ_STATE_ENABLE: 6228 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6229 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6230 TIME_STAMP_INT_ENABLE, 1); 6231 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6232 GENERIC0_INT_ENABLE, 1); 6233 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6234 break; 6235 default: 6236 break; 6237 } 6238 } 6239 6240 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6241 int me, int pipe, 6242 enum amdgpu_interrupt_state state) 6243 { 6244 u32 mec_int_cntl, mec_int_cntl_reg; 6245 6246 /* 6247 * amdgpu controls only the first MEC. That's why this function only 6248 * handles the setting of interrupts for this specific MEC. All other 6249 * pipes' interrupts are set by amdkfd. 6250 */ 6251 6252 if (me == 1) { 6253 switch (pipe) { 6254 case 0: 6255 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6256 break; 6257 case 1: 6258 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 6259 break; 6260 case 2: 6261 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 6262 break; 6263 case 3: 6264 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 6265 break; 6266 default: 6267 DRM_DEBUG("invalid pipe %d\n", pipe); 6268 return; 6269 } 6270 } else { 6271 DRM_DEBUG("invalid me %d\n", me); 6272 return; 6273 } 6274 6275 switch (state) { 6276 case AMDGPU_IRQ_STATE_DISABLE: 6277 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6278 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6279 TIME_STAMP_INT_ENABLE, 0); 6280 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6281 GENERIC0_INT_ENABLE, 0); 6282 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6283 break; 6284 case AMDGPU_IRQ_STATE_ENABLE: 6285 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6286 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6287 TIME_STAMP_INT_ENABLE, 1); 6288 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6289 GENERIC0_INT_ENABLE, 1); 6290 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6291 break; 6292 default: 6293 break; 6294 } 6295 } 6296 6297 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6298 struct amdgpu_irq_src *src, 6299 unsigned type, 6300 enum amdgpu_interrupt_state state) 6301 { 6302 switch (type) { 6303 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6304 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state); 6305 break; 6306 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP: 6307 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state); 6308 break; 6309 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6310 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6311 break; 6312 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6313 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6314 break; 6315 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6316 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6317 break; 6318 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6319 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6320 break; 6321 default: 6322 break; 6323 } 6324 return 0; 6325 } 6326 6327 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, 6328 struct amdgpu_irq_src *source, 6329 struct amdgpu_iv_entry *entry) 6330 { 6331 int i; 6332 u8 me_id, pipe_id, queue_id; 6333 struct amdgpu_ring *ring; 6334 uint32_t mes_queue_id = entry->src_data[0]; 6335 6336 DRM_DEBUG("IH: CP EOP\n"); 6337 6338 if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) { 6339 struct amdgpu_mes_queue *queue; 6340 6341 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK; 6342 6343 spin_lock(&adev->mes.queue_id_lock); 6344 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id); 6345 if (queue) { 6346 DRM_DEBUG("process mes queue id = %d\n", mes_queue_id); 6347 amdgpu_fence_process(queue->ring); 6348 } 6349 spin_unlock(&adev->mes.queue_id_lock); 6350 } else { 6351 me_id = (entry->ring_id & 0x0c) >> 2; 6352 pipe_id = (entry->ring_id & 0x03) >> 0; 6353 queue_id = (entry->ring_id & 0x70) >> 4; 6354 6355 switch (me_id) { 6356 case 0: 6357 if (pipe_id == 0) 6358 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6359 else 6360 amdgpu_fence_process(&adev->gfx.gfx_ring[1]); 6361 break; 6362 case 1: 6363 case 2: 6364 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6365 ring = &adev->gfx.compute_ring[i]; 6366 /* Per-queue interrupt is supported for MEC starting from VI. 6367 * The interrupt can only be enabled/disabled per pipe instead 6368 * of per queue. 6369 */ 6370 if ((ring->me == me_id) && 6371 (ring->pipe == pipe_id) && 6372 (ring->queue == queue_id)) 6373 amdgpu_fence_process(ring); 6374 } 6375 break; 6376 } 6377 } 6378 6379 return 0; 6380 } 6381 6382 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6383 struct amdgpu_irq_src *source, 6384 unsigned int type, 6385 enum amdgpu_interrupt_state state) 6386 { 6387 u32 cp_int_cntl_reg, cp_int_cntl; 6388 int i, j; 6389 6390 switch (state) { 6391 case AMDGPU_IRQ_STATE_DISABLE: 6392 case AMDGPU_IRQ_STATE_ENABLE: 6393 for (i = 0; i < adev->gfx.me.num_me; i++) { 6394 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6395 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6396 6397 if (cp_int_cntl_reg) { 6398 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6399 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6400 PRIV_REG_INT_ENABLE, 6401 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6402 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6403 } 6404 } 6405 } 6406 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6407 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6408 /* MECs start at 1 */ 6409 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6410 6411 if (cp_int_cntl_reg) { 6412 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6413 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6414 PRIV_REG_INT_ENABLE, 6415 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6416 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6417 } 6418 } 6419 } 6420 break; 6421 default: 6422 break; 6423 } 6424 6425 return 0; 6426 } 6427 6428 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6429 struct amdgpu_irq_src *source, 6430 unsigned type, 6431 enum amdgpu_interrupt_state state) 6432 { 6433 u32 cp_int_cntl_reg, cp_int_cntl; 6434 int i, j; 6435 6436 switch (state) { 6437 case AMDGPU_IRQ_STATE_DISABLE: 6438 case AMDGPU_IRQ_STATE_ENABLE: 6439 for (i = 0; i < adev->gfx.me.num_me; i++) { 6440 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6441 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6442 6443 if (cp_int_cntl_reg) { 6444 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6445 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6446 OPCODE_ERROR_INT_ENABLE, 6447 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6448 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6449 } 6450 } 6451 } 6452 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6453 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6454 /* MECs start at 1 */ 6455 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6456 6457 if (cp_int_cntl_reg) { 6458 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6459 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6460 OPCODE_ERROR_INT_ENABLE, 6461 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6462 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6463 } 6464 } 6465 } 6466 break; 6467 default: 6468 break; 6469 } 6470 return 0; 6471 } 6472 6473 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6474 struct amdgpu_irq_src *source, 6475 unsigned int type, 6476 enum amdgpu_interrupt_state state) 6477 { 6478 u32 cp_int_cntl_reg, cp_int_cntl; 6479 int i, j; 6480 6481 switch (state) { 6482 case AMDGPU_IRQ_STATE_DISABLE: 6483 case AMDGPU_IRQ_STATE_ENABLE: 6484 for (i = 0; i < adev->gfx.me.num_me; i++) { 6485 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6486 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6487 6488 if (cp_int_cntl_reg) { 6489 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6490 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6491 PRIV_INSTR_INT_ENABLE, 6492 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6493 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6494 } 6495 } 6496 } 6497 break; 6498 default: 6499 break; 6500 } 6501 6502 return 0; 6503 } 6504 6505 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, 6506 struct amdgpu_iv_entry *entry) 6507 { 6508 u8 me_id, pipe_id, queue_id; 6509 struct amdgpu_ring *ring; 6510 int i; 6511 6512 me_id = (entry->ring_id & 0x0c) >> 2; 6513 pipe_id = (entry->ring_id & 0x03) >> 0; 6514 queue_id = (entry->ring_id & 0x70) >> 4; 6515 6516 switch (me_id) { 6517 case 0: 6518 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6519 ring = &adev->gfx.gfx_ring[i]; 6520 if (ring->me == me_id && ring->pipe == pipe_id && 6521 ring->queue == queue_id) 6522 drm_sched_fault(&ring->sched); 6523 } 6524 break; 6525 case 1: 6526 case 2: 6527 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6528 ring = &adev->gfx.compute_ring[i]; 6529 if (ring->me == me_id && ring->pipe == pipe_id && 6530 ring->queue == queue_id) 6531 drm_sched_fault(&ring->sched); 6532 } 6533 break; 6534 default: 6535 BUG(); 6536 break; 6537 } 6538 } 6539 6540 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev, 6541 struct amdgpu_irq_src *source, 6542 struct amdgpu_iv_entry *entry) 6543 { 6544 DRM_ERROR("Illegal register access in command stream\n"); 6545 gfx_v11_0_handle_priv_fault(adev, entry); 6546 return 0; 6547 } 6548 6549 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev, 6550 struct amdgpu_irq_src *source, 6551 struct amdgpu_iv_entry *entry) 6552 { 6553 DRM_ERROR("Illegal opcode in command stream \n"); 6554 gfx_v11_0_handle_priv_fault(adev, entry); 6555 return 0; 6556 } 6557 6558 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev, 6559 struct amdgpu_irq_src *source, 6560 struct amdgpu_iv_entry *entry) 6561 { 6562 DRM_ERROR("Illegal instruction in command stream\n"); 6563 gfx_v11_0_handle_priv_fault(adev, entry); 6564 return 0; 6565 } 6566 6567 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev, 6568 struct amdgpu_irq_src *source, 6569 struct amdgpu_iv_entry *entry) 6570 { 6571 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq) 6572 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry); 6573 6574 return 0; 6575 } 6576 6577 #if 0 6578 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev, 6579 struct amdgpu_irq_src *src, 6580 unsigned int type, 6581 enum amdgpu_interrupt_state state) 6582 { 6583 uint32_t tmp, target; 6584 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring); 6585 6586 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6587 target += ring->pipe; 6588 6589 switch (type) { 6590 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 6591 if (state == AMDGPU_IRQ_STATE_DISABLE) { 6592 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6593 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6594 GENERIC2_INT_ENABLE, 0); 6595 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6596 6597 tmp = RREG32_SOC15_IP(GC, target); 6598 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6599 GENERIC2_INT_ENABLE, 0); 6600 WREG32_SOC15_IP(GC, target, tmp); 6601 } else { 6602 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6603 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6604 GENERIC2_INT_ENABLE, 1); 6605 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6606 6607 tmp = RREG32_SOC15_IP(GC, target); 6608 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6609 GENERIC2_INT_ENABLE, 1); 6610 WREG32_SOC15_IP(GC, target, tmp); 6611 } 6612 break; 6613 default: 6614 BUG(); /* kiq only support GENERIC2_INT now */ 6615 break; 6616 } 6617 return 0; 6618 } 6619 #endif 6620 6621 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring) 6622 { 6623 const unsigned int gcr_cntl = 6624 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) | 6625 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) | 6626 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) | 6627 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) | 6628 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) | 6629 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) | 6630 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) | 6631 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1); 6632 6633 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 6634 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); 6635 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ 6636 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6637 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6638 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6639 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6640 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6641 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ 6642 } 6643 6644 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid) 6645 { 6646 struct amdgpu_device *adev = ring->adev; 6647 int r; 6648 6649 if (amdgpu_sriov_vf(adev)) 6650 return -EINVAL; 6651 6652 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false); 6653 if (r) 6654 return r; 6655 6656 r = amdgpu_bo_reserve(ring->mqd_obj, false); 6657 if (unlikely(r != 0)) { 6658 dev_err(adev->dev, "fail to resv mqd_obj\n"); 6659 return r; 6660 } 6661 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 6662 if (!r) { 6663 r = gfx_v11_0_kgq_init_queue(ring, true); 6664 amdgpu_bo_kunmap(ring->mqd_obj); 6665 ring->mqd_ptr = NULL; 6666 } 6667 amdgpu_bo_unreserve(ring->mqd_obj); 6668 if (r) { 6669 dev_err(adev->dev, "fail to unresv mqd_obj\n"); 6670 return r; 6671 } 6672 6673 r = amdgpu_mes_map_legacy_queue(adev, ring); 6674 if (r) { 6675 dev_err(adev->dev, "failed to remap kgq\n"); 6676 return r; 6677 } 6678 6679 return amdgpu_ring_test_ring(ring); 6680 } 6681 6682 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, unsigned int vmid) 6683 { 6684 struct amdgpu_device *adev = ring->adev; 6685 int r = 0; 6686 6687 if (amdgpu_sriov_vf(adev)) 6688 return -EINVAL; 6689 6690 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true); 6691 if (r) { 6692 dev_err(adev->dev, "reset via MMIO failed %d\n", r); 6693 return r; 6694 } 6695 6696 r = amdgpu_bo_reserve(ring->mqd_obj, false); 6697 if (unlikely(r != 0)) { 6698 dev_err(adev->dev, "fail to resv mqd_obj\n"); 6699 return r; 6700 } 6701 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 6702 if (!r) { 6703 r = gfx_v11_0_kcq_init_queue(ring, true); 6704 amdgpu_bo_kunmap(ring->mqd_obj); 6705 ring->mqd_ptr = NULL; 6706 } 6707 amdgpu_bo_unreserve(ring->mqd_obj); 6708 if (r) { 6709 dev_err(adev->dev, "fail to unresv mqd_obj\n"); 6710 return r; 6711 } 6712 r = amdgpu_mes_map_legacy_queue(adev, ring); 6713 if (r) { 6714 dev_err(adev->dev, "failed to remap kcq\n"); 6715 return r; 6716 } 6717 6718 return amdgpu_ring_test_ring(ring); 6719 } 6720 6721 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 6722 { 6723 struct amdgpu_device *adev = ip_block->adev; 6724 uint32_t i, j, k, reg, index = 0; 6725 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6726 6727 if (!adev->gfx.ip_dump_core) 6728 return; 6729 6730 for (i = 0; i < reg_count; i++) 6731 drm_printf(p, "%-50s \t 0x%08x\n", 6732 gc_reg_list_11_0[i].reg_name, 6733 adev->gfx.ip_dump_core[i]); 6734 6735 /* print compute queue registers for all instances */ 6736 if (!adev->gfx.ip_dump_compute_queues) 6737 return; 6738 6739 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6740 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 6741 adev->gfx.mec.num_mec, 6742 adev->gfx.mec.num_pipe_per_mec, 6743 adev->gfx.mec.num_queue_per_pipe); 6744 6745 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6746 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6747 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6748 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 6749 for (reg = 0; reg < reg_count; reg++) { 6750 drm_printf(p, "%-50s \t 0x%08x\n", 6751 gc_cp_reg_list_11[reg].reg_name, 6752 adev->gfx.ip_dump_compute_queues[index + reg]); 6753 } 6754 index += reg_count; 6755 } 6756 } 6757 } 6758 6759 /* print gfx queue registers for all instances */ 6760 if (!adev->gfx.ip_dump_gfx_queues) 6761 return; 6762 6763 index = 0; 6764 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 6765 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n", 6766 adev->gfx.me.num_me, 6767 adev->gfx.me.num_pipe_per_me, 6768 adev->gfx.me.num_queue_per_pipe); 6769 6770 for (i = 0; i < adev->gfx.me.num_me; i++) { 6771 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6772 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 6773 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k); 6774 for (reg = 0; reg < reg_count; reg++) { 6775 drm_printf(p, "%-50s \t 0x%08x\n", 6776 gc_gfx_queue_reg_list_11[reg].reg_name, 6777 adev->gfx.ip_dump_gfx_queues[index + reg]); 6778 } 6779 index += reg_count; 6780 } 6781 } 6782 } 6783 } 6784 6785 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block) 6786 { 6787 struct amdgpu_device *adev = ip_block->adev; 6788 uint32_t i, j, k, reg, index = 0; 6789 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6790 6791 if (!adev->gfx.ip_dump_core) 6792 return; 6793 6794 amdgpu_gfx_off_ctrl(adev, false); 6795 for (i = 0; i < reg_count; i++) 6796 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i])); 6797 amdgpu_gfx_off_ctrl(adev, true); 6798 6799 /* dump compute queue registers for all instances */ 6800 if (!adev->gfx.ip_dump_compute_queues) 6801 return; 6802 6803 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6804 amdgpu_gfx_off_ctrl(adev, false); 6805 mutex_lock(&adev->srbm_mutex); 6806 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6807 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6808 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6809 /* ME0 is for GFX so start from 1 for CP */ 6810 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0); 6811 for (reg = 0; reg < reg_count; reg++) { 6812 adev->gfx.ip_dump_compute_queues[index + reg] = 6813 RREG32(SOC15_REG_ENTRY_OFFSET( 6814 gc_cp_reg_list_11[reg])); 6815 } 6816 index += reg_count; 6817 } 6818 } 6819 } 6820 soc21_grbm_select(adev, 0, 0, 0, 0); 6821 mutex_unlock(&adev->srbm_mutex); 6822 amdgpu_gfx_off_ctrl(adev, true); 6823 6824 /* dump gfx queue registers for all instances */ 6825 if (!adev->gfx.ip_dump_gfx_queues) 6826 return; 6827 6828 index = 0; 6829 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 6830 amdgpu_gfx_off_ctrl(adev, false); 6831 mutex_lock(&adev->srbm_mutex); 6832 for (i = 0; i < adev->gfx.me.num_me; i++) { 6833 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6834 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 6835 soc21_grbm_select(adev, i, j, k, 0); 6836 6837 for (reg = 0; reg < reg_count; reg++) { 6838 adev->gfx.ip_dump_gfx_queues[index + reg] = 6839 RREG32(SOC15_REG_ENTRY_OFFSET( 6840 gc_gfx_queue_reg_list_11[reg])); 6841 } 6842 index += reg_count; 6843 } 6844 } 6845 } 6846 soc21_grbm_select(adev, 0, 0, 0, 0); 6847 mutex_unlock(&adev->srbm_mutex); 6848 amdgpu_gfx_off_ctrl(adev, true); 6849 } 6850 6851 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 6852 { 6853 /* Emit the cleaner shader */ 6854 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 6855 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 6856 } 6857 6858 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring) 6859 { 6860 amdgpu_gfx_profile_ring_begin_use(ring); 6861 6862 amdgpu_gfx_enforce_isolation_ring_begin_use(ring); 6863 } 6864 6865 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) 6866 { 6867 amdgpu_gfx_profile_ring_end_use(ring); 6868 6869 amdgpu_gfx_enforce_isolation_ring_end_use(ring); 6870 } 6871 6872 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { 6873 .name = "gfx_v11_0", 6874 .early_init = gfx_v11_0_early_init, 6875 .late_init = gfx_v11_0_late_init, 6876 .sw_init = gfx_v11_0_sw_init, 6877 .sw_fini = gfx_v11_0_sw_fini, 6878 .hw_init = gfx_v11_0_hw_init, 6879 .hw_fini = gfx_v11_0_hw_fini, 6880 .suspend = gfx_v11_0_suspend, 6881 .resume = gfx_v11_0_resume, 6882 .is_idle = gfx_v11_0_is_idle, 6883 .wait_for_idle = gfx_v11_0_wait_for_idle, 6884 .soft_reset = gfx_v11_0_soft_reset, 6885 .check_soft_reset = gfx_v11_0_check_soft_reset, 6886 .post_soft_reset = gfx_v11_0_post_soft_reset, 6887 .set_clockgating_state = gfx_v11_0_set_clockgating_state, 6888 .set_powergating_state = gfx_v11_0_set_powergating_state, 6889 .get_clockgating_state = gfx_v11_0_get_clockgating_state, 6890 .dump_ip_state = gfx_v11_ip_dump, 6891 .print_ip_state = gfx_v11_ip_print, 6892 }; 6893 6894 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { 6895 .type = AMDGPU_RING_TYPE_GFX, 6896 .align_mask = 0xff, 6897 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6898 .support_64bit_ptrs = true, 6899 .secure_submission_supported = true, 6900 .get_rptr = gfx_v11_0_ring_get_rptr_gfx, 6901 .get_wptr = gfx_v11_0_ring_get_wptr_gfx, 6902 .set_wptr = gfx_v11_0_ring_set_wptr_gfx, 6903 .emit_frame_size = /* totally 247 maximum if 16 IBs */ 6904 5 + /* update_spm_vmid */ 6905 5 + /* COND_EXEC */ 6906 22 + /* SET_Q_PREEMPTION_MODE */ 6907 7 + /* PIPELINE_SYNC */ 6908 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6909 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6910 4 + /* VM_FLUSH */ 6911 8 + /* FENCE for VM_FLUSH */ 6912 20 + /* GDS switch */ 6913 5 + /* COND_EXEC */ 6914 7 + /* HDP_flush */ 6915 4 + /* VGT_flush */ 6916 31 + /* DE_META */ 6917 3 + /* CNTX_CTRL */ 6918 5 + /* HDP_INVL */ 6919 22 + /* SET_Q_PREEMPTION_MODE */ 6920 8 + 8 + /* FENCE x2 */ 6921 8 + /* gfx_v11_0_emit_mem_sync */ 6922 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 6923 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */ 6924 .emit_ib = gfx_v11_0_ring_emit_ib_gfx, 6925 .emit_fence = gfx_v11_0_ring_emit_fence, 6926 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 6927 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 6928 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 6929 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 6930 .test_ring = gfx_v11_0_ring_test_ring, 6931 .test_ib = gfx_v11_0_ring_test_ib, 6932 .insert_nop = gfx_v11_ring_insert_nop, 6933 .pad_ib = amdgpu_ring_generic_pad_ib, 6934 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl, 6935 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow, 6936 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec, 6937 .preempt_ib = gfx_v11_0_ring_preempt_ib, 6938 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl, 6939 .emit_wreg = gfx_v11_0_ring_emit_wreg, 6940 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 6941 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 6942 .soft_recovery = gfx_v11_0_ring_soft_recovery, 6943 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 6944 .reset = gfx_v11_0_reset_kgq, 6945 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 6946 .begin_use = gfx_v11_0_ring_begin_use, 6947 .end_use = gfx_v11_0_ring_end_use, 6948 }; 6949 6950 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = { 6951 .type = AMDGPU_RING_TYPE_COMPUTE, 6952 .align_mask = 0xff, 6953 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6954 .support_64bit_ptrs = true, 6955 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 6956 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 6957 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 6958 .emit_frame_size = 6959 5 + /* update_spm_vmid */ 6960 20 + /* gfx_v11_0_ring_emit_gds_switch */ 6961 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 6962 5 + /* hdp invalidate */ 6963 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 6964 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6965 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6966 2 + /* gfx_v11_0_ring_emit_vm_flush */ 6967 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */ 6968 8 + /* gfx_v11_0_emit_mem_sync */ 6969 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 6970 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 6971 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 6972 .emit_fence = gfx_v11_0_ring_emit_fence, 6973 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 6974 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 6975 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 6976 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 6977 .test_ring = gfx_v11_0_ring_test_ring, 6978 .test_ib = gfx_v11_0_ring_test_ib, 6979 .insert_nop = gfx_v11_ring_insert_nop, 6980 .pad_ib = amdgpu_ring_generic_pad_ib, 6981 .emit_wreg = gfx_v11_0_ring_emit_wreg, 6982 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 6983 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 6984 .soft_recovery = gfx_v11_0_ring_soft_recovery, 6985 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 6986 .reset = gfx_v11_0_reset_kcq, 6987 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 6988 .begin_use = gfx_v11_0_ring_begin_use, 6989 .end_use = gfx_v11_0_ring_end_use, 6990 }; 6991 6992 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = { 6993 .type = AMDGPU_RING_TYPE_KIQ, 6994 .align_mask = 0xff, 6995 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6996 .support_64bit_ptrs = true, 6997 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 6998 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 6999 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7000 .emit_frame_size = 7001 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7002 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7003 5 + /*hdp invalidate */ 7004 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7005 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7006 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7007 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7008 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7009 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7010 .emit_fence = gfx_v11_0_ring_emit_fence_kiq, 7011 .test_ring = gfx_v11_0_ring_test_ring, 7012 .test_ib = gfx_v11_0_ring_test_ib, 7013 .insert_nop = amdgpu_ring_insert_nop, 7014 .pad_ib = amdgpu_ring_generic_pad_ib, 7015 .emit_rreg = gfx_v11_0_ring_emit_rreg, 7016 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7017 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7018 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7019 }; 7020 7021 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev) 7022 { 7023 int i; 7024 7025 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq; 7026 7027 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7028 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx; 7029 7030 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7031 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute; 7032 } 7033 7034 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = { 7035 .set = gfx_v11_0_set_eop_interrupt_state, 7036 .process = gfx_v11_0_eop_irq, 7037 }; 7038 7039 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = { 7040 .set = gfx_v11_0_set_priv_reg_fault_state, 7041 .process = gfx_v11_0_priv_reg_irq, 7042 }; 7043 7044 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = { 7045 .set = gfx_v11_0_set_bad_op_fault_state, 7046 .process = gfx_v11_0_bad_op_irq, 7047 }; 7048 7049 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = { 7050 .set = gfx_v11_0_set_priv_inst_fault_state, 7051 .process = gfx_v11_0_priv_inst_irq, 7052 }; 7053 7054 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = { 7055 .process = gfx_v11_0_rlc_gc_fed_irq, 7056 }; 7057 7058 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev) 7059 { 7060 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7061 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs; 7062 7063 adev->gfx.priv_reg_irq.num_types = 1; 7064 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs; 7065 7066 adev->gfx.bad_op_irq.num_types = 1; 7067 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs; 7068 7069 adev->gfx.priv_inst_irq.num_types = 1; 7070 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs; 7071 7072 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */ 7073 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs; 7074 7075 } 7076 7077 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev) 7078 { 7079 if (adev->flags & AMD_IS_APU) 7080 adev->gfx.imu.mode = MISSION_MODE; 7081 else 7082 adev->gfx.imu.mode = DEBUG_MODE; 7083 7084 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs; 7085 } 7086 7087 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev) 7088 { 7089 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs; 7090 } 7091 7092 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev) 7093 { 7094 unsigned total_cu = adev->gfx.config.max_cu_per_sh * 7095 adev->gfx.config.max_sh_per_se * 7096 adev->gfx.config.max_shader_engines; 7097 7098 adev->gds.gds_size = 0x1000; 7099 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1; 7100 adev->gds.gws_size = 64; 7101 adev->gds.oa_size = 16; 7102 } 7103 7104 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev) 7105 { 7106 /* set gfx eng mqd */ 7107 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size = 7108 sizeof(struct v11_gfx_mqd); 7109 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd = 7110 gfx_v11_0_gfx_mqd_init; 7111 /* set compute eng mqd */ 7112 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size = 7113 sizeof(struct v11_compute_mqd); 7114 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd = 7115 gfx_v11_0_compute_mqd_init; 7116 } 7117 7118 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, 7119 u32 bitmap) 7120 { 7121 u32 data; 7122 7123 if (!bitmap) 7124 return; 7125 7126 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7127 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7128 7129 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data); 7130 } 7131 7132 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev) 7133 { 7134 u32 data, wgp_bitmask; 7135 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG); 7136 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG); 7137 7138 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7139 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7140 7141 wgp_bitmask = 7142 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1); 7143 7144 return (~data) & wgp_bitmask; 7145 } 7146 7147 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev) 7148 { 7149 u32 wgp_idx, wgp_active_bitmap; 7150 u32 cu_bitmap_per_wgp, cu_active_bitmap; 7151 7152 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev); 7153 cu_active_bitmap = 0; 7154 7155 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) { 7156 /* if there is one WGP enabled, it means 2 CUs will be enabled */ 7157 cu_bitmap_per_wgp = 3 << (2 * wgp_idx); 7158 if (wgp_active_bitmap & (1 << wgp_idx)) 7159 cu_active_bitmap |= cu_bitmap_per_wgp; 7160 } 7161 7162 return cu_active_bitmap; 7163 } 7164 7165 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 7166 struct amdgpu_cu_info *cu_info) 7167 { 7168 int i, j, k, counter, active_cu_number = 0; 7169 u32 mask, bitmap; 7170 unsigned disable_masks[8 * 2]; 7171 7172 if (!adev || !cu_info) 7173 return -EINVAL; 7174 7175 amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2); 7176 7177 mutex_lock(&adev->grbm_idx_mutex); 7178 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7179 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7180 bitmap = i * adev->gfx.config.max_sh_per_se + j; 7181 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1)) 7182 continue; 7183 mask = 1; 7184 counter = 0; 7185 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7186 if (i < 8 && j < 2) 7187 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh( 7188 adev, disable_masks[i * 2 + j]); 7189 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev); 7190 7191 /** 7192 * GFX11 could support more than 4 SEs, while the bitmap 7193 * in cu_info struct is 4x4 and ioctl interface struct 7194 * drm_amdgpu_info_device should keep stable. 7195 * So we use last two columns of bitmap to store cu mask for 7196 * SEs 4 to 7, the layout of the bitmap is as below: 7197 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]} 7198 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]} 7199 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]} 7200 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]} 7201 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]} 7202 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]} 7203 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]} 7204 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]} 7205 */ 7206 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap; 7207 7208 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { 7209 if (bitmap & mask) 7210 counter++; 7211 7212 mask <<= 1; 7213 } 7214 active_cu_number += counter; 7215 } 7216 } 7217 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7218 mutex_unlock(&adev->grbm_idx_mutex); 7219 7220 cu_info->number = active_cu_number; 7221 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7222 7223 return 0; 7224 } 7225 7226 const struct amdgpu_ip_block_version gfx_v11_0_ip_block = 7227 { 7228 .type = AMD_IP_BLOCK_TYPE_GFX, 7229 .major = 11, 7230 .minor = 0, 7231 .rev = 0, 7232 .funcs = &gfx_v11_0_ip_funcs, 7233 }; 7234