xref: /linux-6.15/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision 8c4e4fd6)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 
27 #include <drm/drm_cache.h>
28 
29 #include "amdgpu.h"
30 #include "gmc_v9_0.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "amdgpu_gem.h"
33 
34 #include "hdp/hdp_4_0_offset.h"
35 #include "hdp/hdp_4_0_sh_mask.h"
36 #include "gc/gc_9_0_sh_mask.h"
37 #include "dce/dce_12_0_offset.h"
38 #include "dce/dce_12_0_sh_mask.h"
39 #include "vega10_enum.h"
40 #include "mmhub/mmhub_1_0_offset.h"
41 #include "athub/athub_1_0_sh_mask.h"
42 #include "athub/athub_1_0_offset.h"
43 #include "oss/osssys_4_0_offset.h"
44 
45 #include "soc15.h"
46 #include "soc15d.h"
47 #include "soc15_common.h"
48 #include "umc/umc_6_0_sh_mask.h"
49 
50 #include "gfxhub_v1_0.h"
51 #include "mmhub_v1_0.h"
52 #include "athub_v1_0.h"
53 #include "gfxhub_v1_1.h"
54 #include "mmhub_v9_4.h"
55 #include "umc_v6_1.h"
56 #include "umc_v6_0.h"
57 
58 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
59 
60 #include "amdgpu_ras.h"
61 #include "amdgpu_xgmi.h"
62 
63 /* add these here since we already include dce12 headers and these are for DCN */
64 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
65 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
66 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
67 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
68 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
69 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
70 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
71 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
72 
73 
74 static const char *gfxhub_client_ids[] = {
75 	"CB",
76 	"DB",
77 	"IA",
78 	"WD",
79 	"CPF",
80 	"CPC",
81 	"CPG",
82 	"RLC",
83 	"TCP",
84 	"SQC (inst)",
85 	"SQC (data)",
86 	"SQG",
87 	"PA",
88 };
89 
90 static const char *mmhub_client_ids_raven[][2] = {
91 	[0][0] = "MP1",
92 	[1][0] = "MP0",
93 	[2][0] = "VCN",
94 	[3][0] = "VCNU",
95 	[4][0] = "HDP",
96 	[5][0] = "DCE",
97 	[13][0] = "UTCL2",
98 	[19][0] = "TLS",
99 	[26][0] = "OSS",
100 	[27][0] = "SDMA0",
101 	[0][1] = "MP1",
102 	[1][1] = "MP0",
103 	[2][1] = "VCN",
104 	[3][1] = "VCNU",
105 	[4][1] = "HDP",
106 	[5][1] = "XDP",
107 	[6][1] = "DBGU0",
108 	[7][1] = "DCE",
109 	[8][1] = "DCEDWB0",
110 	[9][1] = "DCEDWB1",
111 	[26][1] = "OSS",
112 	[27][1] = "SDMA0",
113 };
114 
115 static const char *mmhub_client_ids_renoir[][2] = {
116 	[0][0] = "MP1",
117 	[1][0] = "MP0",
118 	[2][0] = "HDP",
119 	[4][0] = "DCEDMC",
120 	[5][0] = "DCEVGA",
121 	[13][0] = "UTCL2",
122 	[19][0] = "TLS",
123 	[26][0] = "OSS",
124 	[27][0] = "SDMA0",
125 	[28][0] = "VCN",
126 	[29][0] = "VCNU",
127 	[30][0] = "JPEG",
128 	[0][1] = "MP1",
129 	[1][1] = "MP0",
130 	[2][1] = "HDP",
131 	[3][1] = "XDP",
132 	[6][1] = "DBGU0",
133 	[7][1] = "DCEDMC",
134 	[8][1] = "DCEVGA",
135 	[9][1] = "DCEDWB",
136 	[26][1] = "OSS",
137 	[27][1] = "SDMA0",
138 	[28][1] = "VCN",
139 	[29][1] = "VCNU",
140 	[30][1] = "JPEG",
141 };
142 
143 static const char *mmhub_client_ids_vega10[][2] = {
144 	[0][0] = "MP0",
145 	[1][0] = "UVD",
146 	[2][0] = "UVDU",
147 	[3][0] = "HDP",
148 	[13][0] = "UTCL2",
149 	[14][0] = "OSS",
150 	[15][0] = "SDMA1",
151 	[32+0][0] = "VCE0",
152 	[32+1][0] = "VCE0U",
153 	[32+2][0] = "XDMA",
154 	[32+3][0] = "DCE",
155 	[32+4][0] = "MP1",
156 	[32+14][0] = "SDMA0",
157 	[0][1] = "MP0",
158 	[1][1] = "UVD",
159 	[2][1] = "UVDU",
160 	[3][1] = "DBGU0",
161 	[4][1] = "HDP",
162 	[5][1] = "XDP",
163 	[14][1] = "OSS",
164 	[15][1] = "SDMA0",
165 	[32+0][1] = "VCE0",
166 	[32+1][1] = "VCE0U",
167 	[32+2][1] = "XDMA",
168 	[32+3][1] = "DCE",
169 	[32+4][1] = "DCEDWB",
170 	[32+5][1] = "MP1",
171 	[32+6][1] = "DBGU1",
172 	[32+14][1] = "SDMA1",
173 };
174 
175 static const char *mmhub_client_ids_vega12[][2] = {
176 	[0][0] = "MP0",
177 	[1][0] = "VCE0",
178 	[2][0] = "VCE0U",
179 	[3][0] = "HDP",
180 	[13][0] = "UTCL2",
181 	[14][0] = "OSS",
182 	[15][0] = "SDMA1",
183 	[32+0][0] = "DCE",
184 	[32+1][0] = "XDMA",
185 	[32+2][0] = "UVD",
186 	[32+3][0] = "UVDU",
187 	[32+4][0] = "MP1",
188 	[32+15][0] = "SDMA0",
189 	[0][1] = "MP0",
190 	[1][1] = "VCE0",
191 	[2][1] = "VCE0U",
192 	[3][1] = "DBGU0",
193 	[4][1] = "HDP",
194 	[5][1] = "XDP",
195 	[14][1] = "OSS",
196 	[15][1] = "SDMA0",
197 	[32+0][1] = "DCE",
198 	[32+1][1] = "DCEDWB",
199 	[32+2][1] = "XDMA",
200 	[32+3][1] = "UVD",
201 	[32+4][1] = "UVDU",
202 	[32+5][1] = "MP1",
203 	[32+6][1] = "DBGU1",
204 	[32+15][1] = "SDMA1",
205 };
206 
207 static const char *mmhub_client_ids_vega20[][2] = {
208 	[0][0] = "XDMA",
209 	[1][0] = "DCE",
210 	[2][0] = "VCE0",
211 	[3][0] = "VCE0U",
212 	[4][0] = "UVD",
213 	[5][0] = "UVD1U",
214 	[13][0] = "OSS",
215 	[14][0] = "HDP",
216 	[15][0] = "SDMA0",
217 	[32+0][0] = "UVD",
218 	[32+1][0] = "UVDU",
219 	[32+2][0] = "MP1",
220 	[32+3][0] = "MP0",
221 	[32+12][0] = "UTCL2",
222 	[32+14][0] = "SDMA1",
223 	[0][1] = "XDMA",
224 	[1][1] = "DCE",
225 	[2][1] = "DCEDWB",
226 	[3][1] = "VCE0",
227 	[4][1] = "VCE0U",
228 	[5][1] = "UVD1",
229 	[6][1] = "UVD1U",
230 	[7][1] = "DBGU0",
231 	[8][1] = "XDP",
232 	[13][1] = "OSS",
233 	[14][1] = "HDP",
234 	[15][1] = "SDMA0",
235 	[32+0][1] = "UVD",
236 	[32+1][1] = "UVDU",
237 	[32+2][1] = "DBGU1",
238 	[32+3][1] = "MP1",
239 	[32+4][1] = "MP0",
240 	[32+14][1] = "SDMA1",
241 };
242 
243 static const char *mmhub_client_ids_arcturus[][2] = {
244 	[2][0] = "MP1",
245 	[3][0] = "MP0",
246 	[10][0] = "UTCL2",
247 	[13][0] = "OSS",
248 	[14][0] = "HDP",
249 	[15][0] = "SDMA0",
250 	[32+15][0] = "SDMA1",
251 	[64+15][0] = "SDMA2",
252 	[96+15][0] = "SDMA3",
253 	[128+15][0] = "SDMA4",
254 	[160+11][0] = "JPEG",
255 	[160+12][0] = "VCN",
256 	[160+13][0] = "VCNU",
257 	[160+15][0] = "SDMA5",
258 	[192+10][0] = "UTCL2",
259 	[192+11][0] = "JPEG1",
260 	[192+12][0] = "VCN1",
261 	[192+13][0] = "VCN1U",
262 	[192+15][0] = "SDMA6",
263 	[224+15][0] = "SDMA7",
264 	[0][1] = "DBGU1",
265 	[1][1] = "XDP",
266 	[2][1] = "MP1",
267 	[3][1] = "MP0",
268 	[13][1] = "OSS",
269 	[14][1] = "HDP",
270 	[15][1] = "SDMA0",
271 	[32+15][1] = "SDMA1",
272 	[64+15][1] = "SDMA2",
273 	[96+15][1] = "SDMA3",
274 	[128+15][1] = "SDMA4",
275 	[160+11][1] = "JPEG",
276 	[160+12][1] = "VCN",
277 	[160+13][1] = "VCNU",
278 	[160+15][1] = "SDMA5",
279 	[192+11][1] = "JPEG1",
280 	[192+12][1] = "VCN1",
281 	[192+13][1] = "VCN1U",
282 	[192+15][1] = "SDMA6",
283 	[224+15][1] = "SDMA7",
284 };
285 
286 static const u32 golden_settings_vega10_hdp[] =
287 {
288 	0xf64, 0x0fffffff, 0x00000000,
289 	0xf65, 0x0fffffff, 0x00000000,
290 	0xf66, 0x0fffffff, 0x00000000,
291 	0xf67, 0x0fffffff, 0x00000000,
292 	0xf68, 0x0fffffff, 0x00000000,
293 	0xf6a, 0x0fffffff, 0x00000000,
294 	0xf6b, 0x0fffffff, 0x00000000,
295 	0xf6c, 0x0fffffff, 0x00000000,
296 	0xf6d, 0x0fffffff, 0x00000000,
297 	0xf6e, 0x0fffffff, 0x00000000,
298 };
299 
300 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
301 {
302 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
303 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
304 };
305 
306 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
307 {
308 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
309 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
310 };
311 
312 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
313 	(0x000143c0 + 0x00000000),
314 	(0x000143c0 + 0x00000800),
315 	(0x000143c0 + 0x00001000),
316 	(0x000143c0 + 0x00001800),
317 	(0x000543c0 + 0x00000000),
318 	(0x000543c0 + 0x00000800),
319 	(0x000543c0 + 0x00001000),
320 	(0x000543c0 + 0x00001800),
321 	(0x000943c0 + 0x00000000),
322 	(0x000943c0 + 0x00000800),
323 	(0x000943c0 + 0x00001000),
324 	(0x000943c0 + 0x00001800),
325 	(0x000d43c0 + 0x00000000),
326 	(0x000d43c0 + 0x00000800),
327 	(0x000d43c0 + 0x00001000),
328 	(0x000d43c0 + 0x00001800),
329 	(0x001143c0 + 0x00000000),
330 	(0x001143c0 + 0x00000800),
331 	(0x001143c0 + 0x00001000),
332 	(0x001143c0 + 0x00001800),
333 	(0x001543c0 + 0x00000000),
334 	(0x001543c0 + 0x00000800),
335 	(0x001543c0 + 0x00001000),
336 	(0x001543c0 + 0x00001800),
337 	(0x001943c0 + 0x00000000),
338 	(0x001943c0 + 0x00000800),
339 	(0x001943c0 + 0x00001000),
340 	(0x001943c0 + 0x00001800),
341 	(0x001d43c0 + 0x00000000),
342 	(0x001d43c0 + 0x00000800),
343 	(0x001d43c0 + 0x00001000),
344 	(0x001d43c0 + 0x00001800),
345 };
346 
347 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
348 	(0x000143e0 + 0x00000000),
349 	(0x000143e0 + 0x00000800),
350 	(0x000143e0 + 0x00001000),
351 	(0x000143e0 + 0x00001800),
352 	(0x000543e0 + 0x00000000),
353 	(0x000543e0 + 0x00000800),
354 	(0x000543e0 + 0x00001000),
355 	(0x000543e0 + 0x00001800),
356 	(0x000943e0 + 0x00000000),
357 	(0x000943e0 + 0x00000800),
358 	(0x000943e0 + 0x00001000),
359 	(0x000943e0 + 0x00001800),
360 	(0x000d43e0 + 0x00000000),
361 	(0x000d43e0 + 0x00000800),
362 	(0x000d43e0 + 0x00001000),
363 	(0x000d43e0 + 0x00001800),
364 	(0x001143e0 + 0x00000000),
365 	(0x001143e0 + 0x00000800),
366 	(0x001143e0 + 0x00001000),
367 	(0x001143e0 + 0x00001800),
368 	(0x001543e0 + 0x00000000),
369 	(0x001543e0 + 0x00000800),
370 	(0x001543e0 + 0x00001000),
371 	(0x001543e0 + 0x00001800),
372 	(0x001943e0 + 0x00000000),
373 	(0x001943e0 + 0x00000800),
374 	(0x001943e0 + 0x00001000),
375 	(0x001943e0 + 0x00001800),
376 	(0x001d43e0 + 0x00000000),
377 	(0x001d43e0 + 0x00000800),
378 	(0x001d43e0 + 0x00001000),
379 	(0x001d43e0 + 0x00001800),
380 };
381 
382 static const uint32_t ecc_umc_mcumc_status_addrs[] = {
383 	(0x000143c2 + 0x00000000),
384 	(0x000143c2 + 0x00000800),
385 	(0x000143c2 + 0x00001000),
386 	(0x000143c2 + 0x00001800),
387 	(0x000543c2 + 0x00000000),
388 	(0x000543c2 + 0x00000800),
389 	(0x000543c2 + 0x00001000),
390 	(0x000543c2 + 0x00001800),
391 	(0x000943c2 + 0x00000000),
392 	(0x000943c2 + 0x00000800),
393 	(0x000943c2 + 0x00001000),
394 	(0x000943c2 + 0x00001800),
395 	(0x000d43c2 + 0x00000000),
396 	(0x000d43c2 + 0x00000800),
397 	(0x000d43c2 + 0x00001000),
398 	(0x000d43c2 + 0x00001800),
399 	(0x001143c2 + 0x00000000),
400 	(0x001143c2 + 0x00000800),
401 	(0x001143c2 + 0x00001000),
402 	(0x001143c2 + 0x00001800),
403 	(0x001543c2 + 0x00000000),
404 	(0x001543c2 + 0x00000800),
405 	(0x001543c2 + 0x00001000),
406 	(0x001543c2 + 0x00001800),
407 	(0x001943c2 + 0x00000000),
408 	(0x001943c2 + 0x00000800),
409 	(0x001943c2 + 0x00001000),
410 	(0x001943c2 + 0x00001800),
411 	(0x001d43c2 + 0x00000000),
412 	(0x001d43c2 + 0x00000800),
413 	(0x001d43c2 + 0x00001000),
414 	(0x001d43c2 + 0x00001800),
415 };
416 
417 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
418 		struct amdgpu_irq_src *src,
419 		unsigned type,
420 		enum amdgpu_interrupt_state state)
421 {
422 	u32 bits, i, tmp, reg;
423 
424 	/* Devices newer then VEGA10/12 shall have these programming
425 	     sequences performed by PSP BL */
426 	if (adev->asic_type >= CHIP_VEGA20)
427 		return 0;
428 
429 	bits = 0x7f;
430 
431 	switch (state) {
432 	case AMDGPU_IRQ_STATE_DISABLE:
433 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
434 			reg = ecc_umc_mcumc_ctrl_addrs[i];
435 			tmp = RREG32(reg);
436 			tmp &= ~bits;
437 			WREG32(reg, tmp);
438 		}
439 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
440 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
441 			tmp = RREG32(reg);
442 			tmp &= ~bits;
443 			WREG32(reg, tmp);
444 		}
445 		break;
446 	case AMDGPU_IRQ_STATE_ENABLE:
447 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
448 			reg = ecc_umc_mcumc_ctrl_addrs[i];
449 			tmp = RREG32(reg);
450 			tmp |= bits;
451 			WREG32(reg, tmp);
452 		}
453 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
454 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
455 			tmp = RREG32(reg);
456 			tmp |= bits;
457 			WREG32(reg, tmp);
458 		}
459 		break;
460 	default:
461 		break;
462 	}
463 
464 	return 0;
465 }
466 
467 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
468 					struct amdgpu_irq_src *src,
469 					unsigned type,
470 					enum amdgpu_interrupt_state state)
471 {
472 	struct amdgpu_vmhub *hub;
473 	u32 tmp, reg, bits, i, j;
474 
475 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
480 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
481 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
482 
483 	switch (state) {
484 	case AMDGPU_IRQ_STATE_DISABLE:
485 		for (j = 0; j < adev->num_vmhubs; j++) {
486 			hub = &adev->vmhub[j];
487 			for (i = 0; i < 16; i++) {
488 				reg = hub->vm_context0_cntl + i;
489 				tmp = RREG32(reg);
490 				tmp &= ~bits;
491 				WREG32(reg, tmp);
492 			}
493 		}
494 		break;
495 	case AMDGPU_IRQ_STATE_ENABLE:
496 		for (j = 0; j < adev->num_vmhubs; j++) {
497 			hub = &adev->vmhub[j];
498 			for (i = 0; i < 16; i++) {
499 				reg = hub->vm_context0_cntl + i;
500 				tmp = RREG32(reg);
501 				tmp |= bits;
502 				WREG32(reg, tmp);
503 			}
504 		}
505 	default:
506 		break;
507 	}
508 
509 	return 0;
510 }
511 
512 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
513 				struct amdgpu_irq_src *source,
514 				struct amdgpu_iv_entry *entry)
515 {
516 	struct amdgpu_vmhub *hub;
517 	bool retry_fault = !!(entry->src_data[1] & 0x80);
518 	uint32_t status = 0, cid = 0, rw = 0;
519 	u64 addr;
520 	char hub_name[10];
521 	const char *mmhub_cid;
522 
523 	addr = (u64)entry->src_data[0] << 12;
524 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
525 
526 	if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
527 						    entry->timestamp))
528 		return 1; /* This also prevents sending it to KFD */
529 
530 	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
531 		snprintf(hub_name, sizeof(hub_name), "mmhub0");
532 		hub = &adev->vmhub[AMDGPU_MMHUB_0];
533 	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
534 		snprintf(hub_name, sizeof(hub_name), "mmhub1");
535 		hub = &adev->vmhub[AMDGPU_MMHUB_1];
536 	} else {
537 		snprintf(hub_name, sizeof(hub_name), "gfxhub0");
538 		hub = &adev->vmhub[AMDGPU_GFXHUB_0];
539 	}
540 
541 	/* If it's the first fault for this address, process it normally */
542 	if (retry_fault && !in_interrupt() &&
543 	    amdgpu_vm_handle_fault(adev, entry->pasid, addr))
544 		return 1; /* This also prevents sending it to KFD */
545 
546 	if (!amdgpu_sriov_vf(adev)) {
547 		/*
548 		 * Issue a dummy read to wait for the status register to
549 		 * be updated to avoid reading an incorrect value due to
550 		 * the new fast GRBM interface.
551 		 */
552 		if (entry->vmid_src == AMDGPU_GFXHUB_0)
553 			RREG32(hub->vm_l2_pro_fault_status);
554 
555 		status = RREG32(hub->vm_l2_pro_fault_status);
556 		cid = REG_GET_FIELD(status,
557 				    VM_L2_PROTECTION_FAULT_STATUS, CID);
558 		rw = REG_GET_FIELD(status,
559 				   VM_L2_PROTECTION_FAULT_STATUS, RW);
560 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
561 	}
562 
563 	if (printk_ratelimit()) {
564 		struct amdgpu_task_info task_info;
565 
566 		memset(&task_info, 0, sizeof(struct amdgpu_task_info));
567 		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
568 
569 		dev_err(adev->dev,
570 			"[%s] %s page fault (src_id:%u ring:%u vmid:%u "
571 			"pasid:%u, for process %s pid %d thread %s pid %d)\n",
572 			hub_name, retry_fault ? "retry" : "no-retry",
573 			entry->src_id, entry->ring_id, entry->vmid,
574 			entry->pasid, task_info.process_name, task_info.tgid,
575 			task_info.task_name, task_info.pid);
576 		dev_err(adev->dev, "  in page starting at address 0x%016llx from client %d\n",
577 			addr, entry->client_id);
578 		if (!amdgpu_sriov_vf(adev)) {
579 			dev_err(adev->dev,
580 				"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
581 				status);
582 			if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
583 				dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
584 					cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid],
585 					cid);
586 			} else {
587 				switch (adev->asic_type) {
588 				case CHIP_VEGA10:
589 					mmhub_cid = mmhub_client_ids_vega10[cid][rw];
590 					break;
591 				case CHIP_VEGA12:
592 					mmhub_cid = mmhub_client_ids_vega12[cid][rw];
593 					break;
594 				case CHIP_VEGA20:
595 					mmhub_cid = mmhub_client_ids_vega20[cid][rw];
596 					break;
597 				case CHIP_ARCTURUS:
598 					mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
599 					break;
600 				case CHIP_RAVEN:
601 					mmhub_cid = mmhub_client_ids_raven[cid][rw];
602 					break;
603 				case CHIP_RENOIR:
604 					mmhub_cid = mmhub_client_ids_renoir[cid][rw];
605 					break;
606 				default:
607 					mmhub_cid = NULL;
608 					break;
609 				}
610 				dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
611 					mmhub_cid ? mmhub_cid : "unknown", cid);
612 			}
613 			dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
614 				REG_GET_FIELD(status,
615 				VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
616 			dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
617 				REG_GET_FIELD(status,
618 				VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
619 			dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
620 				REG_GET_FIELD(status,
621 				VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
622 			dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
623 				REG_GET_FIELD(status,
624 				VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
625 			dev_err(adev->dev, "\t RW: 0x%x\n", rw);
626 		}
627 	}
628 
629 	return 0;
630 }
631 
632 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
633 	.set = gmc_v9_0_vm_fault_interrupt_state,
634 	.process = gmc_v9_0_process_interrupt,
635 };
636 
637 
638 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
639 	.set = gmc_v9_0_ecc_interrupt_state,
640 	.process = amdgpu_umc_process_ecc_irq,
641 };
642 
643 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
644 {
645 	adev->gmc.vm_fault.num_types = 1;
646 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
647 
648 	if (!amdgpu_sriov_vf(adev)) {
649 		adev->gmc.ecc_irq.num_types = 1;
650 		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
651 	}
652 }
653 
654 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
655 					uint32_t flush_type)
656 {
657 	u32 req = 0;
658 
659 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
660 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
661 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
662 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
663 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
664 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
665 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
666 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
667 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
668 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
669 
670 	return req;
671 }
672 
673 /**
674  * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
675  *
676  * @adev: amdgpu_device pointer
677  * @vmhub: vmhub type
678  *
679  */
680 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
681 				       uint32_t vmhub)
682 {
683 	return ((vmhub == AMDGPU_MMHUB_0 ||
684 		 vmhub == AMDGPU_MMHUB_1) &&
685 		(!amdgpu_sriov_vf(adev)) &&
686 		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
687 		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
688 }
689 
690 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
691 					uint8_t vmid, uint16_t *p_pasid)
692 {
693 	uint32_t value;
694 
695 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
696 		     + vmid);
697 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
698 
699 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
700 }
701 
702 /*
703  * GART
704  * VMID 0 is the physical GPU addresses as used by the kernel.
705  * VMIDs 1-15 are used for userspace clients and are handled
706  * by the amdgpu vm/hsa code.
707  */
708 
709 /**
710  * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
711  *
712  * @adev: amdgpu_device pointer
713  * @vmid: vm instance to flush
714  * @flush_type: the flush type
715  *
716  * Flush the TLB for the requested page table using certain type.
717  */
718 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
719 					uint32_t vmhub, uint32_t flush_type)
720 {
721 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
722 	const unsigned eng = 17;
723 	u32 j, inv_req, inv_req2, tmp;
724 	struct amdgpu_vmhub *hub;
725 
726 	BUG_ON(vmhub >= adev->num_vmhubs);
727 
728 	hub = &adev->vmhub[vmhub];
729 	if (adev->gmc.xgmi.num_physical_nodes &&
730 	    adev->asic_type == CHIP_VEGA20) {
731 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
732 		 * heavy-weight TLB flush (type 2), which flushes
733 		 * both. Due to a race condition with concurrent
734 		 * memory accesses using the same TLB cache line, we
735 		 * still need a second TLB flush after this.
736 		 */
737 		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
738 		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
739 	} else {
740 		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
741 		inv_req2 = 0;
742 	}
743 
744 	/* This is necessary for a HW workaround under SRIOV as well
745 	 * as GFXOFF under bare metal
746 	 */
747 	if (adev->gfx.kiq.ring.sched.ready &&
748 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
749 	    down_read_trylock(&adev->reset_sem)) {
750 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
751 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
752 
753 		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
754 						   1 << vmid);
755 		up_read(&adev->reset_sem);
756 		return;
757 	}
758 
759 	spin_lock(&adev->gmc.invalidate_lock);
760 
761 	/*
762 	 * It may lose gpuvm invalidate acknowldege state across power-gating
763 	 * off cycle, add semaphore acquire before invalidation and semaphore
764 	 * release after invalidation to avoid entering power gated state
765 	 * to WA the Issue
766 	 */
767 
768 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
769 	if (use_semaphore) {
770 		for (j = 0; j < adev->usec_timeout; j++) {
771 			/* a read return value of 1 means semaphore acuqire */
772 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem +
773 					    hub->eng_distance * eng);
774 			if (tmp & 0x1)
775 				break;
776 			udelay(1);
777 		}
778 
779 		if (j >= adev->usec_timeout)
780 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
781 	}
782 
783 	do {
784 		WREG32_NO_KIQ(hub->vm_inv_eng0_req +
785 			      hub->eng_distance * eng, inv_req);
786 
787 		/*
788 		 * Issue a dummy read to wait for the ACK register to
789 		 * be cleared to avoid a false ACK due to the new fast
790 		 * GRBM interface.
791 		 */
792 		if (vmhub == AMDGPU_GFXHUB_0)
793 			RREG32_NO_KIQ(hub->vm_inv_eng0_req +
794 				      hub->eng_distance * eng);
795 
796 		for (j = 0; j < adev->usec_timeout; j++) {
797 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack +
798 					    hub->eng_distance * eng);
799 			if (tmp & (1 << vmid))
800 				break;
801 			udelay(1);
802 		}
803 
804 		inv_req = inv_req2;
805 		inv_req2 = 0;
806 	} while (inv_req);
807 
808 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
809 	if (use_semaphore)
810 		/*
811 		 * add semaphore release after invalidation,
812 		 * write with 0 means semaphore release
813 		 */
814 		WREG32_NO_KIQ(hub->vm_inv_eng0_sem +
815 			      hub->eng_distance * eng, 0);
816 
817 	spin_unlock(&adev->gmc.invalidate_lock);
818 
819 	if (j < adev->usec_timeout)
820 		return;
821 
822 	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
823 }
824 
825 /**
826  * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
827  *
828  * @adev: amdgpu_device pointer
829  * @pasid: pasid to be flush
830  *
831  * Flush the TLB for the requested pasid.
832  */
833 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
834 					uint16_t pasid, uint32_t flush_type,
835 					bool all_hub)
836 {
837 	int vmid, i;
838 	signed long r;
839 	uint32_t seq;
840 	uint16_t queried_pasid;
841 	bool ret;
842 	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
843 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
844 
845 	if (amdgpu_in_reset(adev))
846 		return -EIO;
847 
848 	if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) {
849 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
850 		 * heavy-weight TLB flush (type 2), which flushes
851 		 * both. Due to a race condition with concurrent
852 		 * memory accesses using the same TLB cache line, we
853 		 * still need a second TLB flush after this.
854 		 */
855 		bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
856 				       adev->asic_type == CHIP_VEGA20);
857 		/* 2 dwords flush + 8 dwords fence */
858 		unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
859 
860 		if (vega20_xgmi_wa)
861 			ndw += kiq->pmf->invalidate_tlbs_size;
862 
863 		spin_lock(&adev->gfx.kiq.ring_lock);
864 		/* 2 dwords flush + 8 dwords fence */
865 		amdgpu_ring_alloc(ring, ndw);
866 		if (vega20_xgmi_wa)
867 			kiq->pmf->kiq_invalidate_tlbs(ring,
868 						      pasid, 2, all_hub);
869 		kiq->pmf->kiq_invalidate_tlbs(ring,
870 					pasid, flush_type, all_hub);
871 		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
872 		if (r) {
873 			amdgpu_ring_undo(ring);
874 			spin_unlock(&adev->gfx.kiq.ring_lock);
875 			up_read(&adev->reset_sem);
876 			return -ETIME;
877 		}
878 
879 		amdgpu_ring_commit(ring);
880 		spin_unlock(&adev->gfx.kiq.ring_lock);
881 		r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
882 		if (r < 1) {
883 			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
884 			up_read(&adev->reset_sem);
885 			return -ETIME;
886 		}
887 		up_read(&adev->reset_sem);
888 		return 0;
889 	}
890 
891 	for (vmid = 1; vmid < 16; vmid++) {
892 
893 		ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
894 				&queried_pasid);
895 		if (ret && queried_pasid == pasid) {
896 			if (all_hub) {
897 				for (i = 0; i < adev->num_vmhubs; i++)
898 					gmc_v9_0_flush_gpu_tlb(adev, vmid,
899 							i, flush_type);
900 			} else {
901 				gmc_v9_0_flush_gpu_tlb(adev, vmid,
902 						AMDGPU_GFXHUB_0, flush_type);
903 			}
904 			break;
905 		}
906 	}
907 
908 	return 0;
909 
910 }
911 
912 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
913 					    unsigned vmid, uint64_t pd_addr)
914 {
915 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
916 	struct amdgpu_device *adev = ring->adev;
917 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
918 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
919 	unsigned eng = ring->vm_inv_eng;
920 
921 	/*
922 	 * It may lose gpuvm invalidate acknowldege state across power-gating
923 	 * off cycle, add semaphore acquire before invalidation and semaphore
924 	 * release after invalidation to avoid entering power gated state
925 	 * to WA the Issue
926 	 */
927 
928 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
929 	if (use_semaphore)
930 		/* a read return value of 1 means semaphore acuqire */
931 		amdgpu_ring_emit_reg_wait(ring,
932 					  hub->vm_inv_eng0_sem +
933 					  hub->eng_distance * eng, 0x1, 0x1);
934 
935 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
936 			      (hub->ctx_addr_distance * vmid),
937 			      lower_32_bits(pd_addr));
938 
939 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
940 			      (hub->ctx_addr_distance * vmid),
941 			      upper_32_bits(pd_addr));
942 
943 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
944 					    hub->eng_distance * eng,
945 					    hub->vm_inv_eng0_ack +
946 					    hub->eng_distance * eng,
947 					    req, 1 << vmid);
948 
949 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
950 	if (use_semaphore)
951 		/*
952 		 * add semaphore release after invalidation,
953 		 * write with 0 means semaphore release
954 		 */
955 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
956 				      hub->eng_distance * eng, 0);
957 
958 	return pd_addr;
959 }
960 
961 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
962 					unsigned pasid)
963 {
964 	struct amdgpu_device *adev = ring->adev;
965 	uint32_t reg;
966 
967 	/* Do nothing because there's no lut register for mmhub1. */
968 	if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
969 		return;
970 
971 	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
972 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
973 	else
974 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
975 
976 	amdgpu_ring_emit_wreg(ring, reg, pasid);
977 }
978 
979 /*
980  * PTE format on VEGA 10:
981  * 63:59 reserved
982  * 58:57 mtype
983  * 56 F
984  * 55 L
985  * 54 P
986  * 53 SW
987  * 52 T
988  * 50:48 reserved
989  * 47:12 4k physical page base address
990  * 11:7 fragment
991  * 6 write
992  * 5 read
993  * 4 exe
994  * 3 Z
995  * 2 snooped
996  * 1 system
997  * 0 valid
998  *
999  * PDE format on VEGA 10:
1000  * 63:59 block fragment size
1001  * 58:55 reserved
1002  * 54 P
1003  * 53:48 reserved
1004  * 47:6 physical base address of PD or PTE
1005  * 5:3 reserved
1006  * 2 C
1007  * 1 system
1008  * 0 valid
1009  */
1010 
1011 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1012 
1013 {
1014 	switch (flags) {
1015 	case AMDGPU_VM_MTYPE_DEFAULT:
1016 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1017 	case AMDGPU_VM_MTYPE_NC:
1018 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1019 	case AMDGPU_VM_MTYPE_WC:
1020 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1021 	case AMDGPU_VM_MTYPE_RW:
1022 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1023 	case AMDGPU_VM_MTYPE_CC:
1024 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1025 	case AMDGPU_VM_MTYPE_UC:
1026 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1027 	default:
1028 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1029 	}
1030 }
1031 
1032 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1033 				uint64_t *addr, uint64_t *flags)
1034 {
1035 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1036 		*addr = adev->vm_manager.vram_base_offset + *addr -
1037 			adev->gmc.vram_start;
1038 	BUG_ON(*addr & 0xFFFF00000000003FULL);
1039 
1040 	if (!adev->gmc.translate_further)
1041 		return;
1042 
1043 	if (level == AMDGPU_VM_PDB1) {
1044 		/* Set the block fragment size */
1045 		if (!(*flags & AMDGPU_PDE_PTE))
1046 			*flags |= AMDGPU_PDE_BFS(0x9);
1047 
1048 	} else if (level == AMDGPU_VM_PDB0) {
1049 		if (*flags & AMDGPU_PDE_PTE)
1050 			*flags &= ~AMDGPU_PDE_PTE;
1051 		else
1052 			*flags |= AMDGPU_PTE_TF;
1053 	}
1054 }
1055 
1056 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1057 				struct amdgpu_bo_va_mapping *mapping,
1058 				uint64_t *flags)
1059 {
1060 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1061 	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1062 
1063 	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1064 	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1065 
1066 	if (mapping->flags & AMDGPU_PTE_PRT) {
1067 		*flags |= AMDGPU_PTE_PRT;
1068 		*flags &= ~AMDGPU_PTE_VALID;
1069 	}
1070 
1071 	if (adev->asic_type == CHIP_ARCTURUS &&
1072 	    !(*flags & AMDGPU_PTE_SYSTEM) &&
1073 	    mapping->bo_va->is_xgmi)
1074 		*flags |= AMDGPU_PTE_SNOOPED;
1075 }
1076 
1077 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1078 {
1079 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1080 	unsigned size;
1081 
1082 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1083 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1084 	} else {
1085 		u32 viewport;
1086 
1087 		switch (adev->asic_type) {
1088 		case CHIP_RAVEN:
1089 		case CHIP_RENOIR:
1090 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1091 			size = (REG_GET_FIELD(viewport,
1092 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1093 				REG_GET_FIELD(viewport,
1094 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1095 				4);
1096 			break;
1097 		case CHIP_VEGA10:
1098 		case CHIP_VEGA12:
1099 		case CHIP_VEGA20:
1100 		default:
1101 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1102 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1103 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1104 				4);
1105 			break;
1106 		}
1107 	}
1108 
1109 	return size;
1110 }
1111 
1112 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1113 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1114 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1115 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1116 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1117 	.map_mtype = gmc_v9_0_map_mtype,
1118 	.get_vm_pde = gmc_v9_0_get_vm_pde,
1119 	.get_vm_pte = gmc_v9_0_get_vm_pte,
1120 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1121 };
1122 
1123 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1124 {
1125 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1126 }
1127 
1128 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1129 {
1130 	switch (adev->asic_type) {
1131 	case CHIP_VEGA10:
1132 		adev->umc.funcs = &umc_v6_0_funcs;
1133 		break;
1134 	case CHIP_VEGA20:
1135 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1136 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1137 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1138 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1139 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1140 		adev->umc.funcs = &umc_v6_1_funcs;
1141 		break;
1142 	case CHIP_ARCTURUS:
1143 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1144 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1145 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1146 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1147 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1148 		adev->umc.funcs = &umc_v6_1_funcs;
1149 		break;
1150 	default:
1151 		break;
1152 	}
1153 }
1154 
1155 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1156 {
1157 	switch (adev->asic_type) {
1158 	case CHIP_ARCTURUS:
1159 		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1160 		break;
1161 	default:
1162 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1163 		break;
1164 	}
1165 }
1166 
1167 static int gmc_v9_0_early_init(void *handle)
1168 {
1169 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1170 
1171 	gmc_v9_0_set_gmc_funcs(adev);
1172 	gmc_v9_0_set_irq_funcs(adev);
1173 	gmc_v9_0_set_umc_funcs(adev);
1174 	gmc_v9_0_set_mmhub_funcs(adev);
1175 
1176 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1177 	adev->gmc.shared_aperture_end =
1178 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1179 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1180 	adev->gmc.private_aperture_end =
1181 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1182 
1183 	return 0;
1184 }
1185 
1186 static int gmc_v9_0_late_init(void *handle)
1187 {
1188 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1189 	int r;
1190 
1191 	amdgpu_bo_late_init(adev);
1192 
1193 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1194 	if (r)
1195 		return r;
1196 	/* Check if ecc is available */
1197 	if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
1198 		r = amdgpu_atomfirmware_mem_ecc_supported(adev);
1199 		if (!r) {
1200 			DRM_INFO("ECC is not present.\n");
1201 			if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1202 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1203 		} else
1204 			DRM_INFO("ECC is active.\n");
1205 
1206 		r = amdgpu_atomfirmware_sram_ecc_supported(adev);
1207 		if (!r)
1208 			DRM_INFO("SRAM ECC is not present.\n");
1209 		else
1210 			DRM_INFO("SRAM ECC is active.\n");
1211 	}
1212 
1213 	if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
1214 		adev->mmhub.funcs->reset_ras_error_count(adev);
1215 
1216 	r = amdgpu_gmc_ras_late_init(adev);
1217 	if (r)
1218 		return r;
1219 
1220 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1221 }
1222 
1223 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1224 					struct amdgpu_gmc *mc)
1225 {
1226 	u64 base = 0;
1227 
1228 	if (!amdgpu_sriov_vf(adev))
1229 		base = adev->mmhub.funcs->get_fb_location(adev);
1230 
1231 	/* add the xgmi offset of the physical node */
1232 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1233 	amdgpu_gmc_vram_location(adev, mc, base);
1234 	amdgpu_gmc_gart_location(adev, mc);
1235 	amdgpu_gmc_agp_location(adev, mc);
1236 	/* base offset of vram pages */
1237 	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
1238 
1239 	/* XXX: add the xgmi offset of the physical node? */
1240 	adev->vm_manager.vram_base_offset +=
1241 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1242 }
1243 
1244 /**
1245  * gmc_v9_0_mc_init - initialize the memory controller driver params
1246  *
1247  * @adev: amdgpu_device pointer
1248  *
1249  * Look up the amount of vram, vram width, and decide how to place
1250  * vram and gart within the GPU's physical address space.
1251  * Returns 0 for success.
1252  */
1253 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1254 {
1255 	int r;
1256 
1257 	/* size in MB on si */
1258 	adev->gmc.mc_vram_size =
1259 		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1260 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1261 
1262 	if (!(adev->flags & AMD_IS_APU)) {
1263 		r = amdgpu_device_resize_fb_bar(adev);
1264 		if (r)
1265 			return r;
1266 	}
1267 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1268 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1269 
1270 #ifdef CONFIG_X86_64
1271 	if (adev->flags & AMD_IS_APU) {
1272 		adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev);
1273 		adev->gmc.aper_size = adev->gmc.real_vram_size;
1274 	}
1275 #endif
1276 	/* In case the PCI BAR is larger than the actual amount of vram */
1277 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1278 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1279 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
1280 
1281 	/* set the gart size */
1282 	if (amdgpu_gart_size == -1) {
1283 		switch (adev->asic_type) {
1284 		case CHIP_VEGA10:  /* all engines support GPUVM */
1285 		case CHIP_VEGA12:  /* all engines support GPUVM */
1286 		case CHIP_VEGA20:
1287 		case CHIP_ARCTURUS:
1288 		default:
1289 			adev->gmc.gart_size = 512ULL << 20;
1290 			break;
1291 		case CHIP_RAVEN:   /* DCE SG support */
1292 		case CHIP_RENOIR:
1293 			adev->gmc.gart_size = 1024ULL << 20;
1294 			break;
1295 		}
1296 	} else {
1297 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1298 	}
1299 
1300 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1301 
1302 	return 0;
1303 }
1304 
1305 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1306 {
1307 	int r;
1308 
1309 	if (adev->gart.bo) {
1310 		WARN(1, "VEGA10 PCIE GART already initialized\n");
1311 		return 0;
1312 	}
1313 	/* Initialize common gart structure */
1314 	r = amdgpu_gart_init(adev);
1315 	if (r)
1316 		return r;
1317 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1318 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1319 				 AMDGPU_PTE_EXECUTABLE;
1320 	return amdgpu_gart_table_vram_alloc(adev);
1321 }
1322 
1323 /**
1324  * gmc_v9_0_save_registers - saves regs
1325  *
1326  * @adev: amdgpu_device pointer
1327  *
1328  * This saves potential register values that should be
1329  * restored upon resume
1330  */
1331 static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1332 {
1333 	if (adev->asic_type == CHIP_RAVEN)
1334 		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1335 }
1336 
1337 static int gmc_v9_0_sw_init(void *handle)
1338 {
1339 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
1340 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1341 
1342 	gfxhub_v1_0_init(adev);
1343 
1344 	adev->mmhub.funcs->init(adev);
1345 
1346 	spin_lock_init(&adev->gmc.invalidate_lock);
1347 
1348 	r = amdgpu_atomfirmware_get_vram_info(adev,
1349 		&vram_width, &vram_type, &vram_vendor);
1350 	if (amdgpu_sriov_vf(adev))
1351 		/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1352 		 * and DF related registers is not readable, seems hardcord is the
1353 		 * only way to set the correct vram_width
1354 		 */
1355 		adev->gmc.vram_width = 2048;
1356 	else if (amdgpu_emu_mode != 1)
1357 		adev->gmc.vram_width = vram_width;
1358 
1359 	if (!adev->gmc.vram_width) {
1360 		int chansize, numchan;
1361 
1362 		/* hbm memory channel size */
1363 		if (adev->flags & AMD_IS_APU)
1364 			chansize = 64;
1365 		else
1366 			chansize = 128;
1367 
1368 		numchan = adev->df.funcs->get_hbm_channel_number(adev);
1369 		adev->gmc.vram_width = numchan * chansize;
1370 	}
1371 
1372 	adev->gmc.vram_type = vram_type;
1373 	adev->gmc.vram_vendor = vram_vendor;
1374 	switch (adev->asic_type) {
1375 	case CHIP_RAVEN:
1376 		adev->num_vmhubs = 2;
1377 
1378 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1379 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1380 		} else {
1381 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
1382 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1383 			adev->gmc.translate_further =
1384 				adev->vm_manager.num_level > 1;
1385 		}
1386 		break;
1387 	case CHIP_VEGA10:
1388 	case CHIP_VEGA12:
1389 	case CHIP_VEGA20:
1390 	case CHIP_RENOIR:
1391 		adev->num_vmhubs = 2;
1392 
1393 
1394 		/*
1395 		 * To fulfill 4-level page support,
1396 		 * vm size is 256TB (48bit), maximum size of Vega10,
1397 		 * block size 512 (9bit)
1398 		 */
1399 		/* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
1400 		if (amdgpu_sriov_vf(adev))
1401 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1402 		else
1403 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1404 		break;
1405 	case CHIP_ARCTURUS:
1406 		adev->num_vmhubs = 3;
1407 
1408 		/* Keep the vm size same with Vega20 */
1409 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1410 		break;
1411 	default:
1412 		break;
1413 	}
1414 
1415 	/* This interrupt is VMC page fault.*/
1416 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1417 				&adev->gmc.vm_fault);
1418 	if (r)
1419 		return r;
1420 
1421 	if (adev->asic_type == CHIP_ARCTURUS) {
1422 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1423 					&adev->gmc.vm_fault);
1424 		if (r)
1425 			return r;
1426 	}
1427 
1428 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1429 				&adev->gmc.vm_fault);
1430 
1431 	if (r)
1432 		return r;
1433 
1434 	if (!amdgpu_sriov_vf(adev)) {
1435 		/* interrupt sent to DF. */
1436 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1437 				      &adev->gmc.ecc_irq);
1438 		if (r)
1439 			return r;
1440 	}
1441 
1442 	/* Set the internal MC address mask
1443 	 * This is the max address of the GPU's
1444 	 * internal address space.
1445 	 */
1446 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
1447 
1448 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
1449 	if (r) {
1450 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
1451 		return r;
1452 	}
1453 	adev->need_swiotlb = drm_need_swiotlb(44);
1454 
1455 	if (adev->gmc.xgmi.supported) {
1456 		r = gfxhub_v1_1_get_xgmi_info(adev);
1457 		if (r)
1458 			return r;
1459 	}
1460 
1461 	r = gmc_v9_0_mc_init(adev);
1462 	if (r)
1463 		return r;
1464 
1465 	amdgpu_gmc_get_vbios_allocations(adev);
1466 
1467 	/* Memory manager */
1468 	r = amdgpu_bo_init(adev);
1469 	if (r)
1470 		return r;
1471 
1472 	r = gmc_v9_0_gart_init(adev);
1473 	if (r)
1474 		return r;
1475 
1476 	/*
1477 	 * number of VMs
1478 	 * VMID 0 is reserved for System
1479 	 * amdgpu graphics/compute will use VMIDs 1..n-1
1480 	 * amdkfd will use VMIDs n..15
1481 	 *
1482 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
1483 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
1484 	 * for video processing.
1485 	 */
1486 	adev->vm_manager.first_kfd_vmid =
1487 		adev->asic_type == CHIP_ARCTURUS ? 3 : 8;
1488 
1489 	amdgpu_vm_manager_init(adev);
1490 
1491 	gmc_v9_0_save_registers(adev);
1492 
1493 	return 0;
1494 }
1495 
1496 static int gmc_v9_0_sw_fini(void *handle)
1497 {
1498 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1499 
1500 	amdgpu_gmc_ras_fini(adev);
1501 	amdgpu_gem_force_release(adev);
1502 	amdgpu_vm_manager_fini(adev);
1503 	amdgpu_gart_table_vram_free(adev);
1504 	amdgpu_bo_fini(adev);
1505 	amdgpu_gart_fini(adev);
1506 
1507 	return 0;
1508 }
1509 
1510 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1511 {
1512 
1513 	switch (adev->asic_type) {
1514 	case CHIP_VEGA10:
1515 		if (amdgpu_sriov_vf(adev))
1516 			break;
1517 		fallthrough;
1518 	case CHIP_VEGA20:
1519 		soc15_program_register_sequence(adev,
1520 						golden_settings_mmhub_1_0_0,
1521 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
1522 		soc15_program_register_sequence(adev,
1523 						golden_settings_athub_1_0_0,
1524 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1525 		break;
1526 	case CHIP_VEGA12:
1527 		break;
1528 	case CHIP_RAVEN:
1529 		/* TODO for renoir */
1530 		soc15_program_register_sequence(adev,
1531 						golden_settings_athub_1_0_0,
1532 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1533 		break;
1534 	default:
1535 		break;
1536 	}
1537 }
1538 
1539 /**
1540  * gmc_v9_0_restore_registers - restores regs
1541  *
1542  * @adev: amdgpu_device pointer
1543  *
1544  * This restores register values, saved at suspend.
1545  */
1546 void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
1547 {
1548 	if (adev->asic_type == CHIP_RAVEN) {
1549 		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
1550 		WARN_ON(adev->gmc.sdpif_register !=
1551 			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
1552 	}
1553 }
1554 
1555 /**
1556  * gmc_v9_0_gart_enable - gart enable
1557  *
1558  * @adev: amdgpu_device pointer
1559  */
1560 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1561 {
1562 	int r;
1563 
1564 	if (adev->gart.bo == NULL) {
1565 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1566 		return -EINVAL;
1567 	}
1568 	r = amdgpu_gart_table_vram_pin(adev);
1569 	if (r)
1570 		return r;
1571 
1572 	r = gfxhub_v1_0_gart_enable(adev);
1573 	if (r)
1574 		return r;
1575 
1576 	r = adev->mmhub.funcs->gart_enable(adev);
1577 	if (r)
1578 		return r;
1579 
1580 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1581 		 (unsigned)(adev->gmc.gart_size >> 20),
1582 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1583 	adev->gart.ready = true;
1584 	return 0;
1585 }
1586 
1587 static int gmc_v9_0_hw_init(void *handle)
1588 {
1589 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1590 	bool value;
1591 	int r, i;
1592 	u32 tmp;
1593 
1594 	/* The sequence of these two function calls matters.*/
1595 	gmc_v9_0_init_golden_registers(adev);
1596 
1597 	if (adev->mode_info.num_crtc) {
1598 		if (adev->asic_type != CHIP_ARCTURUS) {
1599 			/* Lockout access through VGA aperture*/
1600 			WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1601 
1602 			/* disable VGA render */
1603 			WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1604 		}
1605 	}
1606 
1607 	amdgpu_device_program_register_sequence(adev,
1608 						golden_settings_vega10_hdp,
1609 						ARRAY_SIZE(golden_settings_vega10_hdp));
1610 
1611 	if (adev->mmhub.funcs->update_power_gating)
1612 		adev->mmhub.funcs->update_power_gating(adev, true);
1613 
1614 	switch (adev->asic_type) {
1615 	case CHIP_ARCTURUS:
1616 		WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1);
1617 		break;
1618 	default:
1619 		break;
1620 	}
1621 
1622 	WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
1623 
1624 	tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1625 	WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
1626 
1627 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));
1628 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
1629 
1630 	/* After HDP is initialized, flush HDP.*/
1631 	adev->nbio.funcs->hdp_flush(adev, NULL);
1632 
1633 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1634 		value = false;
1635 	else
1636 		value = true;
1637 
1638 	if (!amdgpu_sriov_vf(adev)) {
1639 		gfxhub_v1_0_set_fault_enable_default(adev, value);
1640 		adev->mmhub.funcs->set_fault_enable_default(adev, value);
1641 	}
1642 	for (i = 0; i < adev->num_vmhubs; ++i)
1643 		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
1644 
1645 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
1646 		adev->umc.funcs->init_registers(adev);
1647 
1648 	r = gmc_v9_0_gart_enable(adev);
1649 
1650 	return r;
1651 }
1652 
1653 /**
1654  * gmc_v9_0_gart_disable - gart disable
1655  *
1656  * @adev: amdgpu_device pointer
1657  *
1658  * This disables all VM page table.
1659  */
1660 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1661 {
1662 	gfxhub_v1_0_gart_disable(adev);
1663 	adev->mmhub.funcs->gart_disable(adev);
1664 	amdgpu_gart_table_vram_unpin(adev);
1665 }
1666 
1667 static int gmc_v9_0_hw_fini(void *handle)
1668 {
1669 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1670 
1671 	if (amdgpu_sriov_vf(adev)) {
1672 		/* full access mode, so don't touch any GMC register */
1673 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1674 		return 0;
1675 	}
1676 
1677 	amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
1678 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1679 	gmc_v9_0_gart_disable(adev);
1680 
1681 	return 0;
1682 }
1683 
1684 static int gmc_v9_0_suspend(void *handle)
1685 {
1686 	int r;
1687 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1688 
1689 	r = gmc_v9_0_hw_fini(adev);
1690 	if (r)
1691 		return r;
1692 
1693 	return 0;
1694 }
1695 
1696 static int gmc_v9_0_resume(void *handle)
1697 {
1698 	int r;
1699 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1700 
1701 	r = gmc_v9_0_hw_init(adev);
1702 	if (r)
1703 		return r;
1704 
1705 	amdgpu_vmid_reset_all(adev);
1706 
1707 	return 0;
1708 }
1709 
1710 static bool gmc_v9_0_is_idle(void *handle)
1711 {
1712 	/* MC is always ready in GMC v9.*/
1713 	return true;
1714 }
1715 
1716 static int gmc_v9_0_wait_for_idle(void *handle)
1717 {
1718 	/* There is no need to wait for MC idle in GMC v9.*/
1719 	return 0;
1720 }
1721 
1722 static int gmc_v9_0_soft_reset(void *handle)
1723 {
1724 	/* XXX for emulation.*/
1725 	return 0;
1726 }
1727 
1728 static int gmc_v9_0_set_clockgating_state(void *handle,
1729 					enum amd_clockgating_state state)
1730 {
1731 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1732 
1733 	adev->mmhub.funcs->set_clockgating(adev, state);
1734 
1735 	athub_v1_0_set_clockgating(adev, state);
1736 
1737 	return 0;
1738 }
1739 
1740 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1741 {
1742 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1743 
1744 	adev->mmhub.funcs->get_clockgating(adev, flags);
1745 
1746 	athub_v1_0_get_clockgating(adev, flags);
1747 }
1748 
1749 static int gmc_v9_0_set_powergating_state(void *handle,
1750 					enum amd_powergating_state state)
1751 {
1752 	return 0;
1753 }
1754 
1755 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1756 	.name = "gmc_v9_0",
1757 	.early_init = gmc_v9_0_early_init,
1758 	.late_init = gmc_v9_0_late_init,
1759 	.sw_init = gmc_v9_0_sw_init,
1760 	.sw_fini = gmc_v9_0_sw_fini,
1761 	.hw_init = gmc_v9_0_hw_init,
1762 	.hw_fini = gmc_v9_0_hw_fini,
1763 	.suspend = gmc_v9_0_suspend,
1764 	.resume = gmc_v9_0_resume,
1765 	.is_idle = gmc_v9_0_is_idle,
1766 	.wait_for_idle = gmc_v9_0_wait_for_idle,
1767 	.soft_reset = gmc_v9_0_soft_reset,
1768 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
1769 	.set_powergating_state = gmc_v9_0_set_powergating_state,
1770 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
1771 };
1772 
1773 const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1774 {
1775 	.type = AMD_IP_BLOCK_TYPE_GMC,
1776 	.major = 9,
1777 	.minor = 0,
1778 	.rev = 0,
1779 	.funcs = &gmc_v9_0_ip_funcs,
1780 };
1781