1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_bo.h> 7 8 #include "intel_display_types.h" 9 #include "intel_dpt.h" 10 #include "intel_fb.h" 11 #include "intel_fb_pin.h" 12 #include "intel_fbdev.h" 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_ggtt.h" 16 #include "xe_pm.h" 17 18 static void 19 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 20 u32 width, u32 height, u32 src_stride, u32 dst_stride) 21 { 22 struct xe_device *xe = xe_bo_device(bo); 23 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 24 u32 column, row; 25 26 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 27 * by writing dpt/ggtt in a different order? 28 */ 29 30 for (column = 0; column < width; column++) { 31 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 32 33 for (row = 0; row < height; row++) { 34 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 35 xe->pat.idx[XE_CACHE_NONE]); 36 37 iosys_map_wr(map, *dpt_ofs, u64, pte); 38 *dpt_ofs += 8; 39 src_idx -= src_stride; 40 } 41 42 /* The DE ignores the PTEs for the padding tiles */ 43 *dpt_ofs += (dst_stride - height) * 8; 44 } 45 46 /* Align to next page */ 47 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 48 } 49 50 static void 51 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 52 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 53 u32 dst_stride) 54 { 55 struct xe_device *xe = xe_bo_device(bo); 56 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 57 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index) 58 = ggtt->pt_ops->pte_encode_bo; 59 u32 column, row; 60 61 for (row = 0; row < height; row++) { 62 u32 src_idx = src_stride * row + bo_ofs; 63 64 for (column = 0; column < width; column++) { 65 iosys_map_wr(map, *dpt_ofs, u64, 66 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 67 xe->pat.idx[XE_CACHE_NONE])); 68 69 *dpt_ofs += 8; 70 src_idx++; 71 } 72 73 /* The DE ignores the PTEs for the padding tiles */ 74 *dpt_ofs += (dst_stride - width) * 8; 75 } 76 77 /* Align to next page */ 78 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 79 } 80 81 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, 82 const struct i915_gtt_view *view, 83 struct i915_vma *vma, 84 u64 physical_alignment) 85 { 86 struct xe_device *xe = to_xe_device(fb->base.dev); 87 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 88 struct xe_ggtt *ggtt = tile0->mem.ggtt; 89 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 90 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; 91 u32 dpt_size, size = bo->ttm.base.size; 92 93 if (view->type == I915_GTT_VIEW_NORMAL) 94 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 95 else if (view->type == I915_GTT_VIEW_REMAPPED) 96 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 97 XE_PAGE_SIZE); 98 else 99 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 100 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 101 XE_PAGE_SIZE); 102 103 if (IS_DGFX(xe)) 104 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 105 dpt_size, ~0ull, 106 ttm_bo_type_kernel, 107 XE_BO_FLAG_VRAM0 | 108 XE_BO_FLAG_GGTT | 109 XE_BO_FLAG_PAGETABLE, 110 physical_alignment); 111 else 112 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 113 dpt_size, ~0ull, 114 ttm_bo_type_kernel, 115 XE_BO_FLAG_STOLEN | 116 XE_BO_FLAG_GGTT | 117 XE_BO_FLAG_PAGETABLE, 118 physical_alignment); 119 if (IS_ERR(dpt)) 120 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 121 dpt_size, ~0ull, 122 ttm_bo_type_kernel, 123 XE_BO_FLAG_SYSTEM | 124 XE_BO_FLAG_GGTT | 125 XE_BO_FLAG_PAGETABLE, 126 physical_alignment); 127 if (IS_ERR(dpt)) 128 return PTR_ERR(dpt); 129 130 if (view->type == I915_GTT_VIEW_NORMAL) { 131 u32 x; 132 133 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 134 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE, 135 xe->pat.idx[XE_CACHE_NONE]); 136 137 iosys_map_wr(&dpt->vmap, x * 8, u64, pte); 138 } 139 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 140 const struct intel_remapped_info *remap_info = &view->remapped; 141 u32 i, dpt_ofs = 0; 142 143 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 144 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 145 remap_info->plane[i].offset, 146 remap_info->plane[i].width, 147 remap_info->plane[i].height, 148 remap_info->plane[i].src_stride, 149 remap_info->plane[i].dst_stride); 150 151 } else { 152 const struct intel_rotation_info *rot_info = &view->rotated; 153 u32 i, dpt_ofs = 0; 154 155 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 156 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 157 rot_info->plane[i].offset, 158 rot_info->plane[i].width, 159 rot_info->plane[i].height, 160 rot_info->plane[i].src_stride, 161 rot_info->plane[i].dst_stride); 162 } 163 164 vma->dpt = dpt; 165 vma->node = dpt->ggtt_node[tile0->id]; 166 return 0; 167 } 168 169 static void 170 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs, 171 u32 width, u32 height, u32 src_stride, u32 dst_stride) 172 { 173 struct xe_device *xe = xe_bo_device(bo); 174 u32 column, row; 175 176 for (column = 0; column < width; column++) { 177 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 178 179 for (row = 0; row < height; row++) { 180 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 181 xe->pat.idx[XE_CACHE_NONE]); 182 183 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte); 184 *ggtt_ofs += XE_PAGE_SIZE; 185 src_idx -= src_stride; 186 } 187 188 /* The DE ignores the PTEs for the padding tiles */ 189 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 190 } 191 } 192 193 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, 194 const struct i915_gtt_view *view, 195 struct i915_vma *vma, 196 u64 physical_alignment) 197 { 198 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 199 struct xe_bo *bo = gem_to_xe_bo(obj); 200 struct xe_device *xe = to_xe_device(fb->base.dev); 201 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 202 u32 align; 203 int ret; 204 205 /* TODO: Consider sharing framebuffer mapping? 206 * embed i915_vma inside intel_framebuffer 207 */ 208 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile)); 209 ret = mutex_lock_interruptible(&ggtt->lock); 210 if (ret) 211 goto out; 212 213 align = XE_PAGE_SIZE; 214 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 215 align = max_t(u32, align, SZ_64K); 216 217 if (bo->ggtt_node[ggtt->tile->id] && view->type == I915_GTT_VIEW_NORMAL) { 218 vma->node = bo->ggtt_node[ggtt->tile->id]; 219 } else if (view->type == I915_GTT_VIEW_NORMAL) { 220 u32 x, size = bo->ttm.base.size; 221 222 vma->node = xe_ggtt_node_init(ggtt); 223 if (IS_ERR(vma->node)) { 224 ret = PTR_ERR(vma->node); 225 goto out_unlock; 226 } 227 228 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); 229 if (ret) { 230 xe_ggtt_node_fini(vma->node); 231 goto out_unlock; 232 } 233 234 for (x = 0; x < size; x += XE_PAGE_SIZE) { 235 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x, 236 xe->pat.idx[XE_CACHE_NONE]); 237 238 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node->base.start + x, pte); 239 } 240 } else { 241 u32 i, ggtt_ofs; 242 const struct intel_rotation_info *rot_info = &view->rotated; 243 244 /* display seems to use tiles instead of bytes here, so convert it back.. */ 245 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE; 246 247 vma->node = xe_ggtt_node_init(ggtt); 248 if (IS_ERR(vma->node)) { 249 ret = PTR_ERR(vma->node); 250 goto out_unlock; 251 } 252 253 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); 254 if (ret) { 255 xe_ggtt_node_fini(vma->node); 256 goto out_unlock; 257 } 258 259 ggtt_ofs = vma->node->base.start; 260 261 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 262 write_ggtt_rotated(bo, ggtt, &ggtt_ofs, 263 rot_info->plane[i].offset, 264 rot_info->plane[i].width, 265 rot_info->plane[i].height, 266 rot_info->plane[i].src_stride, 267 rot_info->plane[i].dst_stride); 268 } 269 270 out_unlock: 271 mutex_unlock(&ggtt->lock); 272 out: 273 xe_pm_runtime_put(tile_to_xe(ggtt->tile)); 274 return ret; 275 } 276 277 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, 278 const struct i915_gtt_view *view, 279 u64 physical_alignment) 280 { 281 struct drm_device *dev = fb->base.dev; 282 struct xe_device *xe = to_xe_device(dev); 283 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 284 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 285 struct xe_bo *bo = gem_to_xe_bo(obj); 286 int ret; 287 288 if (!vma) 289 return ERR_PTR(-ENODEV); 290 291 refcount_set(&vma->ref, 1); 292 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 293 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 294 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 295 struct xe_tile *tile = xe_device_get_root_tile(xe); 296 297 /* 298 * If we need to able to access the clear-color value stored in 299 * the buffer, then we require that such buffers are also CPU 300 * accessible. This is important on small-bar systems where 301 * only some subset of VRAM is CPU accessible. 302 */ 303 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { 304 ret = -EINVAL; 305 goto err; 306 } 307 } 308 309 /* 310 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 311 * assumptions are incorrect for framebuffers 312 */ 313 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL); 314 if (ret) 315 goto err; 316 317 if (IS_DGFX(xe)) 318 ret = xe_bo_migrate(bo, XE_PL_VRAM0); 319 else 320 ret = xe_bo_validate(bo, NULL, true); 321 if (!ret) 322 ttm_bo_pin(&bo->ttm); 323 ttm_bo_unreserve(&bo->ttm); 324 if (ret) 325 goto err; 326 327 vma->bo = bo; 328 if (intel_fb_uses_dpt(&fb->base)) 329 ret = __xe_pin_fb_vma_dpt(fb, view, vma, physical_alignment); 330 else 331 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, physical_alignment); 332 if (ret) 333 goto err_unpin; 334 335 /* Ensure DPT writes are flushed */ 336 xe_device_l2_flush(xe); 337 return vma; 338 339 err_unpin: 340 ttm_bo_reserve(&bo->ttm, false, false, NULL); 341 ttm_bo_unpin(&bo->ttm); 342 ttm_bo_unreserve(&bo->ttm); 343 err: 344 kfree(vma); 345 return ERR_PTR(ret); 346 } 347 348 static void __xe_unpin_fb_vma(struct i915_vma *vma) 349 { 350 u8 tile_id = vma->node->ggtt->tile->id; 351 352 if (!refcount_dec_and_test(&vma->ref)) 353 return; 354 355 if (vma->dpt) 356 xe_bo_unpin_map_no_vm(vma->dpt); 357 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node[tile_id]) || 358 vma->bo->ggtt_node[tile_id]->base.start != vma->node->base.start) 359 xe_ggtt_node_remove(vma->node, false); 360 361 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 362 ttm_bo_unpin(&vma->bo->ttm); 363 ttm_bo_unreserve(&vma->bo->ttm); 364 kfree(vma); 365 } 366 367 struct i915_vma * 368 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, 369 const struct i915_gtt_view *view, 370 unsigned int alignment, 371 unsigned int phys_alignment, 372 unsigned int vtd_guard, 373 bool uses_fence, 374 unsigned long *out_flags) 375 { 376 *out_flags = 0; 377 378 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment); 379 } 380 381 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) 382 { 383 __xe_unpin_fb_vma(vma); 384 } 385 386 static bool reuse_vma(struct intel_plane_state *new_plane_state, 387 const struct intel_plane_state *old_plane_state) 388 { 389 struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb); 390 struct xe_device *xe = to_xe_device(fb->base.dev); 391 struct i915_vma *vma; 392 393 if (old_plane_state->hw.fb == new_plane_state->hw.fb && 394 !memcmp(&old_plane_state->view.gtt, 395 &new_plane_state->view.gtt, 396 sizeof(new_plane_state->view.gtt))) { 397 vma = old_plane_state->ggtt_vma; 398 goto found; 399 } 400 401 if (fb == intel_fbdev_framebuffer(xe->display.fbdev.fbdev)) { 402 vma = intel_fbdev_vma_pointer(xe->display.fbdev.fbdev); 403 if (vma) 404 goto found; 405 } 406 407 return false; 408 409 found: 410 refcount_inc(&vma->ref); 411 new_plane_state->ggtt_vma = vma; 412 return true; 413 } 414 415 int intel_plane_pin_fb(struct intel_plane_state *new_plane_state, 416 const struct intel_plane_state *old_plane_state) 417 { 418 struct drm_framebuffer *fb = new_plane_state->hw.fb; 419 struct drm_gem_object *obj = intel_fb_bo(fb); 420 struct xe_bo *bo = gem_to_xe_bo(obj); 421 struct i915_vma *vma; 422 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 423 struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); 424 u64 phys_alignment = plane->min_alignment(plane, fb, 0); 425 426 if (reuse_vma(new_plane_state, old_plane_state)) 427 return 0; 428 429 /* We reject creating !SCANOUT fb's, so this is weird.. */ 430 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT)); 431 432 vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, phys_alignment); 433 434 if (IS_ERR(vma)) 435 return PTR_ERR(vma); 436 437 new_plane_state->ggtt_vma = vma; 438 return 0; 439 } 440 441 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 442 { 443 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 444 old_plane_state->ggtt_vma = NULL; 445 } 446 447 /* 448 * For Xe introduce dummy intel_dpt_create which just return NULL, 449 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0; 450 */ 451 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb) 452 { 453 return NULL; 454 } 455 456 void intel_dpt_destroy(struct i915_address_space *vm) 457 { 458 return; 459 } 460 461 u64 intel_dpt_offset(struct i915_vma *dpt_vma) 462 { 463 return 0; 464 } 465