1 /* 2 * Copyright © 2008-2010 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <[email protected]> 25 * Zou Nan hai <[email protected]> 26 * Xiang Hai hao<[email protected]> 27 * 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <dev/drm2/drmP.h> 34 #include <dev/drm2/drm.h> 35 #include <dev/drm2/i915/i915_drm.h> 36 #include <dev/drm2/i915/i915_drv.h> 37 #include <dev/drm2/i915/intel_drv.h> 38 #include <dev/drm2/i915/intel_ringbuffer.h> 39 #include <sys/sched.h> 40 #include <sys/sf_buf.h> 41 42 /* 43 * 965+ support PIPE_CONTROL commands, which provide finer grained control 44 * over cache flushing. 45 */ 46 struct pipe_control { 47 struct drm_i915_gem_object *obj; 48 volatile u32 *cpu_page; 49 u32 gtt_offset; 50 }; 51 52 void 53 i915_trace_irq_get(struct intel_ring_buffer *ring, uint32_t seqno) 54 { 55 56 if (ring->trace_irq_seqno == 0) { 57 mtx_lock(&ring->irq_lock); 58 if (ring->irq_get(ring)) 59 ring->trace_irq_seqno = seqno; 60 mtx_unlock(&ring->irq_lock); 61 } 62 } 63 64 static inline int ring_space(struct intel_ring_buffer *ring) 65 { 66 int space = (ring->head & HEAD_ADDR) - (ring->tail + 8); 67 if (space < 0) 68 space += ring->size; 69 return space; 70 } 71 72 static int 73 render_ring_flush(struct intel_ring_buffer *ring, 74 uint32_t invalidate_domains, 75 uint32_t flush_domains) 76 { 77 struct drm_device *dev = ring->dev; 78 uint32_t cmd; 79 int ret; 80 81 /* 82 * read/write caches: 83 * 84 * I915_GEM_DOMAIN_RENDER is always invalidated, but is 85 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is 86 * also flushed at 2d versus 3d pipeline switches. 87 * 88 * read-only caches: 89 * 90 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if 91 * MI_READ_FLUSH is set, and is always flushed on 965. 92 * 93 * I915_GEM_DOMAIN_COMMAND may not exist? 94 * 95 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is 96 * invalidated when MI_EXE_FLUSH is set. 97 * 98 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is 99 * invalidated with every MI_FLUSH. 100 * 101 * TLBs: 102 * 103 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND 104 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and 105 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER 106 * are flushed at any MI_FLUSH. 107 */ 108 109 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; 110 if ((invalidate_domains|flush_domains) & 111 I915_GEM_DOMAIN_RENDER) 112 cmd &= ~MI_NO_WRITE_FLUSH; 113 if (INTEL_INFO(dev)->gen < 4) { 114 /* 115 * On the 965, the sampler cache always gets flushed 116 * and this bit is reserved. 117 */ 118 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) 119 cmd |= MI_READ_FLUSH; 120 } 121 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) 122 cmd |= MI_EXE_FLUSH; 123 124 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && 125 (IS_G4X(dev) || IS_GEN5(dev))) 126 cmd |= MI_INVALIDATE_ISP; 127 128 ret = intel_ring_begin(ring, 2); 129 if (ret) 130 return ret; 131 132 intel_ring_emit(ring, cmd); 133 intel_ring_emit(ring, MI_NOOP); 134 intel_ring_advance(ring); 135 136 return 0; 137 } 138 139 /** 140 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for 141 * implementing two workarounds on gen6. From section 1.4.7.1 142 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: 143 * 144 * [DevSNB-C+{W/A}] Before any depth stall flush (including those 145 * produced by non-pipelined state commands), software needs to first 146 * send a PIPE_CONTROL with no bits set except Post-Sync Operation != 147 * 0. 148 * 149 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable 150 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. 151 * 152 * And the workaround for these two requires this workaround first: 153 * 154 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent 155 * BEFORE the pipe-control with a post-sync op and no write-cache 156 * flushes. 157 * 158 * And this last workaround is tricky because of the requirements on 159 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM 160 * volume 2 part 1: 161 * 162 * "1 of the following must also be set: 163 * - Render Target Cache Flush Enable ([12] of DW1) 164 * - Depth Cache Flush Enable ([0] of DW1) 165 * - Stall at Pixel Scoreboard ([1] of DW1) 166 * - Depth Stall ([13] of DW1) 167 * - Post-Sync Operation ([13] of DW1) 168 * - Notify Enable ([8] of DW1)" 169 * 170 * The cache flushes require the workaround flush that triggered this 171 * one, so we can't use it. Depth stall would trigger the same. 172 * Post-sync nonzero is what triggered this second workaround, so we 173 * can't use that one either. Notify enable is IRQs, which aren't 174 * really our business. That leaves only stall at scoreboard. 175 */ 176 static int 177 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring) 178 { 179 struct pipe_control *pc = ring->private; 180 u32 scratch_addr = pc->gtt_offset + 128; 181 int ret; 182 183 184 ret = intel_ring_begin(ring, 6); 185 if (ret) 186 return ret; 187 188 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); 189 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | 190 PIPE_CONTROL_STALL_AT_SCOREBOARD); 191 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ 192 intel_ring_emit(ring, 0); /* low dword */ 193 intel_ring_emit(ring, 0); /* high dword */ 194 intel_ring_emit(ring, MI_NOOP); 195 intel_ring_advance(ring); 196 197 ret = intel_ring_begin(ring, 6); 198 if (ret) 199 return ret; 200 201 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); 202 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); 203 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ 204 intel_ring_emit(ring, 0); 205 intel_ring_emit(ring, 0); 206 intel_ring_emit(ring, MI_NOOP); 207 intel_ring_advance(ring); 208 209 return 0; 210 } 211 212 static int 213 gen6_render_ring_flush(struct intel_ring_buffer *ring, 214 u32 invalidate_domains, u32 flush_domains) 215 { 216 u32 flags = 0; 217 struct pipe_control *pc = ring->private; 218 u32 scratch_addr = pc->gtt_offset + 128; 219 int ret; 220 221 /* Force SNB workarounds for PIPE_CONTROL flushes */ 222 intel_emit_post_sync_nonzero_flush(ring); 223 224 /* Just flush everything. Experiments have shown that reducing the 225 * number of bits based on the write domains has little performance 226 * impact. 227 */ 228 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 229 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 230 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 231 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 232 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 233 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 234 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 235 236 ret = intel_ring_begin(ring, 6); 237 if (ret) 238 return ret; 239 240 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); 241 intel_ring_emit(ring, flags); 242 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); 243 intel_ring_emit(ring, 0); /* lower dword */ 244 intel_ring_emit(ring, 0); /* uppwer dword */ 245 intel_ring_emit(ring, MI_NOOP); 246 intel_ring_advance(ring); 247 248 return 0; 249 } 250 251 static void ring_write_tail(struct intel_ring_buffer *ring, 252 uint32_t value) 253 { 254 drm_i915_private_t *dev_priv = ring->dev->dev_private; 255 I915_WRITE_TAIL(ring, value); 256 } 257 258 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring) 259 { 260 drm_i915_private_t *dev_priv = ring->dev->dev_private; 261 uint32_t acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ? 262 RING_ACTHD(ring->mmio_base) : ACTHD; 263 264 return I915_READ(acthd_reg); 265 } 266 267 static int init_ring_common(struct intel_ring_buffer *ring) 268 { 269 drm_i915_private_t *dev_priv = ring->dev->dev_private; 270 struct drm_i915_gem_object *obj = ring->obj; 271 uint32_t head; 272 273 /* Stop the ring if it's running. */ 274 I915_WRITE_CTL(ring, 0); 275 I915_WRITE_HEAD(ring, 0); 276 ring->write_tail(ring, 0); 277 278 /* Initialize the ring. */ 279 I915_WRITE_START(ring, obj->gtt_offset); 280 head = I915_READ_HEAD(ring) & HEAD_ADDR; 281 282 /* G45 ring initialization fails to reset head to zero */ 283 if (head != 0) { 284 DRM_DEBUG("%s head not reset to zero " 285 "ctl %08x head %08x tail %08x start %08x\n", 286 ring->name, 287 I915_READ_CTL(ring), 288 I915_READ_HEAD(ring), 289 I915_READ_TAIL(ring), 290 I915_READ_START(ring)); 291 292 I915_WRITE_HEAD(ring, 0); 293 294 if (I915_READ_HEAD(ring) & HEAD_ADDR) { 295 DRM_ERROR("failed to set %s head to zero " 296 "ctl %08x head %08x tail %08x start %08x\n", 297 ring->name, 298 I915_READ_CTL(ring), 299 I915_READ_HEAD(ring), 300 I915_READ_TAIL(ring), 301 I915_READ_START(ring)); 302 } 303 } 304 305 I915_WRITE_CTL(ring, 306 ((ring->size - PAGE_SIZE) & RING_NR_PAGES) 307 | RING_VALID); 308 309 /* If the head is still not zero, the ring is dead */ 310 if (_intel_wait_for(ring->dev, 311 (I915_READ_CTL(ring) & RING_VALID) != 0 && 312 I915_READ_START(ring) == obj->gtt_offset && 313 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 314 50, 1, "915rii")) { 315 DRM_ERROR("%s initialization failed " 316 "ctl %08x head %08x tail %08x start %08x\n", 317 ring->name, 318 I915_READ_CTL(ring), 319 I915_READ_HEAD(ring), 320 I915_READ_TAIL(ring), 321 I915_READ_START(ring)); 322 return -EIO; 323 } 324 325 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) 326 i915_kernel_lost_context(ring->dev); 327 else { 328 ring->head = I915_READ_HEAD(ring); 329 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; 330 ring->space = ring_space(ring); 331 } 332 333 return 0; 334 } 335 336 static int 337 init_pipe_control(struct intel_ring_buffer *ring) 338 { 339 struct pipe_control *pc; 340 struct drm_i915_gem_object *obj; 341 int ret; 342 343 if (ring->private) 344 return 0; 345 346 pc = malloc(sizeof(*pc), DRM_I915_GEM, M_WAITOK); 347 if (!pc) 348 return -ENOMEM; 349 350 obj = i915_gem_alloc_object(ring->dev, 4096); 351 if (obj == NULL) { 352 DRM_ERROR("Failed to allocate seqno page\n"); 353 ret = -ENOMEM; 354 goto err; 355 } 356 357 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); 358 359 ret = i915_gem_object_pin(obj, 4096, true); 360 if (ret) 361 goto err_unref; 362 363 pc->gtt_offset = obj->gtt_offset; 364 pc->cpu_page = (uint32_t *)kmem_alloc_nofault(kernel_map, PAGE_SIZE); 365 if (pc->cpu_page == NULL) 366 goto err_unpin; 367 pmap_qenter((uintptr_t)pc->cpu_page, &obj->pages[0], 1); 368 pmap_invalidate_range(kernel_pmap, (vm_offset_t)pc->cpu_page, 369 (vm_offset_t)pc->cpu_page + PAGE_SIZE); 370 pmap_invalidate_cache_range((vm_offset_t)pc->cpu_page, 371 (vm_offset_t)pc->cpu_page + PAGE_SIZE); 372 373 pc->obj = obj; 374 ring->private = pc; 375 return 0; 376 377 err_unpin: 378 i915_gem_object_unpin(obj); 379 err_unref: 380 drm_gem_object_unreference(&obj->base); 381 err: 382 free(pc, DRM_I915_GEM); 383 return ret; 384 } 385 386 static void 387 cleanup_pipe_control(struct intel_ring_buffer *ring) 388 { 389 struct pipe_control *pc = ring->private; 390 struct drm_i915_gem_object *obj; 391 392 if (!ring->private) 393 return; 394 395 obj = pc->obj; 396 pmap_qremove((vm_offset_t)pc->cpu_page, 1); 397 pmap_invalidate_range(kernel_pmap, (vm_offset_t)pc->cpu_page, 398 (vm_offset_t)pc->cpu_page + PAGE_SIZE); 399 kmem_free(kernel_map, (uintptr_t)pc->cpu_page, PAGE_SIZE); 400 i915_gem_object_unpin(obj); 401 drm_gem_object_unreference(&obj->base); 402 403 free(pc, DRM_I915_GEM); 404 ring->private = NULL; 405 } 406 407 static int init_render_ring(struct intel_ring_buffer *ring) 408 { 409 struct drm_device *dev = ring->dev; 410 struct drm_i915_private *dev_priv = dev->dev_private; 411 int ret = init_ring_common(ring); 412 413 if (INTEL_INFO(dev)->gen > 3) { 414 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; 415 I915_WRITE(MI_MODE, mode); 416 if (IS_GEN7(dev)) 417 I915_WRITE(GFX_MODE_GEN7, 418 GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) | 419 GFX_MODE_ENABLE(GFX_REPLAY_MODE)); 420 } 421 422 if (INTEL_INFO(dev)->gen >= 5) { 423 ret = init_pipe_control(ring); 424 if (ret) 425 return ret; 426 } 427 428 429 if (IS_GEN6(dev)) { 430 /* From the Sandybridge PRM, volume 1 part 3, page 24: 431 * "If this bit is set, STCunit will have LRA as replacement 432 * policy. [...] This bit must be reset. LRA replacement 433 * policy is not supported." 434 */ 435 I915_WRITE(CACHE_MODE_0, 436 CM0_STC_EVICT_DISABLE_LRA_SNB << CM0_MASK_SHIFT); 437 } 438 439 if (INTEL_INFO(dev)->gen >= 6) { 440 I915_WRITE(INSTPM, 441 INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING); 442 } 443 444 return ret; 445 } 446 447 static void render_ring_cleanup(struct intel_ring_buffer *ring) 448 { 449 if (!ring->private) 450 return; 451 452 cleanup_pipe_control(ring); 453 } 454 455 static void 456 update_mboxes(struct intel_ring_buffer *ring, 457 u32 seqno, 458 u32 mmio_offset) 459 { 460 intel_ring_emit(ring, MI_SEMAPHORE_MBOX | 461 MI_SEMAPHORE_GLOBAL_GTT | 462 MI_SEMAPHORE_REGISTER | 463 MI_SEMAPHORE_UPDATE); 464 intel_ring_emit(ring, seqno); 465 intel_ring_emit(ring, mmio_offset); 466 } 467 468 /** 469 * gen6_add_request - Update the semaphore mailbox registers 470 * 471 * @ring - ring that is adding a request 472 * @seqno - return seqno stuck into the ring 473 * 474 * Update the mailbox registers in the *other* rings with the current seqno. 475 * This acts like a signal in the canonical semaphore. 476 */ 477 static int 478 gen6_add_request(struct intel_ring_buffer *ring, 479 u32 *seqno) 480 { 481 u32 mbox1_reg; 482 u32 mbox2_reg; 483 int ret; 484 485 ret = intel_ring_begin(ring, 10); 486 if (ret) 487 return ret; 488 489 mbox1_reg = ring->signal_mbox[0]; 490 mbox2_reg = ring->signal_mbox[1]; 491 492 *seqno = i915_gem_next_request_seqno(ring); 493 494 update_mboxes(ring, *seqno, mbox1_reg); 495 update_mboxes(ring, *seqno, mbox2_reg); 496 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); 497 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 498 intel_ring_emit(ring, *seqno); 499 intel_ring_emit(ring, MI_USER_INTERRUPT); 500 intel_ring_advance(ring); 501 502 return 0; 503 } 504 505 /** 506 * intel_ring_sync - sync the waiter to the signaller on seqno 507 * 508 * @waiter - ring that is waiting 509 * @signaller - ring which has, or will signal 510 * @seqno - seqno which the waiter will block on 511 */ 512 static int 513 intel_ring_sync(struct intel_ring_buffer *waiter, 514 struct intel_ring_buffer *signaller, 515 int ring, 516 u32 seqno) 517 { 518 int ret; 519 u32 dw1 = MI_SEMAPHORE_MBOX | 520 MI_SEMAPHORE_COMPARE | 521 MI_SEMAPHORE_REGISTER; 522 523 ret = intel_ring_begin(waiter, 4); 524 if (ret) 525 return ret; 526 527 intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]); 528 intel_ring_emit(waiter, seqno); 529 intel_ring_emit(waiter, 0); 530 intel_ring_emit(waiter, MI_NOOP); 531 intel_ring_advance(waiter); 532 533 return 0; 534 } 535 536 int render_ring_sync_to(struct intel_ring_buffer *waiter, 537 struct intel_ring_buffer *signaller, u32 seqno); 538 int gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter, 539 struct intel_ring_buffer *signaller, u32 seqno); 540 int gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter, 541 struct intel_ring_buffer *signaller, u32 seqno); 542 543 /* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */ 544 int 545 render_ring_sync_to(struct intel_ring_buffer *waiter, 546 struct intel_ring_buffer *signaller, 547 u32 seqno) 548 { 549 KASSERT(signaller->semaphore_register[RCS] != MI_SEMAPHORE_SYNC_INVALID, 550 ("valid RCS semaphore")); 551 return intel_ring_sync(waiter, 552 signaller, 553 RCS, 554 seqno); 555 } 556 557 /* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */ 558 int 559 gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter, 560 struct intel_ring_buffer *signaller, 561 u32 seqno) 562 { 563 KASSERT(signaller->semaphore_register[VCS] != MI_SEMAPHORE_SYNC_INVALID, 564 ("Valid VCS semaphore")); 565 return intel_ring_sync(waiter, 566 signaller, 567 VCS, 568 seqno); 569 } 570 571 /* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */ 572 int 573 gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter, 574 struct intel_ring_buffer *signaller, 575 u32 seqno) 576 { 577 KASSERT(signaller->semaphore_register[BCS] != MI_SEMAPHORE_SYNC_INVALID, 578 ("Valid BCS semaphore")); 579 return intel_ring_sync(waiter, 580 signaller, 581 BCS, 582 seqno); 583 } 584 585 #define PIPE_CONTROL_FLUSH(ring__, addr__) \ 586 do { \ 587 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \ 588 PIPE_CONTROL_DEPTH_STALL); \ 589 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \ 590 intel_ring_emit(ring__, 0); \ 591 intel_ring_emit(ring__, 0); \ 592 } while (0) 593 594 static int 595 pc_render_add_request(struct intel_ring_buffer *ring, 596 uint32_t *result) 597 { 598 u32 seqno = i915_gem_next_request_seqno(ring); 599 struct pipe_control *pc = ring->private; 600 u32 scratch_addr = pc->gtt_offset + 128; 601 int ret; 602 603 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently 604 * incoherent with writes to memory, i.e. completely fubar, 605 * so we need to use PIPE_NOTIFY instead. 606 * 607 * However, we also need to workaround the qword write 608 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to 609 * memory before requesting an interrupt. 610 */ 611 ret = intel_ring_begin(ring, 32); 612 if (ret) 613 return ret; 614 615 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | 616 PIPE_CONTROL_WRITE_FLUSH | 617 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); 618 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); 619 intel_ring_emit(ring, seqno); 620 intel_ring_emit(ring, 0); 621 PIPE_CONTROL_FLUSH(ring, scratch_addr); 622 scratch_addr += 128; /* write to separate cachelines */ 623 PIPE_CONTROL_FLUSH(ring, scratch_addr); 624 scratch_addr += 128; 625 PIPE_CONTROL_FLUSH(ring, scratch_addr); 626 scratch_addr += 128; 627 PIPE_CONTROL_FLUSH(ring, scratch_addr); 628 scratch_addr += 128; 629 PIPE_CONTROL_FLUSH(ring, scratch_addr); 630 scratch_addr += 128; 631 PIPE_CONTROL_FLUSH(ring, scratch_addr); 632 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | 633 PIPE_CONTROL_WRITE_FLUSH | 634 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 635 PIPE_CONTROL_NOTIFY); 636 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); 637 intel_ring_emit(ring, seqno); 638 intel_ring_emit(ring, 0); 639 intel_ring_advance(ring); 640 641 *result = seqno; 642 return 0; 643 } 644 645 static int 646 render_ring_add_request(struct intel_ring_buffer *ring, 647 uint32_t *result) 648 { 649 u32 seqno = i915_gem_next_request_seqno(ring); 650 int ret; 651 652 ret = intel_ring_begin(ring, 4); 653 if (ret) 654 return ret; 655 656 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); 657 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 658 intel_ring_emit(ring, seqno); 659 intel_ring_emit(ring, MI_USER_INTERRUPT); 660 intel_ring_advance(ring); 661 662 *result = seqno; 663 return 0; 664 } 665 666 static u32 667 gen6_ring_get_seqno(struct intel_ring_buffer *ring) 668 { 669 struct drm_device *dev = ring->dev; 670 671 /* Workaround to force correct ordering between irq and seqno writes on 672 * ivb (and maybe also on snb) by reading from a CS register (like 673 * ACTHD) before reading the status page. */ 674 if (/* IS_GEN6(dev) || */IS_GEN7(dev)) 675 intel_ring_get_active_head(ring); 676 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 677 } 678 679 static uint32_t 680 ring_get_seqno(struct intel_ring_buffer *ring) 681 { 682 if (ring->status_page.page_addr == NULL) 683 return (-1); 684 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 685 } 686 687 static uint32_t 688 pc_render_get_seqno(struct intel_ring_buffer *ring) 689 { 690 struct pipe_control *pc = ring->private; 691 if (pc != NULL) 692 return pc->cpu_page[0]; 693 else 694 return (-1); 695 } 696 697 static void 698 ironlake_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask) 699 { 700 dev_priv->gt_irq_mask &= ~mask; 701 I915_WRITE(GTIMR, dev_priv->gt_irq_mask); 702 POSTING_READ(GTIMR); 703 } 704 705 static void 706 ironlake_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask) 707 { 708 dev_priv->gt_irq_mask |= mask; 709 I915_WRITE(GTIMR, dev_priv->gt_irq_mask); 710 POSTING_READ(GTIMR); 711 } 712 713 static void 714 i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask) 715 { 716 dev_priv->irq_mask &= ~mask; 717 I915_WRITE(IMR, dev_priv->irq_mask); 718 POSTING_READ(IMR); 719 } 720 721 static void 722 i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask) 723 { 724 dev_priv->irq_mask |= mask; 725 I915_WRITE(IMR, dev_priv->irq_mask); 726 POSTING_READ(IMR); 727 } 728 729 static bool 730 render_ring_get_irq(struct intel_ring_buffer *ring) 731 { 732 struct drm_device *dev = ring->dev; 733 drm_i915_private_t *dev_priv = dev->dev_private; 734 735 if (!dev->irq_enabled) 736 return false; 737 738 mtx_assert(&ring->irq_lock, MA_OWNED); 739 if (ring->irq_refcount++ == 0) { 740 if (HAS_PCH_SPLIT(dev)) 741 ironlake_enable_irq(dev_priv, 742 GT_PIPE_NOTIFY | GT_USER_INTERRUPT); 743 else 744 i915_enable_irq(dev_priv, I915_USER_INTERRUPT); 745 } 746 747 return true; 748 } 749 750 static void 751 render_ring_put_irq(struct intel_ring_buffer *ring) 752 { 753 struct drm_device *dev = ring->dev; 754 drm_i915_private_t *dev_priv = dev->dev_private; 755 756 mtx_assert(&ring->irq_lock, MA_OWNED); 757 if (--ring->irq_refcount == 0) { 758 if (HAS_PCH_SPLIT(dev)) 759 ironlake_disable_irq(dev_priv, 760 GT_USER_INTERRUPT | 761 GT_PIPE_NOTIFY); 762 else 763 i915_disable_irq(dev_priv, I915_USER_INTERRUPT); 764 } 765 } 766 767 void intel_ring_setup_status_page(struct intel_ring_buffer *ring) 768 { 769 struct drm_device *dev = ring->dev; 770 drm_i915_private_t *dev_priv = dev->dev_private; 771 uint32_t mmio = 0; 772 773 /* The ring status page addresses are no longer next to the rest of 774 * the ring registers as of gen7. 775 */ 776 if (IS_GEN7(dev)) { 777 switch (ring->id) { 778 case RCS: 779 mmio = RENDER_HWS_PGA_GEN7; 780 break; 781 case BCS: 782 mmio = BLT_HWS_PGA_GEN7; 783 break; 784 case VCS: 785 mmio = BSD_HWS_PGA_GEN7; 786 break; 787 } 788 } else if (IS_GEN6(dev)) { 789 mmio = RING_HWS_PGA_GEN6(ring->mmio_base); 790 } else { 791 mmio = RING_HWS_PGA(ring->mmio_base); 792 } 793 794 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); 795 POSTING_READ(mmio); 796 } 797 798 static int 799 bsd_ring_flush(struct intel_ring_buffer *ring, 800 uint32_t invalidate_domains, 801 uint32_t flush_domains) 802 { 803 int ret; 804 805 ret = intel_ring_begin(ring, 2); 806 if (ret) 807 return ret; 808 809 intel_ring_emit(ring, MI_FLUSH); 810 intel_ring_emit(ring, MI_NOOP); 811 intel_ring_advance(ring); 812 return 0; 813 } 814 815 static int 816 ring_add_request(struct intel_ring_buffer *ring, 817 uint32_t *result) 818 { 819 uint32_t seqno; 820 int ret; 821 822 ret = intel_ring_begin(ring, 4); 823 if (ret) 824 return ret; 825 826 seqno = i915_gem_next_request_seqno(ring); 827 828 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); 829 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 830 intel_ring_emit(ring, seqno); 831 intel_ring_emit(ring, MI_USER_INTERRUPT); 832 intel_ring_advance(ring); 833 834 *result = seqno; 835 return 0; 836 } 837 838 static bool 839 gen6_ring_get_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag) 840 { 841 struct drm_device *dev = ring->dev; 842 drm_i915_private_t *dev_priv = dev->dev_private; 843 844 if (!dev->irq_enabled) 845 return false; 846 847 gen6_gt_force_wake_get(dev_priv); 848 849 mtx_assert(&ring->irq_lock, MA_OWNED); 850 if (ring->irq_refcount++ == 0) { 851 ring->irq_mask &= ~rflag; 852 I915_WRITE_IMR(ring, ring->irq_mask); 853 ironlake_enable_irq(dev_priv, gflag); 854 } 855 856 return true; 857 } 858 859 static void 860 gen6_ring_put_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag) 861 { 862 struct drm_device *dev = ring->dev; 863 drm_i915_private_t *dev_priv = dev->dev_private; 864 865 mtx_assert(&ring->irq_lock, MA_OWNED); 866 if (--ring->irq_refcount == 0) { 867 ring->irq_mask |= rflag; 868 I915_WRITE_IMR(ring, ring->irq_mask); 869 ironlake_disable_irq(dev_priv, gflag); 870 } 871 872 gen6_gt_force_wake_put(dev_priv); 873 } 874 875 static bool 876 bsd_ring_get_irq(struct intel_ring_buffer *ring) 877 { 878 struct drm_device *dev = ring->dev; 879 drm_i915_private_t *dev_priv = dev->dev_private; 880 881 if (!dev->irq_enabled) 882 return false; 883 884 mtx_assert(&ring->irq_lock, MA_OWNED); 885 if (ring->irq_refcount++ == 0) { 886 if (IS_G4X(dev)) 887 i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT); 888 else 889 ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT); 890 } 891 892 return true; 893 } 894 static void 895 bsd_ring_put_irq(struct intel_ring_buffer *ring) 896 { 897 struct drm_device *dev = ring->dev; 898 drm_i915_private_t *dev_priv = dev->dev_private; 899 900 mtx_assert(&ring->irq_lock, MA_OWNED); 901 if (--ring->irq_refcount == 0) { 902 if (IS_G4X(dev)) 903 i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT); 904 else 905 ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT); 906 } 907 } 908 909 static int 910 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset, 911 uint32_t length) 912 { 913 int ret; 914 915 ret = intel_ring_begin(ring, 2); 916 if (ret) 917 return ret; 918 919 intel_ring_emit(ring, 920 MI_BATCH_BUFFER_START | (2 << 6) | 921 MI_BATCH_NON_SECURE_I965); 922 intel_ring_emit(ring, offset); 923 intel_ring_advance(ring); 924 925 return 0; 926 } 927 928 static int 929 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, 930 uint32_t offset, uint32_t len) 931 { 932 struct drm_device *dev = ring->dev; 933 int ret; 934 935 if (IS_I830(dev) || IS_845G(dev)) { 936 ret = intel_ring_begin(ring, 4); 937 if (ret) 938 return ret; 939 940 intel_ring_emit(ring, MI_BATCH_BUFFER); 941 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE); 942 intel_ring_emit(ring, offset + len - 8); 943 intel_ring_emit(ring, 0); 944 } else { 945 ret = intel_ring_begin(ring, 2); 946 if (ret) 947 return ret; 948 949 if (INTEL_INFO(dev)->gen >= 4) { 950 intel_ring_emit(ring, 951 MI_BATCH_BUFFER_START | (2 << 6) | 952 MI_BATCH_NON_SECURE_I965); 953 intel_ring_emit(ring, offset); 954 } else { 955 intel_ring_emit(ring, 956 MI_BATCH_BUFFER_START | (2 << 6)); 957 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE); 958 } 959 } 960 intel_ring_advance(ring); 961 962 return 0; 963 } 964 965 static void cleanup_status_page(struct intel_ring_buffer *ring) 966 { 967 drm_i915_private_t *dev_priv = ring->dev->dev_private; 968 struct drm_i915_gem_object *obj; 969 970 obj = ring->status_page.obj; 971 if (obj == NULL) 972 return; 973 974 pmap_qremove((vm_offset_t)ring->status_page.page_addr, 1); 975 pmap_invalidate_range(kernel_pmap, 976 (vm_offset_t)ring->status_page.page_addr, 977 (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE); 978 kmem_free(kernel_map, (vm_offset_t)ring->status_page.page_addr, 979 PAGE_SIZE); 980 i915_gem_object_unpin(obj); 981 drm_gem_object_unreference(&obj->base); 982 ring->status_page.obj = NULL; 983 984 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 985 } 986 987 static int init_status_page(struct intel_ring_buffer *ring) 988 { 989 struct drm_device *dev = ring->dev; 990 drm_i915_private_t *dev_priv = dev->dev_private; 991 struct drm_i915_gem_object *obj; 992 int ret; 993 994 obj = i915_gem_alloc_object(dev, 4096); 995 if (obj == NULL) { 996 DRM_ERROR("Failed to allocate status page\n"); 997 ret = -ENOMEM; 998 goto err; 999 } 1000 1001 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); 1002 1003 ret = i915_gem_object_pin(obj, 4096, true); 1004 if (ret != 0) { 1005 goto err_unref; 1006 } 1007 1008 ring->status_page.gfx_addr = obj->gtt_offset; 1009 ring->status_page.page_addr = (void *)kmem_alloc_nofault(kernel_map, 1010 PAGE_SIZE); 1011 if (ring->status_page.page_addr == NULL) { 1012 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 1013 goto err_unpin; 1014 } 1015 pmap_qenter((vm_offset_t)ring->status_page.page_addr, &obj->pages[0], 1016 1); 1017 pmap_invalidate_range(kernel_pmap, 1018 (vm_offset_t)ring->status_page.page_addr, 1019 (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE); 1020 pmap_invalidate_cache_range((vm_offset_t)ring->status_page.page_addr, 1021 (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE); 1022 ring->status_page.obj = obj; 1023 memset(ring->status_page.page_addr, 0, PAGE_SIZE); 1024 1025 intel_ring_setup_status_page(ring); 1026 DRM_DEBUG("i915: init_status_page %s hws offset: 0x%08x\n", 1027 ring->name, ring->status_page.gfx_addr); 1028 1029 return 0; 1030 1031 err_unpin: 1032 i915_gem_object_unpin(obj); 1033 err_unref: 1034 drm_gem_object_unreference(&obj->base); 1035 err: 1036 return ret; 1037 } 1038 1039 static 1040 int intel_init_ring_buffer(struct drm_device *dev, 1041 struct intel_ring_buffer *ring) 1042 { 1043 struct drm_i915_gem_object *obj; 1044 int ret; 1045 1046 ring->dev = dev; 1047 INIT_LIST_HEAD(&ring->active_list); 1048 INIT_LIST_HEAD(&ring->request_list); 1049 INIT_LIST_HEAD(&ring->gpu_write_list); 1050 1051 mtx_init(&ring->irq_lock, "ringb", NULL, MTX_DEF); 1052 ring->irq_mask = ~0; 1053 1054 if (I915_NEED_GFX_HWS(dev)) { 1055 ret = init_status_page(ring); 1056 if (ret) 1057 return ret; 1058 } 1059 1060 obj = i915_gem_alloc_object(dev, ring->size); 1061 if (obj == NULL) { 1062 DRM_ERROR("Failed to allocate ringbuffer\n"); 1063 ret = -ENOMEM; 1064 goto err_hws; 1065 } 1066 1067 ring->obj = obj; 1068 1069 ret = i915_gem_object_pin(obj, PAGE_SIZE, true); 1070 if (ret) 1071 goto err_unref; 1072 1073 ring->map.size = ring->size; 1074 ring->map.offset = dev->agp->base + obj->gtt_offset; 1075 ring->map.type = 0; 1076 ring->map.flags = 0; 1077 ring->map.mtrr = 0; 1078 1079 drm_core_ioremap_wc(&ring->map, dev); 1080 if (ring->map.virtual == NULL) { 1081 DRM_ERROR("Failed to map ringbuffer.\n"); 1082 ret = -EINVAL; 1083 goto err_unpin; 1084 } 1085 1086 ring->virtual_start = ring->map.virtual; 1087 ret = ring->init(ring); 1088 if (ret) 1089 goto err_unmap; 1090 1091 /* Workaround an erratum on the i830 which causes a hang if 1092 * the TAIL pointer points to within the last 2 cachelines 1093 * of the buffer. 1094 */ 1095 ring->effective_size = ring->size; 1096 if (IS_I830(ring->dev) || IS_845G(ring->dev)) 1097 ring->effective_size -= 128; 1098 1099 return 0; 1100 1101 err_unmap: 1102 drm_core_ioremapfree(&ring->map, dev); 1103 err_unpin: 1104 i915_gem_object_unpin(obj); 1105 err_unref: 1106 drm_gem_object_unreference(&obj->base); 1107 ring->obj = NULL; 1108 err_hws: 1109 cleanup_status_page(ring); 1110 return ret; 1111 } 1112 1113 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) 1114 { 1115 struct drm_i915_private *dev_priv; 1116 int ret; 1117 1118 if (ring->obj == NULL) 1119 return; 1120 1121 /* Disable the ring buffer. The ring must be idle at this point */ 1122 dev_priv = ring->dev->dev_private; 1123 ret = intel_wait_ring_idle(ring); 1124 I915_WRITE_CTL(ring, 0); 1125 1126 drm_core_ioremapfree(&ring->map, ring->dev); 1127 1128 i915_gem_object_unpin(ring->obj); 1129 drm_gem_object_unreference(&ring->obj->base); 1130 ring->obj = NULL; 1131 1132 if (ring->cleanup) 1133 ring->cleanup(ring); 1134 1135 cleanup_status_page(ring); 1136 } 1137 1138 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring) 1139 { 1140 unsigned int *virt; 1141 int rem = ring->size - ring->tail; 1142 1143 if (ring->space < rem) { 1144 int ret = intel_wait_ring_buffer(ring, rem); 1145 if (ret) 1146 return ret; 1147 } 1148 1149 virt = (unsigned int *)((char *)ring->virtual_start + ring->tail); 1150 rem /= 8; 1151 while (rem--) { 1152 *virt++ = MI_NOOP; 1153 *virt++ = MI_NOOP; 1154 } 1155 1156 ring->tail = 0; 1157 ring->space = ring_space(ring); 1158 1159 return 0; 1160 } 1161 1162 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno) 1163 { 1164 struct drm_i915_private *dev_priv = ring->dev->dev_private; 1165 bool was_interruptible; 1166 int ret; 1167 1168 /* XXX As we have not yet audited all the paths to check that 1169 * they are ready for ERESTARTSYS from intel_ring_begin, do not 1170 * allow us to be interruptible by a signal. 1171 */ 1172 was_interruptible = dev_priv->mm.interruptible; 1173 dev_priv->mm.interruptible = false; 1174 1175 ret = i915_wait_request(ring, seqno, true); 1176 1177 dev_priv->mm.interruptible = was_interruptible; 1178 1179 return ret; 1180 } 1181 1182 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n) 1183 { 1184 struct drm_i915_gem_request *request; 1185 u32 seqno = 0; 1186 int ret; 1187 1188 i915_gem_retire_requests_ring(ring); 1189 1190 if (ring->last_retired_head != -1) { 1191 ring->head = ring->last_retired_head; 1192 ring->last_retired_head = -1; 1193 ring->space = ring_space(ring); 1194 if (ring->space >= n) 1195 return 0; 1196 } 1197 1198 list_for_each_entry(request, &ring->request_list, list) { 1199 int space; 1200 1201 if (request->tail == -1) 1202 continue; 1203 1204 space = request->tail - (ring->tail + 8); 1205 if (space < 0) 1206 space += ring->size; 1207 if (space >= n) { 1208 seqno = request->seqno; 1209 break; 1210 } 1211 1212 /* Consume this request in case we need more space than 1213 * is available and so need to prevent a race between 1214 * updating last_retired_head and direct reads of 1215 * I915_RING_HEAD. It also provides a nice sanity check. 1216 */ 1217 request->tail = -1; 1218 } 1219 1220 if (seqno == 0) 1221 return -ENOSPC; 1222 1223 ret = intel_ring_wait_seqno(ring, seqno); 1224 if (ret) 1225 return ret; 1226 1227 if (ring->last_retired_head == -1) 1228 return -ENOSPC; 1229 1230 ring->head = ring->last_retired_head; 1231 ring->last_retired_head = -1; 1232 ring->space = ring_space(ring); 1233 if (ring->space < n) 1234 return -ENOSPC; 1235 1236 return 0; 1237 } 1238 1239 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n) 1240 { 1241 struct drm_device *dev = ring->dev; 1242 struct drm_i915_private *dev_priv = dev->dev_private; 1243 int end; 1244 int ret; 1245 1246 ret = intel_ring_wait_request(ring, n); 1247 if (ret != -ENOSPC) 1248 return ret; 1249 1250 CTR1(KTR_DRM, "ring_wait_begin %s", ring->name); 1251 if (drm_core_check_feature(dev, DRIVER_GEM)) 1252 /* With GEM the hangcheck timer should kick us out of the loop, 1253 * leaving it early runs the risk of corrupting GEM state (due 1254 * to running on almost untested codepaths). But on resume 1255 * timers don't work yet, so prevent a complete hang in that 1256 * case by choosing an insanely large timeout. */ 1257 end = ticks + hz * 60; 1258 else 1259 end = ticks + hz * 3; 1260 do { 1261 ring->head = I915_READ_HEAD(ring); 1262 ring->space = ring_space(ring); 1263 if (ring->space >= n) { 1264 CTR1(KTR_DRM, "ring_wait_end %s", ring->name); 1265 return 0; 1266 } 1267 1268 #if 0 1269 if (dev->primary->master) { 1270 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; 1271 if (master_priv->sarea_priv) 1272 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; 1273 } 1274 #else 1275 if (dev_priv->sarea_priv) 1276 dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; 1277 #endif 1278 1279 pause("915rng", 1); 1280 if (atomic_load_acq_32(&dev_priv->mm.wedged) != 0) { 1281 CTR1(KTR_DRM, "ring_wait_end %s wedged", ring->name); 1282 return -EAGAIN; 1283 } 1284 } while (!time_after(ticks, end)); 1285 CTR1(KTR_DRM, "ring_wait_end %s busy", ring->name); 1286 return -EBUSY; 1287 } 1288 1289 int intel_ring_begin(struct intel_ring_buffer *ring, 1290 int num_dwords) 1291 { 1292 struct drm_i915_private *dev_priv = ring->dev->dev_private; 1293 int n = 4*num_dwords; 1294 int ret; 1295 1296 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 1297 return -EIO; 1298 1299 if (ring->tail + n > ring->effective_size) { 1300 ret = intel_wrap_ring_buffer(ring); 1301 if (ret != 0) 1302 return ret; 1303 } 1304 1305 if (ring->space < n) { 1306 ret = intel_wait_ring_buffer(ring, n); 1307 if (ret != 0) 1308 return ret; 1309 } 1310 1311 ring->space -= n; 1312 return 0; 1313 } 1314 1315 void intel_ring_advance(struct intel_ring_buffer *ring) 1316 { 1317 ring->tail &= ring->size - 1; 1318 ring->write_tail(ring, ring->tail); 1319 } 1320 1321 static const struct intel_ring_buffer render_ring = { 1322 .name = "render ring", 1323 .id = RCS, 1324 .mmio_base = RENDER_RING_BASE, 1325 .size = 32 * PAGE_SIZE, 1326 .init = init_render_ring, 1327 .write_tail = ring_write_tail, 1328 .flush = render_ring_flush, 1329 .add_request = render_ring_add_request, 1330 .get_seqno = ring_get_seqno, 1331 .irq_get = render_ring_get_irq, 1332 .irq_put = render_ring_put_irq, 1333 .dispatch_execbuffer = render_ring_dispatch_execbuffer, 1334 .cleanup = render_ring_cleanup, 1335 .sync_to = render_ring_sync_to, 1336 .semaphore_register = {MI_SEMAPHORE_SYNC_INVALID, 1337 MI_SEMAPHORE_SYNC_RV, 1338 MI_SEMAPHORE_SYNC_RB}, 1339 .signal_mbox = {GEN6_VRSYNC, GEN6_BRSYNC}, 1340 }; 1341 1342 /* ring buffer for bit-stream decoder */ 1343 1344 static const struct intel_ring_buffer bsd_ring = { 1345 .name = "bsd ring", 1346 .id = VCS, 1347 .mmio_base = BSD_RING_BASE, 1348 .size = 32 * PAGE_SIZE, 1349 .init = init_ring_common, 1350 .write_tail = ring_write_tail, 1351 .flush = bsd_ring_flush, 1352 .add_request = ring_add_request, 1353 .get_seqno = ring_get_seqno, 1354 .irq_get = bsd_ring_get_irq, 1355 .irq_put = bsd_ring_put_irq, 1356 .dispatch_execbuffer = ring_dispatch_execbuffer, 1357 }; 1358 1359 1360 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, 1361 uint32_t value) 1362 { 1363 drm_i915_private_t *dev_priv = ring->dev->dev_private; 1364 1365 /* Every tail move must follow the sequence below */ 1366 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 1367 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK | 1368 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE); 1369 I915_WRITE(GEN6_BSD_RNCID, 0x0); 1370 1371 if (_intel_wait_for(ring->dev, 1372 (I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) & 1373 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0, 50, 1374 true, "915g6i") != 0) 1375 DRM_ERROR("timed out waiting for IDLE Indicator\n"); 1376 1377 I915_WRITE_TAIL(ring, value); 1378 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 1379 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK | 1380 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE); 1381 } 1382 1383 static int gen6_ring_flush(struct intel_ring_buffer *ring, 1384 uint32_t invalidate, uint32_t flush) 1385 { 1386 uint32_t cmd; 1387 int ret; 1388 1389 ret = intel_ring_begin(ring, 4); 1390 if (ret) 1391 return ret; 1392 1393 cmd = MI_FLUSH_DW; 1394 if (invalidate & I915_GEM_GPU_DOMAINS) 1395 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD; 1396 intel_ring_emit(ring, cmd); 1397 intel_ring_emit(ring, 0); 1398 intel_ring_emit(ring, 0); 1399 intel_ring_emit(ring, MI_NOOP); 1400 intel_ring_advance(ring); 1401 return 0; 1402 } 1403 1404 static int 1405 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, 1406 uint32_t offset, uint32_t len) 1407 { 1408 int ret; 1409 1410 ret = intel_ring_begin(ring, 2); 1411 if (ret) 1412 return ret; 1413 1414 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965); 1415 /* bit0-7 is the length on GEN6+ */ 1416 intel_ring_emit(ring, offset); 1417 intel_ring_advance(ring); 1418 1419 return 0; 1420 } 1421 1422 static bool 1423 gen6_render_ring_get_irq(struct intel_ring_buffer *ring) 1424 { 1425 return gen6_ring_get_irq(ring, 1426 GT_USER_INTERRUPT, 1427 GEN6_RENDER_USER_INTERRUPT); 1428 } 1429 1430 static void 1431 gen6_render_ring_put_irq(struct intel_ring_buffer *ring) 1432 { 1433 return gen6_ring_put_irq(ring, 1434 GT_USER_INTERRUPT, 1435 GEN6_RENDER_USER_INTERRUPT); 1436 } 1437 1438 static bool 1439 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring) 1440 { 1441 return gen6_ring_get_irq(ring, 1442 GT_GEN6_BSD_USER_INTERRUPT, 1443 GEN6_BSD_USER_INTERRUPT); 1444 } 1445 1446 static void 1447 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring) 1448 { 1449 return gen6_ring_put_irq(ring, 1450 GT_GEN6_BSD_USER_INTERRUPT, 1451 GEN6_BSD_USER_INTERRUPT); 1452 } 1453 1454 /* ring buffer for Video Codec for Gen6+ */ 1455 static const struct intel_ring_buffer gen6_bsd_ring = { 1456 .name = "gen6 bsd ring", 1457 .id = VCS, 1458 .mmio_base = GEN6_BSD_RING_BASE, 1459 .size = 32 * PAGE_SIZE, 1460 .init = init_ring_common, 1461 .write_tail = gen6_bsd_ring_write_tail, 1462 .flush = gen6_ring_flush, 1463 .add_request = gen6_add_request, 1464 .get_seqno = gen6_ring_get_seqno, 1465 .irq_get = gen6_bsd_ring_get_irq, 1466 .irq_put = gen6_bsd_ring_put_irq, 1467 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer, 1468 .sync_to = gen6_bsd_ring_sync_to, 1469 .semaphore_register = {MI_SEMAPHORE_SYNC_VR, 1470 MI_SEMAPHORE_SYNC_INVALID, 1471 MI_SEMAPHORE_SYNC_VB}, 1472 .signal_mbox = {GEN6_RVSYNC, GEN6_BVSYNC}, 1473 }; 1474 1475 /* Blitter support (SandyBridge+) */ 1476 1477 static bool 1478 blt_ring_get_irq(struct intel_ring_buffer *ring) 1479 { 1480 return gen6_ring_get_irq(ring, 1481 GT_BLT_USER_INTERRUPT, 1482 GEN6_BLITTER_USER_INTERRUPT); 1483 } 1484 1485 static void 1486 blt_ring_put_irq(struct intel_ring_buffer *ring) 1487 { 1488 gen6_ring_put_irq(ring, 1489 GT_BLT_USER_INTERRUPT, 1490 GEN6_BLITTER_USER_INTERRUPT); 1491 } 1492 1493 static int blt_ring_flush(struct intel_ring_buffer *ring, 1494 uint32_t invalidate, uint32_t flush) 1495 { 1496 uint32_t cmd; 1497 int ret; 1498 1499 ret = intel_ring_begin(ring, 4); 1500 if (ret) 1501 return ret; 1502 1503 cmd = MI_FLUSH_DW; 1504 if (invalidate & I915_GEM_DOMAIN_RENDER) 1505 cmd |= MI_INVALIDATE_TLB; 1506 intel_ring_emit(ring, cmd); 1507 intel_ring_emit(ring, 0); 1508 intel_ring_emit(ring, 0); 1509 intel_ring_emit(ring, MI_NOOP); 1510 intel_ring_advance(ring); 1511 return 0; 1512 } 1513 1514 static const struct intel_ring_buffer gen6_blt_ring = { 1515 .name = "blt ring", 1516 .id = BCS, 1517 .mmio_base = BLT_RING_BASE, 1518 .size = 32 * PAGE_SIZE, 1519 .init = init_ring_common, 1520 .write_tail = ring_write_tail, 1521 .flush = blt_ring_flush, 1522 .add_request = gen6_add_request, 1523 .get_seqno = gen6_ring_get_seqno, 1524 .irq_get = blt_ring_get_irq, 1525 .irq_put = blt_ring_put_irq, 1526 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer, 1527 .sync_to = gen6_blt_ring_sync_to, 1528 .semaphore_register = {MI_SEMAPHORE_SYNC_BR, 1529 MI_SEMAPHORE_SYNC_BV, 1530 MI_SEMAPHORE_SYNC_INVALID}, 1531 .signal_mbox = {GEN6_RBSYNC, GEN6_VBSYNC}, 1532 }; 1533 1534 int intel_init_render_ring_buffer(struct drm_device *dev) 1535 { 1536 drm_i915_private_t *dev_priv = dev->dev_private; 1537 struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; 1538 1539 *ring = render_ring; 1540 if (INTEL_INFO(dev)->gen >= 6) { 1541 ring->add_request = gen6_add_request; 1542 ring->flush = gen6_render_ring_flush; 1543 ring->irq_get = gen6_render_ring_get_irq; 1544 ring->irq_put = gen6_render_ring_put_irq; 1545 ring->get_seqno = gen6_ring_get_seqno; 1546 } else if (IS_GEN5(dev)) { 1547 ring->add_request = pc_render_add_request; 1548 ring->get_seqno = pc_render_get_seqno; 1549 } 1550 1551 if (!I915_NEED_GFX_HWS(dev)) { 1552 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; 1553 memset(ring->status_page.page_addr, 0, PAGE_SIZE); 1554 } 1555 1556 return intel_init_ring_buffer(dev, ring); 1557 } 1558 1559 int intel_render_ring_init_dri(struct drm_device *dev, uint64_t start, 1560 uint32_t size) 1561 { 1562 drm_i915_private_t *dev_priv = dev->dev_private; 1563 struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; 1564 1565 *ring = render_ring; 1566 if (INTEL_INFO(dev)->gen >= 6) { 1567 ring->add_request = gen6_add_request; 1568 ring->irq_get = gen6_render_ring_get_irq; 1569 ring->irq_put = gen6_render_ring_put_irq; 1570 } else if (IS_GEN5(dev)) { 1571 ring->add_request = pc_render_add_request; 1572 ring->get_seqno = pc_render_get_seqno; 1573 } 1574 1575 ring->dev = dev; 1576 INIT_LIST_HEAD(&ring->active_list); 1577 INIT_LIST_HEAD(&ring->request_list); 1578 INIT_LIST_HEAD(&ring->gpu_write_list); 1579 1580 ring->size = size; 1581 ring->effective_size = ring->size; 1582 if (IS_I830(ring->dev)) 1583 ring->effective_size -= 128; 1584 1585 ring->map.offset = start; 1586 ring->map.size = size; 1587 ring->map.type = 0; 1588 ring->map.flags = 0; 1589 ring->map.mtrr = 0; 1590 1591 drm_core_ioremap_wc(&ring->map, dev); 1592 if (ring->map.virtual == NULL) { 1593 DRM_ERROR("can not ioremap virtual address for" 1594 " ring buffer\n"); 1595 return -ENOMEM; 1596 } 1597 1598 ring->virtual_start = (void *)ring->map.virtual; 1599 return 0; 1600 } 1601 1602 int intel_init_bsd_ring_buffer(struct drm_device *dev) 1603 { 1604 drm_i915_private_t *dev_priv = dev->dev_private; 1605 struct intel_ring_buffer *ring = &dev_priv->rings[VCS]; 1606 1607 if (IS_GEN6(dev) || IS_GEN7(dev)) 1608 *ring = gen6_bsd_ring; 1609 else 1610 *ring = bsd_ring; 1611 1612 return intel_init_ring_buffer(dev, ring); 1613 } 1614 1615 int intel_init_blt_ring_buffer(struct drm_device *dev) 1616 { 1617 drm_i915_private_t *dev_priv = dev->dev_private; 1618 struct intel_ring_buffer *ring = &dev_priv->rings[BCS]; 1619 1620 *ring = gen6_blt_ring; 1621 1622 return intel_init_ring_buffer(dev, ring); 1623 } 1624