Lines Matching refs:ring
50 static inline int ring_space(struct intel_ring_buffer *ring) in ring_space() argument
52 int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE); in ring_space()
54 space += ring->size; in ring_space()
59 gen2_render_ring_flush(struct intel_ring_buffer *ring, in gen2_render_ring_flush() argument
73 ret = intel_ring_begin(ring, 2); in gen2_render_ring_flush()
77 intel_ring_emit(ring, cmd); in gen2_render_ring_flush()
78 intel_ring_emit(ring, MI_NOOP); in gen2_render_ring_flush()
79 intel_ring_advance(ring); in gen2_render_ring_flush()
85 gen4_render_ring_flush(struct intel_ring_buffer *ring, in gen4_render_ring_flush() argument
89 struct drm_device *dev = ring->dev; in gen4_render_ring_flush()
131 ret = intel_ring_begin(ring, 2); in gen4_render_ring_flush()
135 intel_ring_emit(ring, cmd); in gen4_render_ring_flush()
136 intel_ring_emit(ring, MI_NOOP); in gen4_render_ring_flush()
137 intel_ring_advance(ring); in gen4_render_ring_flush()
180 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring) in intel_emit_post_sync_nonzero_flush() argument
182 struct pipe_control *pc = ring->private; in intel_emit_post_sync_nonzero_flush()
187 ret = intel_ring_begin(ring, 6); in intel_emit_post_sync_nonzero_flush()
191 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
192 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in intel_emit_post_sync_nonzero_flush()
194 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
195 intel_ring_emit(ring, 0); /* low dword */ in intel_emit_post_sync_nonzero_flush()
196 intel_ring_emit(ring, 0); /* high dword */ in intel_emit_post_sync_nonzero_flush()
197 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
198 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
200 ret = intel_ring_begin(ring, 6); in intel_emit_post_sync_nonzero_flush()
204 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
205 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); in intel_emit_post_sync_nonzero_flush()
206 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
207 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
208 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
209 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
210 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
216 gen6_render_ring_flush(struct intel_ring_buffer *ring, in gen6_render_ring_flush() argument
220 struct pipe_control *pc = ring->private; in gen6_render_ring_flush()
225 ret = intel_emit_post_sync_nonzero_flush(ring); in gen6_render_ring_flush()
255 ret = intel_ring_begin(ring, 4); in gen6_render_ring_flush()
259 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen6_render_ring_flush()
260 intel_ring_emit(ring, flags); in gen6_render_ring_flush()
261 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); in gen6_render_ring_flush()
262 intel_ring_emit(ring, 0); in gen6_render_ring_flush()
263 intel_ring_advance(ring); in gen6_render_ring_flush()
269 gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring) in gen7_render_ring_cs_stall_wa() argument
273 ret = intel_ring_begin(ring, 4); in gen7_render_ring_cs_stall_wa()
277 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_cs_stall_wa()
278 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in gen7_render_ring_cs_stall_wa()
280 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
281 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
282 intel_ring_advance(ring); in gen7_render_ring_cs_stall_wa()
288 gen7_render_ring_flush(struct intel_ring_buffer *ring, in gen7_render_ring_flush() argument
292 struct pipe_control *pc = ring->private; in gen7_render_ring_flush()
329 gen7_render_ring_cs_stall_wa(ring); in gen7_render_ring_flush()
332 ret = intel_ring_begin(ring, 4); in gen7_render_ring_flush()
336 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_flush()
337 intel_ring_emit(ring, flags); in gen7_render_ring_flush()
338 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); in gen7_render_ring_flush()
339 intel_ring_emit(ring, 0); in gen7_render_ring_flush()
340 intel_ring_advance(ring); in gen7_render_ring_flush()
345 static void ring_write_tail(struct intel_ring_buffer *ring, in ring_write_tail() argument
348 drm_i915_private_t *dev_priv = ring->dev->dev_private; in ring_write_tail()
349 I915_WRITE_TAIL(ring, value); in ring_write_tail()
352 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring) in intel_ring_get_active_head() argument
354 drm_i915_private_t *dev_priv = ring->dev->dev_private; in intel_ring_get_active_head()
355 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ? in intel_ring_get_active_head()
356 RING_ACTHD(ring->mmio_base) : ACTHD; in intel_ring_get_active_head()
361 static int init_ring_common(struct intel_ring_buffer *ring) in init_ring_common() argument
363 struct drm_device *dev = ring->dev; in init_ring_common()
365 struct drm_i915_gem_object *obj = ring->obj; in init_ring_common()
373 I915_WRITE_CTL(ring, 0); in init_ring_common()
374 I915_WRITE_HEAD(ring, 0); in init_ring_common()
375 ring->write_tail(ring, 0); in init_ring_common()
377 head = I915_READ_HEAD(ring) & HEAD_ADDR; in init_ring_common()
383 ring->name, in init_ring_common()
384 I915_READ_CTL(ring), in init_ring_common()
385 I915_READ_HEAD(ring), in init_ring_common()
386 I915_READ_TAIL(ring), in init_ring_common()
387 I915_READ_START(ring)); in init_ring_common()
389 I915_WRITE_HEAD(ring, 0); in init_ring_common()
391 if (I915_READ_HEAD(ring) & HEAD_ADDR) { in init_ring_common()
394 ring->name, in init_ring_common()
395 I915_READ_CTL(ring), in init_ring_common()
396 I915_READ_HEAD(ring), in init_ring_common()
397 I915_READ_TAIL(ring), in init_ring_common()
398 I915_READ_START(ring)); in init_ring_common()
406 I915_WRITE_START(ring, obj->gtt_offset); in init_ring_common()
407 I915_WRITE_CTL(ring, in init_ring_common()
408 ((ring->size - PAGE_SIZE) & RING_NR_PAGES) in init_ring_common()
412 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 && in init_ring_common()
413 I915_READ_START(ring) == obj->gtt_offset && in init_ring_common()
414 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { in init_ring_common()
417 ring->name, in init_ring_common()
418 I915_READ_CTL(ring), in init_ring_common()
419 I915_READ_HEAD(ring), in init_ring_common()
420 I915_READ_TAIL(ring), in init_ring_common()
421 I915_READ_START(ring)); in init_ring_common()
426 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) in init_ring_common()
427 i915_kernel_lost_context(ring->dev); in init_ring_common()
429 ring->head = I915_READ_HEAD(ring); in init_ring_common()
430 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; in init_ring_common()
431 ring->space = ring_space(ring); in init_ring_common()
432 ring->last_retired_head = -1; in init_ring_common()
443 init_pipe_control(struct intel_ring_buffer *ring) in init_pipe_control() argument
449 if (ring->private) in init_pipe_control()
456 obj = i915_gem_alloc_object(ring->dev, 4096); in init_pipe_control()
478 ring->private = pc; in init_pipe_control()
491 cleanup_pipe_control(struct intel_ring_buffer *ring) in cleanup_pipe_control() argument
493 struct pipe_control *pc = ring->private; in cleanup_pipe_control()
496 if (!ring->private) in cleanup_pipe_control()
507 ring->private = NULL; in cleanup_pipe_control()
510 static int init_render_ring(struct intel_ring_buffer *ring) in init_render_ring() argument
512 struct drm_device *dev = ring->dev; in init_render_ring()
514 int ret = init_ring_common(ring); in init_render_ring()
537 ret = init_pipe_control(ring); in init_render_ring()
555 ring->itlb_before_ctx_switch = in init_render_ring()
563 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR); in init_render_ring()
568 static void render_ring_cleanup(struct intel_ring_buffer *ring) in render_ring_cleanup() argument
570 struct drm_device *dev = ring->dev; in render_ring_cleanup()
572 if (!ring->private) in render_ring_cleanup()
576 drm_gem_object_unreference(to_gem_object(ring->private)); in render_ring_cleanup()
578 cleanup_pipe_control(ring); in render_ring_cleanup()
582 update_mboxes(struct intel_ring_buffer *ring, in update_mboxes() argument
585 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); in update_mboxes()
586 intel_ring_emit(ring, mmio_offset); in update_mboxes()
587 intel_ring_emit(ring, ring->outstanding_lazy_request); in update_mboxes()
600 gen6_add_request(struct intel_ring_buffer *ring) in gen6_add_request() argument
606 ret = intel_ring_begin(ring, 10); in gen6_add_request()
610 mbox1_reg = ring->signal_mbox[0]; in gen6_add_request()
611 mbox2_reg = ring->signal_mbox[1]; in gen6_add_request()
613 update_mboxes(ring, mbox1_reg); in gen6_add_request()
614 update_mboxes(ring, mbox2_reg); in gen6_add_request()
615 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in gen6_add_request()
616 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in gen6_add_request()
617 intel_ring_emit(ring, ring->outstanding_lazy_request); in gen6_add_request()
618 intel_ring_emit(ring, MI_USER_INTERRUPT); in gen6_add_request()
619 intel_ring_advance(ring); in gen6_add_request()
674 pc_render_add_request(struct intel_ring_buffer *ring) in pc_render_add_request() argument
676 struct pipe_control *pc = ring->private; in pc_render_add_request()
688 ret = intel_ring_begin(ring, 32); in pc_render_add_request()
692 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
695 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
696 intel_ring_emit(ring, ring->outstanding_lazy_request); in pc_render_add_request()
697 intel_ring_emit(ring, 0); in pc_render_add_request()
698 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
700 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
702 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
704 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
706 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
708 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
710 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
714 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
715 intel_ring_emit(ring, ring->outstanding_lazy_request); in pc_render_add_request()
716 intel_ring_emit(ring, 0); in pc_render_add_request()
717 intel_ring_advance(ring); in pc_render_add_request()
723 gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) in gen6_ring_get_seqno() argument
729 intel_ring_get_active_head(ring); in gen6_ring_get_seqno()
730 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in gen6_ring_get_seqno()
734 ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) in ring_get_seqno() argument
736 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in ring_get_seqno()
740 pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) in pc_render_get_seqno() argument
742 struct pipe_control *pc = ring->private; in pc_render_get_seqno()
747 gen5_ring_get_irq(struct intel_ring_buffer *ring) in gen5_ring_get_irq() argument
749 struct drm_device *dev = ring->dev; in gen5_ring_get_irq()
756 if (ring->irq_refcount++ == 0) { in gen5_ring_get_irq()
757 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask; in gen5_ring_get_irq()
767 gen5_ring_put_irq(struct intel_ring_buffer *ring) in gen5_ring_put_irq() argument
769 struct drm_device *dev = ring->dev; in gen5_ring_put_irq()
773 if (--ring->irq_refcount == 0) { in gen5_ring_put_irq()
774 dev_priv->gt_irq_mask |= ring->irq_enable_mask; in gen5_ring_put_irq()
782 i9xx_ring_get_irq(struct intel_ring_buffer *ring) in i9xx_ring_get_irq() argument
784 struct drm_device *dev = ring->dev; in i9xx_ring_get_irq()
791 if (ring->irq_refcount++ == 0) { in i9xx_ring_get_irq()
792 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i9xx_ring_get_irq()
802 i9xx_ring_put_irq(struct intel_ring_buffer *ring) in i9xx_ring_put_irq() argument
804 struct drm_device *dev = ring->dev; in i9xx_ring_put_irq()
808 if (--ring->irq_refcount == 0) { in i9xx_ring_put_irq()
809 dev_priv->irq_mask |= ring->irq_enable_mask; in i9xx_ring_put_irq()
817 i8xx_ring_get_irq(struct intel_ring_buffer *ring) in i8xx_ring_get_irq() argument
819 struct drm_device *dev = ring->dev; in i8xx_ring_get_irq()
826 if (ring->irq_refcount++ == 0) { in i8xx_ring_get_irq()
827 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i8xx_ring_get_irq()
837 i8xx_ring_put_irq(struct intel_ring_buffer *ring) in i8xx_ring_put_irq() argument
839 struct drm_device *dev = ring->dev; in i8xx_ring_put_irq()
843 if (--ring->irq_refcount == 0) { in i8xx_ring_put_irq()
844 dev_priv->irq_mask |= ring->irq_enable_mask; in i8xx_ring_put_irq()
851 void intel_ring_setup_status_page(struct intel_ring_buffer *ring) in intel_ring_setup_status_page() argument
853 struct drm_device *dev = ring->dev; in intel_ring_setup_status_page()
854 drm_i915_private_t *dev_priv = ring->dev->dev_private; in intel_ring_setup_status_page()
861 switch (ring->id) { in intel_ring_setup_status_page()
872 } else if (IS_GEN6(ring->dev)) { in intel_ring_setup_status_page()
873 mmio = RING_HWS_PGA_GEN6(ring->mmio_base); in intel_ring_setup_status_page()
875 mmio = RING_HWS_PGA(ring->mmio_base); in intel_ring_setup_status_page()
878 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); in intel_ring_setup_status_page()
883 bsd_ring_flush(struct intel_ring_buffer *ring, in bsd_ring_flush() argument
889 ret = intel_ring_begin(ring, 2); in bsd_ring_flush()
893 intel_ring_emit(ring, MI_FLUSH); in bsd_ring_flush()
894 intel_ring_emit(ring, MI_NOOP); in bsd_ring_flush()
895 intel_ring_advance(ring); in bsd_ring_flush()
900 i9xx_add_request(struct intel_ring_buffer *ring) in i9xx_add_request() argument
904 ret = intel_ring_begin(ring, 4); in i9xx_add_request()
908 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in i9xx_add_request()
909 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in i9xx_add_request()
910 intel_ring_emit(ring, ring->outstanding_lazy_request); in i9xx_add_request()
911 intel_ring_emit(ring, MI_USER_INTERRUPT); in i9xx_add_request()
912 intel_ring_advance(ring); in i9xx_add_request()
918 gen6_ring_get_irq(struct intel_ring_buffer *ring) in gen6_ring_get_irq() argument
920 struct drm_device *dev = ring->dev; in gen6_ring_get_irq()
932 if (ring->irq_refcount++ == 0) { in gen6_ring_get_irq()
933 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS) in gen6_ring_get_irq()
934 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | in gen6_ring_get_irq()
937 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen6_ring_get_irq()
938 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask; in gen6_ring_get_irq()
948 gen6_ring_put_irq(struct intel_ring_buffer *ring) in gen6_ring_put_irq() argument
950 struct drm_device *dev = ring->dev; in gen6_ring_put_irq()
954 if (--ring->irq_refcount == 0) { in gen6_ring_put_irq()
955 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS) in gen6_ring_put_irq()
956 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR); in gen6_ring_put_irq()
958 I915_WRITE_IMR(ring, ~0); in gen6_ring_put_irq()
959 dev_priv->gt_irq_mask |= ring->irq_enable_mask; in gen6_ring_put_irq()
969 i965_dispatch_execbuffer(struct intel_ring_buffer *ring, in i965_dispatch_execbuffer() argument
975 ret = intel_ring_begin(ring, 2); in i965_dispatch_execbuffer()
979 intel_ring_emit(ring, in i965_dispatch_execbuffer()
983 intel_ring_emit(ring, offset); in i965_dispatch_execbuffer()
984 intel_ring_advance(ring); in i965_dispatch_execbuffer()
992 i830_dispatch_execbuffer(struct intel_ring_buffer *ring, in i830_dispatch_execbuffer() argument
999 ret = intel_ring_begin(ring, 4); in i830_dispatch_execbuffer()
1003 intel_ring_emit(ring, MI_BATCH_BUFFER); in i830_dispatch_execbuffer()
1004 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); in i830_dispatch_execbuffer()
1005 intel_ring_emit(ring, offset + len - 8); in i830_dispatch_execbuffer()
1006 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1007 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1009 struct drm_i915_gem_object *obj = ring->private; in i830_dispatch_execbuffer()
1015 ret = intel_ring_begin(ring, 9+3); in i830_dispatch_execbuffer()
1021 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD | in i830_dispatch_execbuffer()
1024 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096); in i830_dispatch_execbuffer()
1025 intel_ring_emit(ring, 0); in i830_dispatch_execbuffer()
1026 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024); in i830_dispatch_execbuffer()
1027 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1028 intel_ring_emit(ring, 0); in i830_dispatch_execbuffer()
1029 intel_ring_emit(ring, 4096); in i830_dispatch_execbuffer()
1030 intel_ring_emit(ring, offset); in i830_dispatch_execbuffer()
1031 intel_ring_emit(ring, MI_FLUSH); in i830_dispatch_execbuffer()
1034 intel_ring_emit(ring, MI_BATCH_BUFFER); in i830_dispatch_execbuffer()
1035 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); in i830_dispatch_execbuffer()
1036 intel_ring_emit(ring, cs_offset + len - 8); in i830_dispatch_execbuffer()
1037 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1044 i915_dispatch_execbuffer(struct intel_ring_buffer *ring, in i915_dispatch_execbuffer() argument
1050 ret = intel_ring_begin(ring, 2); in i915_dispatch_execbuffer()
1054 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT); in i915_dispatch_execbuffer()
1055 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); in i915_dispatch_execbuffer()
1056 intel_ring_advance(ring); in i915_dispatch_execbuffer()
1061 static void cleanup_status_page(struct intel_ring_buffer *ring) in cleanup_status_page() argument
1065 obj = ring->status_page.obj; in cleanup_status_page()
1069 pmap_qremove((vm_offset_t)ring->status_page.page_addr, 1); in cleanup_status_page()
1070 kva_free((vm_offset_t)ring->status_page.page_addr, in cleanup_status_page()
1074 ring->status_page.obj = NULL; in cleanup_status_page()
1077 static int init_status_page(struct intel_ring_buffer *ring) in init_status_page() argument
1079 struct drm_device *dev = ring->dev; in init_status_page()
1097 ring->status_page.gfx_addr = obj->gtt_offset; in init_status_page()
1098 ring->status_page.page_addr = (void *)kva_alloc(PAGE_SIZE); in init_status_page()
1099 if (ring->status_page.page_addr == NULL) { in init_status_page()
1103 pmap_qenter((vm_offset_t)ring->status_page.page_addr, &obj->pages[0], in init_status_page()
1106 (vm_offset_t)ring->status_page.page_addr, in init_status_page()
1107 (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE); in init_status_page()
1108 ring->status_page.obj = obj; in init_status_page()
1109 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_status_page()
1111 intel_ring_setup_status_page(ring); in init_status_page()
1113 ring->name, ring->status_page.gfx_addr); in init_status_page()
1125 static int init_phys_hws_pga(struct intel_ring_buffer *ring) in init_phys_hws_pga() argument
1127 struct drm_i915_private *dev_priv = ring->dev->dev_private; in init_phys_hws_pga()
1132 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE, BUS_SPACE_MAXADDR); in init_phys_hws_pga()
1138 if (INTEL_INFO(ring->dev)->gen >= 4) in init_phys_hws_pga()
1142 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; in init_phys_hws_pga()
1143 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_phys_hws_pga()
1149 struct intel_ring_buffer *ring) in intel_init_ring_buffer() argument
1155 ring->dev = dev; in intel_init_ring_buffer()
1156 INIT_LIST_HEAD(&ring->active_list); in intel_init_ring_buffer()
1157 INIT_LIST_HEAD(&ring->request_list); in intel_init_ring_buffer()
1158 ring->size = 32 * PAGE_SIZE; in intel_init_ring_buffer()
1159 memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno)); in intel_init_ring_buffer()
1162 init_waitqueue_head(&ring->irq_queue); in intel_init_ring_buffer()
1166 ret = init_status_page(ring); in intel_init_ring_buffer()
1170 BUG_ON(ring->id != RCS); in intel_init_ring_buffer()
1171 ret = init_phys_hws_pga(ring); in intel_init_ring_buffer()
1176 obj = i915_gem_alloc_object(dev, ring->size); in intel_init_ring_buffer()
1183 ring->obj = obj; in intel_init_ring_buffer()
1193 ring->virtual_start = in intel_init_ring_buffer()
1195 dev_priv->mm.gtt->gma_bus_addr + obj->gtt_offset, ring->size, in intel_init_ring_buffer()
1197 if (ring->virtual_start == NULL) { in intel_init_ring_buffer()
1203 ret = ring->init(ring); in intel_init_ring_buffer()
1211 ring->effective_size = ring->size; in intel_init_ring_buffer()
1212 if (IS_I830(ring->dev) || IS_845G(ring->dev)) in intel_init_ring_buffer()
1213 ring->effective_size -= 128; in intel_init_ring_buffer()
1218 pmap_unmapdev((vm_offset_t)ring->virtual_start, ring->size); in intel_init_ring_buffer()
1223 ring->obj = NULL; in intel_init_ring_buffer()
1225 cleanup_status_page(ring); in intel_init_ring_buffer()
1229 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) in intel_cleanup_ring_buffer() argument
1234 if (ring->obj == NULL) in intel_cleanup_ring_buffer()
1238 dev_priv = ring->dev->dev_private; in intel_cleanup_ring_buffer()
1239 ret = intel_ring_idle(ring); in intel_cleanup_ring_buffer()
1242 ring->name, ret); in intel_cleanup_ring_buffer()
1244 I915_WRITE_CTL(ring, 0); in intel_cleanup_ring_buffer()
1246 pmap_unmapdev((vm_offset_t)ring->virtual_start, ring->size); in intel_cleanup_ring_buffer()
1248 i915_gem_object_unpin(ring->obj); in intel_cleanup_ring_buffer()
1249 drm_gem_object_unreference(&ring->obj->base); in intel_cleanup_ring_buffer()
1250 ring->obj = NULL; in intel_cleanup_ring_buffer()
1252 if (ring->cleanup) in intel_cleanup_ring_buffer()
1253 ring->cleanup(ring); in intel_cleanup_ring_buffer()
1255 cleanup_status_page(ring); in intel_cleanup_ring_buffer()
1258 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno) in intel_ring_wait_seqno() argument
1262 ret = i915_wait_seqno(ring, seqno); in intel_ring_wait_seqno()
1264 i915_gem_retire_requests_ring(ring); in intel_ring_wait_seqno()
1269 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n) in intel_ring_wait_request() argument
1275 i915_gem_retire_requests_ring(ring); in intel_ring_wait_request()
1277 if (ring->last_retired_head != -1) { in intel_ring_wait_request()
1278 ring->head = ring->last_retired_head; in intel_ring_wait_request()
1279 ring->last_retired_head = -1; in intel_ring_wait_request()
1280 ring->space = ring_space(ring); in intel_ring_wait_request()
1281 if (ring->space >= n) in intel_ring_wait_request()
1285 list_for_each_entry(request, &ring->request_list, list) { in intel_ring_wait_request()
1291 space = request->tail - (ring->tail + I915_RING_FREE_SPACE); in intel_ring_wait_request()
1293 space += ring->size; in intel_ring_wait_request()
1310 ret = intel_ring_wait_seqno(ring, seqno); in intel_ring_wait_request()
1314 if (WARN_ON(ring->last_retired_head == -1)) in intel_ring_wait_request()
1317 ring->head = ring->last_retired_head; in intel_ring_wait_request()
1318 ring->last_retired_head = -1; in intel_ring_wait_request()
1319 ring->space = ring_space(ring); in intel_ring_wait_request()
1320 if (WARN_ON(ring->space < n)) in intel_ring_wait_request()
1326 static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) in ring_wait_for_space() argument
1328 struct drm_device *dev = ring->dev; in ring_wait_for_space()
1333 ret = intel_ring_wait_request(ring, n); in ring_wait_for_space()
1337 CTR1(KTR_DRM, "ring_wait_begin %s", ring->name); in ring_wait_for_space()
1346 ring->head = I915_READ_HEAD(ring); in ring_wait_for_space()
1347 ring->space = ring_space(ring); in ring_wait_for_space()
1348 if (ring->space >= n) { in ring_wait_for_space()
1349 CTR1(KTR_DRM, "ring_wait_end %s", ring->name); in ring_wait_for_space()
1363 CTR1(KTR_DRM, "ring_wait_end %s wedged", ring->name); in ring_wait_for_space()
1367 CTR1(KTR_DRM, "ring_wait_end %s busy", ring->name); in ring_wait_for_space()
1371 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring) in intel_wrap_ring_buffer() argument
1374 int rem = ring->size - ring->tail; in intel_wrap_ring_buffer()
1376 if (ring->space < rem) { in intel_wrap_ring_buffer()
1377 int ret = ring_wait_for_space(ring, rem); in intel_wrap_ring_buffer()
1382 virt = (uint32_t *)((char *)ring->virtual_start + ring->tail); in intel_wrap_ring_buffer()
1387 ring->tail = 0; in intel_wrap_ring_buffer()
1388 ring->space = ring_space(ring); in intel_wrap_ring_buffer()
1393 int intel_ring_idle(struct intel_ring_buffer *ring) in intel_ring_idle() argument
1399 if (ring->outstanding_lazy_request) { in intel_ring_idle()
1400 ret = i915_add_request(ring, NULL, NULL); in intel_ring_idle()
1406 if (list_empty(&ring->request_list)) in intel_ring_idle()
1409 seqno = list_entry(ring->request_list.prev, in intel_ring_idle()
1413 return i915_wait_seqno(ring, seqno); in intel_ring_idle()
1417 intel_ring_alloc_seqno(struct intel_ring_buffer *ring) in intel_ring_alloc_seqno() argument
1419 if (ring->outstanding_lazy_request) in intel_ring_alloc_seqno()
1422 return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request); in intel_ring_alloc_seqno()
1425 int intel_ring_begin(struct intel_ring_buffer *ring, in intel_ring_begin() argument
1428 drm_i915_private_t *dev_priv = ring->dev->dev_private; in intel_ring_begin()
1437 ret = intel_ring_alloc_seqno(ring); in intel_ring_begin()
1441 if (unlikely(ring->tail + n > ring->effective_size)) { in intel_ring_begin()
1442 ret = intel_wrap_ring_buffer(ring); in intel_ring_begin()
1447 if (unlikely(ring->space < n)) { in intel_ring_begin()
1448 ret = ring_wait_for_space(ring, n); in intel_ring_begin()
1453 ring->space -= n; in intel_ring_begin()
1457 void intel_ring_advance(struct intel_ring_buffer *ring) in intel_ring_advance() argument
1459 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_advance()
1461 ring->tail &= ring->size - 1; in intel_ring_advance()
1462 if (dev_priv->stop_rings & intel_ring_flag(ring)) in intel_ring_advance()
1464 ring->write_tail(ring, ring->tail); in intel_ring_advance()
1468 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, in gen6_bsd_ring_write_tail() argument
1471 drm_i915_private_t *dev_priv = ring->dev->dev_private; in gen6_bsd_ring_write_tail()
1491 I915_WRITE_TAIL(ring, value); in gen6_bsd_ring_write_tail()
1492 POSTING_READ(RING_TAIL(ring->mmio_base)); in gen6_bsd_ring_write_tail()
1501 static int gen6_ring_flush(struct intel_ring_buffer *ring, in gen6_ring_flush() argument
1507 ret = intel_ring_begin(ring, 4); in gen6_ring_flush()
1521 intel_ring_emit(ring, cmd); in gen6_ring_flush()
1522 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_ring_flush()
1523 intel_ring_emit(ring, 0); in gen6_ring_flush()
1524 intel_ring_emit(ring, MI_NOOP); in gen6_ring_flush()
1525 intel_ring_advance(ring); in gen6_ring_flush()
1530 hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, in hsw_ring_dispatch_execbuffer() argument
1536 ret = intel_ring_begin(ring, 2); in hsw_ring_dispatch_execbuffer()
1540 intel_ring_emit(ring, in hsw_ring_dispatch_execbuffer()
1544 intel_ring_emit(ring, offset); in hsw_ring_dispatch_execbuffer()
1545 intel_ring_advance(ring); in hsw_ring_dispatch_execbuffer()
1551 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, in gen6_ring_dispatch_execbuffer() argument
1557 ret = intel_ring_begin(ring, 2); in gen6_ring_dispatch_execbuffer()
1561 intel_ring_emit(ring, in gen6_ring_dispatch_execbuffer()
1565 intel_ring_emit(ring, offset); in gen6_ring_dispatch_execbuffer()
1566 intel_ring_advance(ring); in gen6_ring_dispatch_execbuffer()
1573 static int blt_ring_flush(struct intel_ring_buffer *ring, in blt_ring_flush() argument
1579 ret = intel_ring_begin(ring, 4); in blt_ring_flush()
1593 intel_ring_emit(ring, cmd); in blt_ring_flush()
1594 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in blt_ring_flush()
1595 intel_ring_emit(ring, 0); in blt_ring_flush()
1596 intel_ring_emit(ring, MI_NOOP); in blt_ring_flush()
1597 intel_ring_advance(ring); in blt_ring_flush()
1604 struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; in intel_init_render_ring_buffer() local
1606 ring->name = "render ring"; in intel_init_render_ring_buffer()
1607 ring->id = RCS; in intel_init_render_ring_buffer()
1608 ring->mmio_base = RENDER_RING_BASE; in intel_init_render_ring_buffer()
1611 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
1612 ring->flush = gen7_render_ring_flush; in intel_init_render_ring_buffer()
1614 ring->flush = gen6_render_ring_flush; in intel_init_render_ring_buffer()
1615 ring->irq_get = gen6_ring_get_irq; in intel_init_render_ring_buffer()
1616 ring->irq_put = gen6_ring_put_irq; in intel_init_render_ring_buffer()
1617 ring->irq_enable_mask = GT_USER_INTERRUPT; in intel_init_render_ring_buffer()
1618 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
1619 ring->sync_to = gen6_ring_sync; in intel_init_render_ring_buffer()
1620 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
1621 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV; in intel_init_render_ring_buffer()
1622 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB; in intel_init_render_ring_buffer()
1623 ring->signal_mbox[0] = GEN6_VRSYNC; in intel_init_render_ring_buffer()
1624 ring->signal_mbox[1] = GEN6_BRSYNC; in intel_init_render_ring_buffer()
1626 ring->add_request = pc_render_add_request; in intel_init_render_ring_buffer()
1627 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
1628 ring->get_seqno = pc_render_get_seqno; in intel_init_render_ring_buffer()
1629 ring->irq_get = gen5_ring_get_irq; in intel_init_render_ring_buffer()
1630 ring->irq_put = gen5_ring_put_irq; in intel_init_render_ring_buffer()
1631 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY; in intel_init_render_ring_buffer()
1633 ring->add_request = i9xx_add_request; in intel_init_render_ring_buffer()
1635 ring->flush = gen2_render_ring_flush; in intel_init_render_ring_buffer()
1637 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
1638 ring->get_seqno = ring_get_seqno; in intel_init_render_ring_buffer()
1640 ring->irq_get = i8xx_ring_get_irq; in intel_init_render_ring_buffer()
1641 ring->irq_put = i8xx_ring_put_irq; in intel_init_render_ring_buffer()
1643 ring->irq_get = i9xx_ring_get_irq; in intel_init_render_ring_buffer()
1644 ring->irq_put = i9xx_ring_put_irq; in intel_init_render_ring_buffer()
1646 ring->irq_enable_mask = I915_USER_INTERRUPT; in intel_init_render_ring_buffer()
1648 ring->write_tail = ring_write_tail; in intel_init_render_ring_buffer()
1650 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
1652 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
1654 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_render_ring_buffer()
1656 ring->dispatch_execbuffer = i830_dispatch_execbuffer; in intel_init_render_ring_buffer()
1658 ring->dispatch_execbuffer = i915_dispatch_execbuffer; in intel_init_render_ring_buffer()
1659 ring->init = init_render_ring; in intel_init_render_ring_buffer()
1660 ring->cleanup = render_ring_cleanup; in intel_init_render_ring_buffer()
1680 ring->private = obj; in intel_init_render_ring_buffer()
1683 return intel_init_ring_buffer(dev, ring); in intel_init_render_ring_buffer()
1689 struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; in intel_render_ring_init_dri() local
1692 ring->name = "render ring"; in intel_render_ring_init_dri()
1693 ring->id = RCS; in intel_render_ring_init_dri()
1694 ring->mmio_base = RENDER_RING_BASE; in intel_render_ring_init_dri()
1704 ring->add_request = i9xx_add_request; in intel_render_ring_init_dri()
1706 ring->flush = gen2_render_ring_flush; in intel_render_ring_init_dri()
1708 ring->flush = gen4_render_ring_flush; in intel_render_ring_init_dri()
1709 ring->get_seqno = ring_get_seqno; in intel_render_ring_init_dri()
1711 ring->irq_get = i8xx_ring_get_irq; in intel_render_ring_init_dri()
1712 ring->irq_put = i8xx_ring_put_irq; in intel_render_ring_init_dri()
1714 ring->irq_get = i9xx_ring_get_irq; in intel_render_ring_init_dri()
1715 ring->irq_put = i9xx_ring_put_irq; in intel_render_ring_init_dri()
1717 ring->irq_enable_mask = I915_USER_INTERRUPT; in intel_render_ring_init_dri()
1718 ring->write_tail = ring_write_tail; in intel_render_ring_init_dri()
1720 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_render_ring_init_dri()
1722 ring->dispatch_execbuffer = i830_dispatch_execbuffer; in intel_render_ring_init_dri()
1724 ring->dispatch_execbuffer = i915_dispatch_execbuffer; in intel_render_ring_init_dri()
1725 ring->init = init_render_ring; in intel_render_ring_init_dri()
1726 ring->cleanup = render_ring_cleanup; in intel_render_ring_init_dri()
1728 ring->dev = dev; in intel_render_ring_init_dri()
1729 INIT_LIST_HEAD(&ring->active_list); in intel_render_ring_init_dri()
1730 INIT_LIST_HEAD(&ring->request_list); in intel_render_ring_init_dri()
1732 ring->size = size; in intel_render_ring_init_dri()
1733 ring->effective_size = ring->size; in intel_render_ring_init_dri()
1734 if (IS_I830(ring->dev) || IS_845G(ring->dev)) in intel_render_ring_init_dri()
1735 ring->effective_size -= 128; in intel_render_ring_init_dri()
1737 ring->virtual_start = pmap_mapdev_attr(start, size, in intel_render_ring_init_dri()
1739 if (ring->virtual_start == NULL) { in intel_render_ring_init_dri()
1746 ret = init_phys_hws_pga(ring); in intel_render_ring_init_dri()
1757 struct intel_ring_buffer *ring = &dev_priv->ring[VCS]; in intel_init_bsd_ring_buffer() local
1759 ring->name = "bsd ring"; in intel_init_bsd_ring_buffer()
1760 ring->id = VCS; in intel_init_bsd_ring_buffer()
1762 ring->write_tail = ring_write_tail; in intel_init_bsd_ring_buffer()
1764 ring->mmio_base = GEN6_BSD_RING_BASE; in intel_init_bsd_ring_buffer()
1767 ring->write_tail = gen6_bsd_ring_write_tail; in intel_init_bsd_ring_buffer()
1768 ring->flush = gen6_ring_flush; in intel_init_bsd_ring_buffer()
1769 ring->add_request = gen6_add_request; in intel_init_bsd_ring_buffer()
1770 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd_ring_buffer()
1771 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
1772 ring->irq_get = gen6_ring_get_irq; in intel_init_bsd_ring_buffer()
1773 ring->irq_put = gen6_ring_put_irq; in intel_init_bsd_ring_buffer()
1774 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_bsd_ring_buffer()
1775 ring->sync_to = gen6_ring_sync; in intel_init_bsd_ring_buffer()
1776 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR; in intel_init_bsd_ring_buffer()
1777 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
1778 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB; in intel_init_bsd_ring_buffer()
1779 ring->signal_mbox[0] = GEN6_RVSYNC; in intel_init_bsd_ring_buffer()
1780 ring->signal_mbox[1] = GEN6_BVSYNC; in intel_init_bsd_ring_buffer()
1782 ring->mmio_base = BSD_RING_BASE; in intel_init_bsd_ring_buffer()
1783 ring->flush = bsd_ring_flush; in intel_init_bsd_ring_buffer()
1784 ring->add_request = i9xx_add_request; in intel_init_bsd_ring_buffer()
1785 ring->get_seqno = ring_get_seqno; in intel_init_bsd_ring_buffer()
1787 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
1788 ring->irq_get = gen5_ring_get_irq; in intel_init_bsd_ring_buffer()
1789 ring->irq_put = gen5_ring_put_irq; in intel_init_bsd_ring_buffer()
1791 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
1792 ring->irq_get = i9xx_ring_get_irq; in intel_init_bsd_ring_buffer()
1793 ring->irq_put = i9xx_ring_put_irq; in intel_init_bsd_ring_buffer()
1795 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_bsd_ring_buffer()
1797 ring->init = init_ring_common; in intel_init_bsd_ring_buffer()
1799 return intel_init_ring_buffer(dev, ring); in intel_init_bsd_ring_buffer()
1805 struct intel_ring_buffer *ring = &dev_priv->ring[BCS]; in intel_init_blt_ring_buffer() local
1807 ring->name = "blitter ring"; in intel_init_blt_ring_buffer()
1808 ring->id = BCS; in intel_init_blt_ring_buffer()
1810 ring->mmio_base = BLT_RING_BASE; in intel_init_blt_ring_buffer()
1811 ring->write_tail = ring_write_tail; in intel_init_blt_ring_buffer()
1812 ring->flush = blt_ring_flush; in intel_init_blt_ring_buffer()
1813 ring->add_request = gen6_add_request; in intel_init_blt_ring_buffer()
1814 ring->get_seqno = gen6_ring_get_seqno; in intel_init_blt_ring_buffer()
1815 ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT; in intel_init_blt_ring_buffer()
1816 ring->irq_get = gen6_ring_get_irq; in intel_init_blt_ring_buffer()
1817 ring->irq_put = gen6_ring_put_irq; in intel_init_blt_ring_buffer()
1818 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
1819 ring->sync_to = gen6_ring_sync; in intel_init_blt_ring_buffer()
1820 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR; in intel_init_blt_ring_buffer()
1821 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV; in intel_init_blt_ring_buffer()
1822 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
1823 ring->signal_mbox[0] = GEN6_RBSYNC; in intel_init_blt_ring_buffer()
1824 ring->signal_mbox[1] = GEN6_VBSYNC; in intel_init_blt_ring_buffer()
1825 ring->init = init_ring_common; in intel_init_blt_ring_buffer()
1827 return intel_init_ring_buffer(dev, ring); in intel_init_blt_ring_buffer()
1831 intel_ring_flush_all_caches(struct intel_ring_buffer *ring) in intel_ring_flush_all_caches() argument
1835 if (!ring->gpu_caches_dirty) in intel_ring_flush_all_caches()
1838 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS); in intel_ring_flush_all_caches()
1842 ring->gpu_caches_dirty = false; in intel_ring_flush_all_caches()
1847 intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring) in intel_ring_invalidate_all_caches() argument
1853 if (ring->gpu_caches_dirty) in intel_ring_invalidate_all_caches()
1856 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); in intel_ring_invalidate_all_caches()
1860 ring->gpu_caches_dirty = false; in intel_ring_invalidate_all_caches()