64 	dev_priv->dma_low +=8;					\
72 	dev_priv->dma_low += 8;
74 static void via_cmdbuf_start(drm_via_private_t * dev_priv);
75 static void via_cmdbuf_pause(drm_via_private_t * dev_priv);
76 static void via_cmdbuf_reset(drm_via_private_t * dev_priv);
77 static void via_cmdbuf_rewind(drm_via_private_t * dev_priv);
78 static int via_wait_idle(drm_via_private_t * dev_priv);
79 static void via_pad_cache(drm_via_private_t * dev_priv, int qwords);
85 static uint32_t via_cmdbuf_space(drm_via_private_t * dev_priv)  in via_cmdbuf_space()  argument
87 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_space()
88 	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;  in via_cmdbuf_space()
90 	return ((hw_addr <= dev_priv->dma_low) ?  in via_cmdbuf_space()
91 		(dev_priv->dma_high + hw_addr - dev_priv->dma_low) :  in via_cmdbuf_space()
92 		(hw_addr - dev_priv->dma_low));  in via_cmdbuf_space()
99 static uint32_t via_cmdbuf_lag(drm_via_private_t * dev_priv)  in via_cmdbuf_lag()  argument
101 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_lag()
102 	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;  in via_cmdbuf_lag()
104 	return ((hw_addr <= dev_priv->dma_low) ?  in via_cmdbuf_lag()
105 		(dev_priv->dma_low - hw_addr) :  in via_cmdbuf_lag()
106 		(dev_priv->dma_wrap + dev_priv->dma_low - hw_addr));  in via_cmdbuf_lag()
114 via_cmdbuf_wait(drm_via_private_t * dev_priv, unsigned int size)  in via_cmdbuf_wait()  argument
116 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_wait()
120 	hw_addr_ptr = dev_priv->hw_addr_ptr;  in via_cmdbuf_wait()
121 	cur_addr = dev_priv->dma_low;  in via_cmdbuf_wait()
145 static inline uint32_t *via_check_dma(drm_via_private_t * dev_priv,  in via_check_dma()  argument
148 	if ((dev_priv->dma_low + size + 4 * CMDBUF_ALIGNMENT_SIZE) >  in via_check_dma()
149 	    dev_priv->dma_high) {  in via_check_dma()
150 		via_cmdbuf_rewind(dev_priv);  in via_check_dma()
152 	if (via_cmdbuf_wait(dev_priv, size) != 0) {  in via_check_dma()
156 	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);  in via_check_dma()
165 		drm_via_private_t *dev_priv =  in via_dma_cleanup()  local
168 		if (dev_priv->ring.virtual_start) {  in via_dma_cleanup()
169 			via_cmdbuf_reset(dev_priv);  in via_dma_cleanup()
171 			drm_core_ioremapfree(&dev_priv->ring.map, dev);  in via_dma_cleanup()
172 			dev_priv->ring.virtual_start = NULL;  in via_dma_cleanup()
176 			blitq = dev_priv->blit_queues + i;  in via_dma_cleanup()
185 			  drm_via_private_t * dev_priv,  in via_initialize()  argument
188 	if (!dev_priv || !dev_priv->mmio) {  in via_initialize()
193 	if (dev_priv->ring.virtual_start != NULL) {  in via_initialize()
203 	if (dev_priv->chipset == VIA_DX9_0) {  in via_initialize()
208 	dev_priv->ring.map.offset = dev->agp->base + init->offset;  in via_initialize()
209 	dev_priv->ring.map.size = init->size;  in via_initialize()
210 	dev_priv->ring.map.type = 0;  in via_initialize()
211 	dev_priv->ring.map.flags = 0;  in via_initialize()
212 	dev_priv->ring.map.mtrr = 0;  in via_initialize()
214 	drm_core_ioremap_wc(&dev_priv->ring.map, dev);  in via_initialize()
216 	if (dev_priv->ring.map.virtual == NULL) {  in via_initialize()
223 	dev_priv->ring.virtual_start = dev_priv->ring.map.virtual;  in via_initialize()
225 	dev_priv->dma_ptr = dev_priv->ring.virtual_start;  in via_initialize()
226 	dev_priv->dma_low = 0;  in via_initialize()
227 	dev_priv->dma_high = init->size;  in via_initialize()
228 	dev_priv->dma_wrap = init->size;  in via_initialize()
229 	dev_priv->dma_offset = init->offset;  in via_initialize()
230 	dev_priv->last_pause_ptr = NULL;  in via_initialize()
231 	dev_priv->hw_addr_ptr =  in via_initialize()
232 		(volatile uint32_t *)((char *)dev_priv->mmio->virtual +  in via_initialize()
235 	via_cmdbuf_start(dev_priv);  in via_initialize()
242 	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;  in via_dma_init()  local
251 			retcode = via_initialize(dev, dev_priv, init);  in via_dma_init()
260 		retcode = (dev_priv->ring.virtual_start != NULL) ?  in via_dma_init()
273 	drm_via_private_t *dev_priv;  in via_dispatch_cmdbuffer()  local
277 	dev_priv = (drm_via_private_t *) dev->dev_private;  in via_dispatch_cmdbuffer()
279 	if (dev_priv->ring.virtual_start == NULL) {  in via_dispatch_cmdbuffer()
288 	if (DRM_COPY_FROM_USER(dev_priv->pci_buf, cmd->buf, cmd->size))  in via_dispatch_cmdbuffer()
298 	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,  in via_dispatch_cmdbuffer()
303 	vb = via_check_dma(dev_priv, (cmd->size < 0x100) ? 0x102 : cmd->size);  in via_dispatch_cmdbuffer()
308 	memcpy(vb, dev_priv->pci_buf, cmd->size);  in via_dispatch_cmdbuffer()
310 	dev_priv->dma_low += cmd->size;  in via_dispatch_cmdbuffer()
318 		via_pad_cache(dev_priv, (0x100 - cmd->size) >> 3);  in via_dispatch_cmdbuffer()
319 	via_cmdbuf_pause(dev_priv);  in via_dispatch_cmdbuffer()
326 	drm_via_private_t *dev_priv = dev->dev_private;  in via_driver_dma_quiescent()  local
328 	if (!via_wait_idle(dev_priv)) {  in via_driver_dma_quiescent()
362 	drm_via_private_t *dev_priv = dev->dev_private;  in via_dispatch_pci_cmdbuffer()  local
368 	if (DRM_COPY_FROM_USER(dev_priv->pci_buf, cmd->buf, cmd->size))  in via_dispatch_pci_cmdbuffer()
372 	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,  in via_dispatch_pci_cmdbuffer()
378 	    via_parse_command_stream(dev, (const uint32_t *)dev_priv->pci_buf,  in via_dispatch_pci_cmdbuffer()
400 static inline uint32_t *via_align_buffer(drm_via_private_t * dev_priv,  in via_align_buffer()  argument
414 static inline uint32_t *via_get_dma(drm_via_private_t * dev_priv)  in via_get_dma()  argument
416 	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);  in via_get_dma()
424 static int via_hook_segment(drm_via_private_t * dev_priv,  in via_hook_segment()  argument
429 	volatile uint32_t *paused_at = dev_priv->last_pause_ptr;  in via_hook_segment()
435 	(void) *(volatile uint32_t *)(via_get_dma(dev_priv) -1);  in via_hook_segment()
441 	reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
442 	ptr = ((volatile char *)paused_at - dev_priv->dma_ptr) +  in via_hook_segment()
443 		dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;  in via_hook_segment()
445 	dev_priv->last_pause_ptr = via_get_dma(dev_priv) - 1;  in via_hook_segment()
454 	diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
460 		reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
461 		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
467 		reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
468 		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
469 		diff &= (dev_priv->dma_high - 1);  in via_hook_segment()
470 		if (diff != 0 && diff < (dev_priv->dma_high >> 1)) {  in via_hook_segment()
473 				  ptr, reader, dev_priv->dma_diff);  in via_hook_segment()
490 static int via_wait_idle(drm_via_private_t * dev_priv)  in via_wait_idle()  argument
504 static uint32_t *via_align_cmd(drm_via_private_t * dev_priv, uint32_t cmd_type,  in via_align_cmd()  argument
514 		via_cmdbuf_wait(dev_priv, 2 * CMDBUF_ALIGNMENT_SIZE);  in via_align_cmd()
516 	vb = via_get_dma(dev_priv);  in via_align_cmd()
519 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_align_cmd()
521 	    ((dev_priv->dma_low & CMDBUF_ALIGNMENT_MASK) >> 3);  in via_align_cmd()
524 	    agp_base + dev_priv->dma_low - 8 + (qw_pad_count << 3);  in via_align_cmd()
529 	vb = via_align_buffer(dev_priv, vb, qw_pad_count - 1);  in via_align_cmd()
534 static void via_cmdbuf_start(drm_via_private_t * dev_priv)  in via_cmdbuf_start()  argument
545 	dev_priv->dma_low = 0;  in via_cmdbuf_start()
547 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_start()
549 	end_addr = agp_base + dev_priv->dma_high;  in via_cmdbuf_start()
556 	dev_priv->last_pause_ptr =  in via_cmdbuf_start()
557 	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0,  in via_cmdbuf_start()
561 	(void) *(volatile uint32_t *)dev_priv->last_pause_ptr;  in via_cmdbuf_start()
574 	dev_priv->dma_diff = 0;  in via_cmdbuf_start()
579 	reader = *(dev_priv->hw_addr_ptr);  in via_cmdbuf_start()
580 	ptr = ((volatile char *)dev_priv->last_pause_ptr - dev_priv->dma_ptr) +  in via_cmdbuf_start()
581 	    dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;  in via_cmdbuf_start()
590 	dev_priv->dma_diff = ptr - reader;  in via_cmdbuf_start()
593 static void via_pad_cache(drm_via_private_t * dev_priv, int qwords)  in via_pad_cache()  argument
597 	via_cmdbuf_wait(dev_priv, qwords + 2);  in via_pad_cache()
598 	vb = via_get_dma(dev_priv);  in via_pad_cache()
600 	via_align_buffer(dev_priv, vb, qwords);  in via_pad_cache()
603 static inline void via_dummy_bitblt(drm_via_private_t * dev_priv)  in via_dummy_bitblt()  argument
605 	uint32_t *vb = via_get_dma(dev_priv);  in via_dummy_bitblt()
611 static void via_cmdbuf_jump(drm_via_private_t * dev_priv)  in via_cmdbuf_jump()  argument
619 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_jump()
620 	via_align_cmd(dev_priv, HC_HAGPBpID_JUMP, 0, &jump_addr_hi,  in via_cmdbuf_jump()
623 	dev_priv->dma_wrap = dev_priv->dma_low;  in via_cmdbuf_jump()
629 	dev_priv->dma_low = 0;  in via_cmdbuf_jump()
630 	if (via_cmdbuf_wait(dev_priv, CMDBUF_ALIGNMENT_SIZE) != 0) {  in via_cmdbuf_jump()
634 	via_dummy_bitblt(dev_priv);  in via_cmdbuf_jump()
635 	via_dummy_bitblt(dev_priv);  in via_cmdbuf_jump()
638 	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
640 	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
644 	dma_low_save1 = dev_priv->dma_low;  in via_cmdbuf_jump()
656 		via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
658 	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
662 	dma_low_save2 = dev_priv->dma_low;  in via_cmdbuf_jump()
663 	dev_priv->dma_low = dma_low_save1;  in via_cmdbuf_jump()
664 	via_hook_segment(dev_priv, jump_addr_hi, jump_addr_lo, 0);  in via_cmdbuf_jump()
665 	dev_priv->dma_low = dma_low_save2;  in via_cmdbuf_jump()
666 	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);  in via_cmdbuf_jump()
670 static void via_cmdbuf_rewind(drm_via_private_t * dev_priv)  in via_cmdbuf_rewind()  argument
672 	via_cmdbuf_jump(dev_priv);  in via_cmdbuf_rewind()
675 static void via_cmdbuf_flush(drm_via_private_t * dev_priv, uint32_t cmd_type)  in via_cmdbuf_flush()  argument
679 	via_align_cmd(dev_priv, cmd_type, 0, &pause_addr_hi, &pause_addr_lo, 0);  in via_cmdbuf_flush()
680 	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);  in via_cmdbuf_flush()
683 static void via_cmdbuf_pause(drm_via_private_t * dev_priv)  in via_cmdbuf_pause()  argument
685 	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_PAUSE);  in via_cmdbuf_pause()
688 static void via_cmdbuf_reset(drm_via_private_t * dev_priv)  in via_cmdbuf_reset()  argument
690 	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_STOP);  in via_cmdbuf_reset()
691 	via_wait_idle(dev_priv);  in via_cmdbuf_reset()
703 	drm_via_private_t *dev_priv;  in via_cmdbuf_size()  local
708 	dev_priv = (drm_via_private_t *) dev->dev_private;  in via_cmdbuf_size()
710 	if (dev_priv->ring.virtual_start == NULL) {  in via_cmdbuf_size()
719 		while (((tmp_size = via_cmdbuf_space(dev_priv)) < d_siz->size)  in via_cmdbuf_size()
731 		while (((tmp_size = via_cmdbuf_lag(dev_priv)) > d_siz->size)  in via_cmdbuf_size()