1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com> 29 */ 30 31 #include <linux/export.h> 32 #include <linux/highmem.h> 33 #include <xen/xen.h> 34 35 #include <drm/drm_cache.h> 36 37 #if defined(CONFIG_X86) 38 #include <asm/smp.h> 39 40 /* 41 * clflushopt is an unordered instruction which needs fencing with mfence or 42 * sfence to avoid ordering issues. For drm_clflush_page this fencing happens 43 * in the caller. 44 */ 45 static void 46 drm_clflush_page(struct page *page) 47 { 48 uint8_t *page_virtual; 49 unsigned int i; 50 const int size = boot_cpu_data.x86_clflush_size; 51 52 if (unlikely(page == NULL)) 53 return; 54 55 page_virtual = kmap_atomic(page); 56 for (i = 0; i < PAGE_SIZE; i += size) 57 clflushopt(page_virtual + i); 58 kunmap_atomic(page_virtual); 59 } 60 61 static void drm_cache_flush_clflush(struct page *pages[], 62 unsigned long num_pages) 63 { 64 unsigned long i; 65 66 mb(); /*Full memory barrier used before so that CLFLUSH is ordered*/ 67 for (i = 0; i < num_pages; i++) 68 drm_clflush_page(*pages++); 69 mb(); /*Also used after CLFLUSH so that all cache is flushed*/ 70 } 71 #endif 72 73 /** 74 * drm_clflush_pages - Flush dcache lines of a set of pages. 75 * @pages: List of pages to be flushed. 76 * @num_pages: Number of pages in the array. 77 * 78 * Flush every data cache line entry that points to an address belonging 79 * to a page in the array. 80 */ 81 void 82 drm_clflush_pages(struct page *pages[], unsigned long num_pages) 83 { 84 85 #if defined(CONFIG_X86) 86 if (static_cpu_has(X86_FEATURE_CLFLUSH)) { 87 drm_cache_flush_clflush(pages, num_pages); 88 return; 89 } 90 91 if (wbinvd_on_all_cpus()) 92 pr_err("Timed out waiting for cache flush\n"); 93 94 #elif defined(__powerpc__) 95 unsigned long i; 96 97 for (i = 0; i < num_pages; i++) { 98 struct page *page = pages[i]; 99 void *page_virtual; 100 101 if (unlikely(page == NULL)) 102 continue; 103 104 page_virtual = kmap_atomic(page); 105 flush_dcache_range((unsigned long)page_virtual, 106 (unsigned long)page_virtual + PAGE_SIZE); 107 kunmap_atomic(page_virtual); 108 } 109 #else 110 pr_err("Architecture has no drm_cache.c support\n"); 111 WARN_ON_ONCE(1); 112 #endif 113 } 114 EXPORT_SYMBOL(drm_clflush_pages); 115 116 /** 117 * drm_clflush_sg - Flush dcache lines pointing to a scather-gather. 118 * @st: struct sg_table. 119 * 120 * Flush every data cache line entry that points to an address in the 121 * sg. 122 */ 123 void 124 drm_clflush_sg(struct sg_table *st) 125 { 126 #if defined(CONFIG_X86) 127 if (static_cpu_has(X86_FEATURE_CLFLUSH)) { 128 struct sg_page_iter sg_iter; 129 130 mb(); /*CLFLUSH is ordered only by using memory barriers*/ 131 for_each_sgtable_page(st, &sg_iter, 0) 132 drm_clflush_page(sg_page_iter_page(&sg_iter)); 133 mb(); /*Make sure that all cache line entry is flushed*/ 134 135 return; 136 } 137 138 if (wbinvd_on_all_cpus()) 139 pr_err("Timed out waiting for cache flush\n"); 140 #else 141 pr_err("Architecture has no drm_cache.c support\n"); 142 WARN_ON_ONCE(1); 143 #endif 144 } 145 EXPORT_SYMBOL(drm_clflush_sg); 146 147 /** 148 * drm_clflush_virt_range - Flush dcache lines of a region 149 * @addr: Initial kernel memory address. 150 * @length: Region size. 151 * 152 * Flush every data cache line entry that points to an address in the 153 * region requested. 154 */ 155 void 156 drm_clflush_virt_range(void *addr, unsigned long length) 157 { 158 #if defined(CONFIG_X86) 159 if (static_cpu_has(X86_FEATURE_CLFLUSH)) { 160 const int size = boot_cpu_data.x86_clflush_size; 161 void *end = addr + length; 162 163 addr = (void *)(((unsigned long)addr) & -size); 164 mb(); /*CLFLUSH is only ordered with a full memory barrier*/ 165 for (; addr < end; addr += size) 166 clflushopt(addr); 167 clflushopt(end - 1); /* force serialisation */ 168 mb(); /*Ensure that evry data cache line entry is flushed*/ 169 return; 170 } 171 172 if (wbinvd_on_all_cpus()) 173 pr_err("Timed out waiting for cache flush\n"); 174 #else 175 pr_err("Architecture has no drm_cache.c support\n"); 176 WARN_ON_ONCE(1); 177 #endif 178 } 179 EXPORT_SYMBOL(drm_clflush_virt_range); 180 181 bool drm_need_swiotlb(int dma_bits) 182 { 183 struct resource *tmp; 184 resource_size_t max_iomem = 0; 185 186 /* 187 * Xen paravirtual hosts require swiotlb regardless of requested dma 188 * transfer size. 189 * 190 * NOTE: Really, what it requires is use of the dma_alloc_coherent 191 * allocator used in ttm_dma_populate() instead of 192 * ttm_populate_and_map_pages(), which bounce buffers so much in 193 * Xen it leads to swiotlb buffer exhaustion. 194 */ 195 if (xen_pv_domain()) 196 return true; 197 198 /* 199 * Enforce dma_alloc_coherent when memory encryption is active as well 200 * for the same reasons as for Xen paravirtual hosts. 201 */ 202 if (mem_encrypt_active()) 203 return true; 204 205 for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling) 206 max_iomem = max(max_iomem, tmp->end); 207 208 return max_iomem > ((u64)1 << dma_bits); 209 } 210 EXPORT_SYMBOL(drm_need_swiotlb); 211