1 /* ===-- clear_cache.c - Implement __clear_cache ---------------------------=== 2 * 3 * The LLVM Compiler Infrastructure 4 * 5 * This file is dual licensed under the MIT and the University of Illinois Open 6 * Source Licenses. See LICENSE.TXT for details. 7 * 8 * ===----------------------------------------------------------------------=== 9 */ 10 11 #include "int_lib.h" 12 #include <assert.h> 13 #include <stddef.h> 14 15 #if __APPLE__ 16 #include <libkern/OSCacheControl.h> 17 #endif 18 19 #if defined(_WIN32) 20 /* Forward declare Win32 APIs since the GCC mode driver does not handle the 21 newer SDKs as well as needed. */ 22 uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress, 23 uintptr_t dwSize); 24 uintptr_t GetCurrentProcess(void); 25 #endif 26 27 #if defined(__FreeBSD__) && defined(__arm__) 28 #include <sys/types.h> 29 #include <machine/sysarch.h> 30 #endif 31 32 #if defined(__NetBSD__) && defined(__arm__) 33 #include <machine/sysarch.h> 34 #endif 35 36 #if defined(__OpenBSD__) && defined(__mips__) 37 #include <sys/types.h> 38 #include <machine/sysarch.h> 39 #endif 40 41 #if defined(__linux__) && defined(__mips__) 42 #include <sys/cachectl.h> 43 #include <sys/syscall.h> 44 #include <unistd.h> 45 #if defined(__ANDROID__) && defined(__LP64__) 46 /* 47 * clear_mips_cache - Invalidates instruction cache for Mips. 48 */ 49 static void clear_mips_cache(const void* Addr, size_t Size) { 50 __asm__ volatile ( 51 ".set push\n" 52 ".set noreorder\n" 53 ".set noat\n" 54 "beq %[Size], $zero, 20f\n" /* If size == 0, branch around. */ 55 "nop\n" 56 "daddu %[Size], %[Addr], %[Size]\n" /* Calculate end address + 1 */ 57 "rdhwr $v0, $1\n" /* Get step size for SYNCI. 58 $1 is $HW_SYNCI_Step */ 59 "beq $v0, $zero, 20f\n" /* If no caches require 60 synchronization, branch 61 around. */ 62 "nop\n" 63 "10:\n" 64 "synci 0(%[Addr])\n" /* Synchronize all caches around 65 address. */ 66 "daddu %[Addr], %[Addr], $v0\n" /* Add step size. */ 67 "sltu $at, %[Addr], %[Size]\n" /* Compare current with end 68 address. */ 69 "bne $at, $zero, 10b\n" /* Branch if more to do. */ 70 "nop\n" 71 "sync\n" /* Clear memory hazards. */ 72 "20:\n" 73 "bal 30f\n" 74 "nop\n" 75 "30:\n" 76 "daddiu $ra, $ra, 12\n" /* $ra has a value of $pc here. 77 Add offset of 12 to point to the 78 instruction after the last nop. 79 */ 80 "jr.hb $ra\n" /* Return, clearing instruction 81 hazards. */ 82 "nop\n" 83 ".set pop\n" 84 : [Addr] "+r"(Addr), [Size] "+r"(Size) 85 :: "at", "ra", "v0", "memory" 86 ); 87 } 88 #endif 89 #endif 90 91 /* 92 * The compiler generates calls to __clear_cache() when creating 93 * trampoline functions on the stack for use with nested functions. 94 * It is expected to invalidate the instruction cache for the 95 * specified range. 96 */ 97 98 void __clear_cache(void *start, void *end) { 99 #if __i386__ || __x86_64__ || defined(_M_IX86) || defined(_M_X64) 100 /* 101 * Intel processors have a unified instruction and data cache 102 * so there is nothing to do 103 */ 104 #elif defined(_WIN32) && (defined(__arm__) || defined(__aarch64__)) 105 FlushInstructionCache(GetCurrentProcess(), start, end - start); 106 #elif defined(__arm__) && !defined(__APPLE__) 107 #if defined(__FreeBSD__) || defined(__NetBSD__) 108 struct arm_sync_icache_args arg; 109 110 arg.addr = (uintptr_t)start; 111 arg.len = (uintptr_t)end - (uintptr_t)start; 112 113 sysarch(ARM_SYNC_ICACHE, &arg); 114 #elif defined(__linux__) 115 /* 116 * We used to include asm/unistd.h for the __ARM_NR_cacheflush define, but 117 * it also brought many other unused defines, as well as a dependency on 118 * kernel headers to be installed. 119 * 120 * This value is stable at least since Linux 3.13 and should remain so for 121 * compatibility reasons, warranting it's re-definition here. 122 */ 123 #define __ARM_NR_cacheflush 0x0f0002 124 register int start_reg __asm("r0") = (int) (intptr_t) start; 125 const register int end_reg __asm("r1") = (int) (intptr_t) end; 126 const register int flags __asm("r2") = 0; 127 const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush; 128 __asm __volatile("svc 0x0" 129 : "=r"(start_reg) 130 : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), 131 "r"(flags)); 132 assert(start_reg == 0 && "Cache flush syscall failed."); 133 #else 134 compilerrt_abort(); 135 #endif 136 #elif defined(__linux__) && defined(__mips__) 137 const uintptr_t start_int = (uintptr_t) start; 138 const uintptr_t end_int = (uintptr_t) end; 139 #if defined(__ANDROID__) && defined(__LP64__) 140 // Call synci implementation for short address range. 141 const uintptr_t address_range_limit = 256; 142 if ((end_int - start_int) <= address_range_limit) { 143 clear_mips_cache(start, (end_int - start_int)); 144 } else { 145 syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); 146 } 147 #else 148 syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); 149 #endif 150 #elif defined(__mips__) && defined(__OpenBSD__) 151 cacheflush(start, (uintptr_t)end - (uintptr_t)start, BCACHE); 152 #elif defined(__aarch64__) && !defined(__APPLE__) 153 uint64_t xstart = (uint64_t)(uintptr_t) start; 154 uint64_t xend = (uint64_t)(uintptr_t) end; 155 uint64_t addr; 156 157 // Get Cache Type Info 158 uint64_t ctr_el0; 159 __asm __volatile("mrs %0, ctr_el0" : "=r"(ctr_el0)); 160 161 /* 162 * dc & ic instructions must use 64bit registers so we don't use 163 * uintptr_t in case this runs in an IPL32 environment. 164 */ 165 const size_t dcache_line_size = 4 << ((ctr_el0 >> 16) & 15); 166 for (addr = xstart & ~(dcache_line_size - 1); addr < xend; 167 addr += dcache_line_size) 168 __asm __volatile("dc cvau, %0" :: "r"(addr)); 169 __asm __volatile("dsb ish"); 170 171 const size_t icache_line_size = 4 << ((ctr_el0 >> 0) & 15); 172 for (addr = xstart & ~(icache_line_size - 1); addr < xend; 173 addr += icache_line_size) 174 __asm __volatile("ic ivau, %0" :: "r"(addr)); 175 __asm __volatile("isb sy"); 176 #elif defined (__powerpc64__) 177 const size_t line_size = 32; 178 const size_t len = (uintptr_t)end - (uintptr_t)start; 179 180 const uintptr_t mask = ~(line_size - 1); 181 const uintptr_t start_line = ((uintptr_t)start) & mask; 182 const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask; 183 184 for (uintptr_t line = start_line; line < end_line; line += line_size) 185 __asm__ volatile("dcbf 0, %0" : : "r"(line)); 186 __asm__ volatile("sync"); 187 188 for (uintptr_t line = start_line; line < end_line; line += line_size) 189 __asm__ volatile("icbi 0, %0" : : "r"(line)); 190 __asm__ volatile("isync"); 191 #else 192 #if __APPLE__ 193 /* On Darwin, sys_icache_invalidate() provides this functionality */ 194 sys_icache_invalidate(start, end-start); 195 #else 196 compilerrt_abort(); 197 #endif 198 #endif 199 } 200