1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Kernel module help for x86. 3 Copyright (C) 2001 Rusty Russell. 4 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/moduleloader.h> 10 #include <linux/elf.h> 11 #include <linux/vmalloc.h> 12 #include <linux/fs.h> 13 #include <linux/string.h> 14 #include <linux/kernel.h> 15 #include <linux/kasan.h> 16 #include <linux/bug.h> 17 #include <linux/mm.h> 18 #include <linux/gfp.h> 19 #include <linux/jump_label.h> 20 #include <linux/random.h> 21 #include <linux/memory.h> 22 #include <linux/stackprotector.h> 23 24 #include <asm/text-patching.h> 25 #include <asm/page.h> 26 #include <asm/setup.h> 27 #include <asm/unwind.h> 28 29 #if 0 30 #define DEBUGP(fmt, ...) \ 31 printk(KERN_DEBUG fmt, ##__VA_ARGS__) 32 #else 33 #define DEBUGP(fmt, ...) \ 34 do { \ 35 if (0) \ 36 printk(KERN_DEBUG fmt, ##__VA_ARGS__); \ 37 } while (0) 38 #endif 39 40 #ifdef CONFIG_X86_32 41 int apply_relocate(Elf32_Shdr *sechdrs, 42 const char *strtab, 43 unsigned int symindex, 44 unsigned int relsec, 45 struct module *me) 46 { 47 unsigned int i; 48 Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr; 49 Elf32_Sym *sym; 50 uint32_t *location; 51 52 DEBUGP("Applying relocate section %u to %u\n", 53 relsec, sechdrs[relsec].sh_info); 54 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 55 /* This is where to make the change */ 56 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 57 + rel[i].r_offset; 58 /* This is the symbol it is referring to. Note that all 59 undefined symbols have been resolved. */ 60 sym = (Elf32_Sym *)sechdrs[symindex].sh_addr 61 + ELF32_R_SYM(rel[i].r_info); 62 63 switch (ELF32_R_TYPE(rel[i].r_info)) { 64 case R_386_32: 65 /* We add the value into the location given */ 66 *location += sym->st_value; 67 break; 68 case R_386_PC32: 69 case R_386_PLT32: 70 /* Add the value, subtract its position */ 71 *location += sym->st_value - (uint32_t)location; 72 break; 73 default: 74 pr_err("%s: Unknown relocation: %u\n", 75 me->name, ELF32_R_TYPE(rel[i].r_info)); 76 return -ENOEXEC; 77 } 78 } 79 return 0; 80 } 81 #else /*X86_64*/ 82 static int __write_relocate_add(Elf64_Shdr *sechdrs, 83 const char *strtab, 84 unsigned int symindex, 85 unsigned int relsec, 86 struct module *me, 87 void *(*write)(void *dest, const void *src, size_t len), 88 bool apply) 89 { 90 unsigned int i; 91 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr; 92 Elf64_Sym *sym; 93 void *loc; 94 u64 val; 95 u64 zero = 0ULL; 96 97 DEBUGP("%s relocate section %u to %u\n", 98 apply ? "Applying" : "Clearing", 99 relsec, sechdrs[relsec].sh_info); 100 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 101 size_t size; 102 103 /* This is where to make the change */ 104 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 105 + rel[i].r_offset; 106 107 /* This is the symbol it is referring to. Note that all 108 undefined symbols have been resolved. */ 109 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 110 + ELF64_R_SYM(rel[i].r_info); 111 112 DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n", 113 (int)ELF64_R_TYPE(rel[i].r_info), 114 sym->st_value, rel[i].r_addend, (u64)loc); 115 116 val = sym->st_value + rel[i].r_addend; 117 118 switch (ELF64_R_TYPE(rel[i].r_info)) { 119 case R_X86_64_NONE: 120 continue; /* nothing to write */ 121 case R_X86_64_64: 122 size = 8; 123 break; 124 case R_X86_64_32: 125 if (val != *(u32 *)&val) 126 goto overflow; 127 size = 4; 128 break; 129 case R_X86_64_32S: 130 if ((s64)val != *(s32 *)&val) 131 goto overflow; 132 size = 4; 133 break; 134 #if defined(CONFIG_STACKPROTECTOR) && \ 135 defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 170000 136 case R_X86_64_REX_GOTPCRELX: { 137 static unsigned long __percpu *const addr = &__stack_chk_guard; 138 139 if (sym->st_value != (u64)addr) { 140 pr_err("%s: Unsupported GOTPCREL relocation\n", me->name); 141 return -ENOEXEC; 142 } 143 144 val = (u64)&addr + rel[i].r_addend; 145 fallthrough; 146 } 147 #endif 148 case R_X86_64_PC32: 149 case R_X86_64_PLT32: 150 val -= (u64)loc; 151 size = 4; 152 break; 153 case R_X86_64_PC64: 154 val -= (u64)loc; 155 size = 8; 156 break; 157 default: 158 pr_err("%s: Unknown rela relocation: %llu\n", 159 me->name, ELF64_R_TYPE(rel[i].r_info)); 160 return -ENOEXEC; 161 } 162 163 if (apply) { 164 void *wr_loc = module_writable_address(me, loc); 165 166 if (memcmp(wr_loc, &zero, size)) { 167 pr_err("x86/modules: Invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n", 168 (int)ELF64_R_TYPE(rel[i].r_info), loc, val); 169 return -ENOEXEC; 170 } 171 write(wr_loc, &val, size); 172 } else { 173 if (memcmp(loc, &val, size)) { 174 pr_warn("x86/modules: Invalid relocation target, existing value does not match expected value for type %d, loc %p, val %Lx\n", 175 (int)ELF64_R_TYPE(rel[i].r_info), loc, val); 176 return -ENOEXEC; 177 } 178 /* FIXME: needs care for ROX module allocations */ 179 write(loc, &zero, size); 180 } 181 } 182 return 0; 183 184 overflow: 185 pr_err("overflow in relocation type %d val %Lx\n", 186 (int)ELF64_R_TYPE(rel[i].r_info), val); 187 pr_err("`%s' likely not compiled with -mcmodel=kernel\n", 188 me->name); 189 return -ENOEXEC; 190 } 191 192 static int write_relocate_add(Elf64_Shdr *sechdrs, 193 const char *strtab, 194 unsigned int symindex, 195 unsigned int relsec, 196 struct module *me, 197 bool apply) 198 { 199 int ret; 200 bool early = me->state == MODULE_STATE_UNFORMED; 201 void *(*write)(void *, const void *, size_t) = memcpy; 202 203 if (!early) { 204 write = text_poke; 205 mutex_lock(&text_mutex); 206 } 207 208 ret = __write_relocate_add(sechdrs, strtab, symindex, relsec, me, 209 write, apply); 210 211 if (!early) { 212 text_poke_sync(); 213 mutex_unlock(&text_mutex); 214 } 215 216 return ret; 217 } 218 219 int apply_relocate_add(Elf64_Shdr *sechdrs, 220 const char *strtab, 221 unsigned int symindex, 222 unsigned int relsec, 223 struct module *me) 224 { 225 return write_relocate_add(sechdrs, strtab, symindex, relsec, me, true); 226 } 227 228 #ifdef CONFIG_LIVEPATCH 229 void clear_relocate_add(Elf64_Shdr *sechdrs, 230 const char *strtab, 231 unsigned int symindex, 232 unsigned int relsec, 233 struct module *me) 234 { 235 write_relocate_add(sechdrs, strtab, symindex, relsec, me, false); 236 } 237 #endif 238 239 #endif 240 241 int module_finalize(const Elf_Ehdr *hdr, 242 const Elf_Shdr *sechdrs, 243 struct module *me) 244 { 245 const Elf_Shdr *s, *alt = NULL, 246 *orc = NULL, *orc_ip = NULL, 247 *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL, 248 *calls = NULL, *cfi = NULL; 249 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 250 251 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 252 if (!strcmp(".altinstructions", secstrings + s->sh_name)) 253 alt = s; 254 if (!strcmp(".orc_unwind", secstrings + s->sh_name)) 255 orc = s; 256 if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) 257 orc_ip = s; 258 if (!strcmp(".retpoline_sites", secstrings + s->sh_name)) 259 retpolines = s; 260 if (!strcmp(".return_sites", secstrings + s->sh_name)) 261 returns = s; 262 if (!strcmp(".call_sites", secstrings + s->sh_name)) 263 calls = s; 264 if (!strcmp(".cfi_sites", secstrings + s->sh_name)) 265 cfi = s; 266 if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name)) 267 ibt_endbr = s; 268 } 269 270 if (retpolines || cfi) { 271 void *rseg = NULL, *cseg = NULL; 272 unsigned int rsize = 0, csize = 0; 273 274 if (retpolines) { 275 rseg = (void *)retpolines->sh_addr; 276 rsize = retpolines->sh_size; 277 } 278 279 if (cfi) { 280 cseg = (void *)cfi->sh_addr; 281 csize = cfi->sh_size; 282 } 283 284 apply_fineibt(rseg, rseg + rsize, cseg, cseg + csize, me); 285 } 286 if (retpolines) { 287 void *rseg = (void *)retpolines->sh_addr; 288 apply_retpolines(rseg, rseg + retpolines->sh_size, me); 289 } 290 if (returns) { 291 void *rseg = (void *)returns->sh_addr; 292 apply_returns(rseg, rseg + returns->sh_size, me); 293 } 294 if (alt) { 295 /* patch .altinstructions */ 296 void *aseg = (void *)alt->sh_addr; 297 apply_alternatives(aseg, aseg + alt->sh_size, me); 298 } 299 if (calls || alt) { 300 struct callthunk_sites cs = {}; 301 302 if (calls) { 303 cs.call_start = (void *)calls->sh_addr; 304 cs.call_end = (void *)calls->sh_addr + calls->sh_size; 305 } 306 307 if (alt) { 308 cs.alt_start = (void *)alt->sh_addr; 309 cs.alt_end = (void *)alt->sh_addr + alt->sh_size; 310 } 311 312 callthunks_patch_module_calls(&cs, me); 313 } 314 if (ibt_endbr) { 315 void *iseg = (void *)ibt_endbr->sh_addr; 316 apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size, me); 317 } 318 319 if (orc && orc_ip) 320 unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size, 321 (void *)orc->sh_addr, orc->sh_size); 322 323 return 0; 324 } 325 326 int module_post_finalize(const Elf_Ehdr *hdr, 327 const Elf_Shdr *sechdrs, 328 struct module *me) 329 { 330 const Elf_Shdr *s, *locks = NULL; 331 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 332 333 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 334 if (!strcmp(".smp_locks", secstrings + s->sh_name)) 335 locks = s; 336 } 337 338 if (locks) { 339 void *lseg = (void *)locks->sh_addr; 340 void *text = me->mem[MOD_TEXT].base; 341 void *text_end = text + me->mem[MOD_TEXT].size; 342 alternatives_smp_module_add(me, me->name, 343 lseg, lseg + locks->sh_size, 344 text, text_end); 345 } 346 347 return 0; 348 } 349 350 void module_arch_cleanup(struct module *mod) 351 { 352 alternatives_smp_module_del(mod); 353 } 354