1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AArch64 loadable module support. 4 * 5 * Copyright (C) 2012 ARM Limited 6 * 7 * Author: Will Deacon <[email protected]> 8 */ 9 10 #define pr_fmt(fmt) "Modules: " fmt 11 12 #include <linux/bitops.h> 13 #include <linux/elf.h> 14 #include <linux/ftrace.h> 15 #include <linux/gfp.h> 16 #include <linux/kasan.h> 17 #include <linux/kernel.h> 18 #include <linux/mm.h> 19 #include <linux/moduleloader.h> 20 #include <linux/random.h> 21 #include <linux/scs.h> 22 #include <linux/vmalloc.h> 23 #include <linux/execmem.h> 24 25 #include <asm/alternative.h> 26 #include <asm/insn.h> 27 #include <asm/scs.h> 28 #include <asm/sections.h> 29 30 static u64 module_direct_base __ro_after_init = 0; 31 static u64 module_plt_base __ro_after_init = 0; 32 33 /* 34 * Choose a random page-aligned base address for a window of 'size' bytes which 35 * entirely contains the interval [start, end - 1]. 36 */ 37 static u64 __init random_bounding_box(u64 size, u64 start, u64 end) 38 { 39 u64 max_pgoff, pgoff; 40 41 if ((end - start) >= size) 42 return 0; 43 44 max_pgoff = (size - (end - start)) / PAGE_SIZE; 45 pgoff = get_random_u32_inclusive(0, max_pgoff); 46 47 return start - pgoff * PAGE_SIZE; 48 } 49 50 /* 51 * Modules may directly reference data and text anywhere within the kernel 52 * image and other modules. References using PREL32 relocations have a +/-2G 53 * range, and so we need to ensure that the entire kernel image and all modules 54 * fall within a 2G window such that these are always within range. 55 * 56 * Modules may directly branch to functions and code within the kernel text, 57 * and to functions and code within other modules. These branches will use 58 * CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure 59 * that the entire kernel text and all module text falls within a 128M window 60 * such that these are always within range. With PLTs, we can expand this to a 61 * 2G window. 62 * 63 * We chose the 128M region to surround the entire kernel image (rather than 64 * just the text) as using the same bounds for the 128M and 2G regions ensures 65 * by construction that we never select a 128M region that is not a subset of 66 * the 2G region. For very large and unusual kernel configurations this means 67 * we may fall back to PLTs where they could have been avoided, but this keeps 68 * the logic significantly simpler. 69 */ 70 static int __init module_init_limits(void) 71 { 72 u64 kernel_end = (u64)_end; 73 u64 kernel_start = (u64)_text; 74 u64 kernel_size = kernel_end - kernel_start; 75 76 /* 77 * The default modules region is placed immediately below the kernel 78 * image, and is large enough to use the full 2G relocation range. 79 */ 80 BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END); 81 BUILD_BUG_ON(MODULES_VSIZE < SZ_2G); 82 83 if (!kaslr_enabled()) { 84 if (kernel_size < SZ_128M) 85 module_direct_base = kernel_end - SZ_128M; 86 if (kernel_size < SZ_2G) 87 module_plt_base = kernel_end - SZ_2G; 88 } else { 89 u64 min = kernel_start; 90 u64 max = kernel_end; 91 92 if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { 93 pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n"); 94 } else { 95 module_direct_base = random_bounding_box(SZ_128M, min, max); 96 if (module_direct_base) { 97 min = module_direct_base; 98 max = module_direct_base + SZ_128M; 99 } 100 } 101 102 module_plt_base = random_bounding_box(SZ_2G, min, max); 103 } 104 105 pr_info("%llu pages in range for non-PLT usage", 106 module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0); 107 pr_info("%llu pages in range for PLT usage", 108 module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0); 109 110 return 0; 111 } 112 113 static struct execmem_info execmem_info __ro_after_init; 114 115 struct execmem_info __init *execmem_arch_setup(void) 116 { 117 unsigned long fallback_start = 0, fallback_end = 0; 118 unsigned long start = 0, end = 0; 119 120 module_init_limits(); 121 122 /* 123 * Where possible, prefer to allocate within direct branch range of the 124 * kernel such that no PLTs are necessary. 125 */ 126 if (module_direct_base) { 127 start = module_direct_base; 128 end = module_direct_base + SZ_128M; 129 130 if (module_plt_base) { 131 fallback_start = module_plt_base; 132 fallback_end = module_plt_base + SZ_2G; 133 } 134 } else if (module_plt_base) { 135 start = module_plt_base; 136 end = module_plt_base + SZ_2G; 137 } 138 139 execmem_info = (struct execmem_info){ 140 .ranges = { 141 [EXECMEM_DEFAULT] = { 142 .start = start, 143 .end = end, 144 .pgprot = PAGE_KERNEL, 145 .alignment = 1, 146 .fallback_start = fallback_start, 147 .fallback_end = fallback_end, 148 }, 149 [EXECMEM_KPROBES] = { 150 .start = VMALLOC_START, 151 .end = VMALLOC_END, 152 .pgprot = PAGE_KERNEL_ROX, 153 .alignment = 1, 154 }, 155 [EXECMEM_BPF] = { 156 .start = VMALLOC_START, 157 .end = VMALLOC_END, 158 .pgprot = PAGE_KERNEL, 159 .alignment = 1, 160 }, 161 }, 162 }; 163 164 return &execmem_info; 165 } 166 167 enum aarch64_reloc_op { 168 RELOC_OP_NONE, 169 RELOC_OP_ABS, 170 RELOC_OP_PREL, 171 RELOC_OP_PAGE, 172 }; 173 174 static u64 do_reloc(enum aarch64_reloc_op reloc_op, __le32 *place, u64 val) 175 { 176 switch (reloc_op) { 177 case RELOC_OP_ABS: 178 return val; 179 case RELOC_OP_PREL: 180 return val - (u64)place; 181 case RELOC_OP_PAGE: 182 return (val & ~0xfff) - ((u64)place & ~0xfff); 183 case RELOC_OP_NONE: 184 return 0; 185 } 186 187 pr_err("do_reloc: unknown relocation operation %d\n", reloc_op); 188 return 0; 189 } 190 191 static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) 192 { 193 s64 sval = do_reloc(op, place, val); 194 195 /* 196 * The ELF psABI for AArch64 documents the 16-bit and 32-bit place 197 * relative and absolute relocations as having a range of [-2^15, 2^16) 198 * or [-2^31, 2^32), respectively. However, in order to be able to 199 * detect overflows reliably, we have to choose whether we interpret 200 * such quantities as signed or as unsigned, and stick with it. 201 * The way we organize our address space requires a signed 202 * interpretation of 32-bit relative references, so let's use that 203 * for all R_AARCH64_PRELxx relocations. This means our upper 204 * bound for overflow detection should be Sxx_MAX rather than Uxx_MAX. 205 */ 206 207 switch (len) { 208 case 16: 209 *(s16 *)place = sval; 210 switch (op) { 211 case RELOC_OP_ABS: 212 if (sval < 0 || sval > U16_MAX) 213 return -ERANGE; 214 break; 215 case RELOC_OP_PREL: 216 if (sval < S16_MIN || sval > S16_MAX) 217 return -ERANGE; 218 break; 219 default: 220 pr_err("Invalid 16-bit data relocation (%d)\n", op); 221 return 0; 222 } 223 break; 224 case 32: 225 *(s32 *)place = sval; 226 switch (op) { 227 case RELOC_OP_ABS: 228 if (sval < 0 || sval > U32_MAX) 229 return -ERANGE; 230 break; 231 case RELOC_OP_PREL: 232 if (sval < S32_MIN || sval > S32_MAX) 233 return -ERANGE; 234 break; 235 default: 236 pr_err("Invalid 32-bit data relocation (%d)\n", op); 237 return 0; 238 } 239 break; 240 case 64: 241 *(s64 *)place = sval; 242 break; 243 default: 244 pr_err("Invalid length (%d) for data relocation\n", len); 245 return 0; 246 } 247 return 0; 248 } 249 250 enum aarch64_insn_movw_imm_type { 251 AARCH64_INSN_IMM_MOVNZ, 252 AARCH64_INSN_IMM_MOVKZ, 253 }; 254 255 static int reloc_insn_movw(enum aarch64_reloc_op op, __le32 *place, u64 val, 256 int lsb, enum aarch64_insn_movw_imm_type imm_type) 257 { 258 u64 imm; 259 s64 sval; 260 u32 insn = le32_to_cpu(*place); 261 262 sval = do_reloc(op, place, val); 263 imm = sval >> lsb; 264 265 if (imm_type == AARCH64_INSN_IMM_MOVNZ) { 266 /* 267 * For signed MOVW relocations, we have to manipulate the 268 * instruction encoding depending on whether or not the 269 * immediate is less than zero. 270 */ 271 insn &= ~(3 << 29); 272 if (sval >= 0) { 273 /* >=0: Set the instruction to MOVZ (opcode 10b). */ 274 insn |= 2 << 29; 275 } else { 276 /* 277 * <0: Set the instruction to MOVN (opcode 00b). 278 * Since we've masked the opcode already, we 279 * don't need to do anything other than 280 * inverting the new immediate field. 281 */ 282 imm = ~imm; 283 } 284 } 285 286 /* Update the instruction with the new encoding. */ 287 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); 288 *place = cpu_to_le32(insn); 289 290 if (imm > U16_MAX) 291 return -ERANGE; 292 293 return 0; 294 } 295 296 static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val, 297 int lsb, int len, enum aarch64_insn_imm_type imm_type) 298 { 299 u64 imm, imm_mask; 300 s64 sval; 301 u32 insn = le32_to_cpu(*place); 302 303 /* Calculate the relocation value. */ 304 sval = do_reloc(op, place, val); 305 sval >>= lsb; 306 307 /* Extract the value bits and shift them to bit 0. */ 308 imm_mask = (BIT(lsb + len) - 1) >> lsb; 309 imm = sval & imm_mask; 310 311 /* Update the instruction's immediate field. */ 312 insn = aarch64_insn_encode_immediate(imm_type, insn, imm); 313 *place = cpu_to_le32(insn); 314 315 /* 316 * Extract the upper value bits (including the sign bit) and 317 * shift them to bit 0. 318 */ 319 sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1); 320 321 /* 322 * Overflow has occurred if the upper bits are not all equal to 323 * the sign bit of the value. 324 */ 325 if ((u64)(sval + 1) >= 2) 326 return -ERANGE; 327 328 return 0; 329 } 330 331 static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs, 332 __le32 *place, u64 val) 333 { 334 u32 insn; 335 336 if (!is_forbidden_offset_for_adrp(place)) 337 return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21, 338 AARCH64_INSN_IMM_ADR); 339 340 /* patch ADRP to ADR if it is in range */ 341 if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21, 342 AARCH64_INSN_IMM_ADR)) { 343 insn = le32_to_cpu(*place); 344 insn &= ~BIT(31); 345 } else { 346 /* out of range for ADR -> emit a veneer */ 347 val = module_emit_veneer_for_adrp(mod, sechdrs, place, val & ~0xfff); 348 if (!val) 349 return -ENOEXEC; 350 insn = aarch64_insn_gen_branch_imm((u64)place, val, 351 AARCH64_INSN_BRANCH_NOLINK); 352 } 353 354 *place = cpu_to_le32(insn); 355 return 0; 356 } 357 358 int apply_relocate_add(Elf64_Shdr *sechdrs, 359 const char *strtab, 360 unsigned int symindex, 361 unsigned int relsec, 362 struct module *me) 363 { 364 unsigned int i; 365 int ovf; 366 bool overflow_check; 367 Elf64_Sym *sym; 368 void *loc; 369 u64 val; 370 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr; 371 372 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 373 /* loc corresponds to P in the AArch64 ELF document. */ 374 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 375 + rel[i].r_offset; 376 377 /* sym is the ELF symbol we're referring to. */ 378 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 379 + ELF64_R_SYM(rel[i].r_info); 380 381 /* val corresponds to (S + A) in the AArch64 ELF document. */ 382 val = sym->st_value + rel[i].r_addend; 383 384 /* Check for overflow by default. */ 385 overflow_check = true; 386 387 /* Perform the static relocation. */ 388 switch (ELF64_R_TYPE(rel[i].r_info)) { 389 /* Null relocations. */ 390 case R_ARM_NONE: 391 case R_AARCH64_NONE: 392 ovf = 0; 393 break; 394 395 /* Data relocations. */ 396 case R_AARCH64_ABS64: 397 overflow_check = false; 398 ovf = reloc_data(RELOC_OP_ABS, loc, val, 64); 399 break; 400 case R_AARCH64_ABS32: 401 ovf = reloc_data(RELOC_OP_ABS, loc, val, 32); 402 break; 403 case R_AARCH64_ABS16: 404 ovf = reloc_data(RELOC_OP_ABS, loc, val, 16); 405 break; 406 case R_AARCH64_PREL64: 407 overflow_check = false; 408 ovf = reloc_data(RELOC_OP_PREL, loc, val, 64); 409 break; 410 case R_AARCH64_PREL32: 411 ovf = reloc_data(RELOC_OP_PREL, loc, val, 32); 412 break; 413 case R_AARCH64_PREL16: 414 ovf = reloc_data(RELOC_OP_PREL, loc, val, 16); 415 break; 416 417 /* MOVW instruction relocations. */ 418 case R_AARCH64_MOVW_UABS_G0_NC: 419 overflow_check = false; 420 fallthrough; 421 case R_AARCH64_MOVW_UABS_G0: 422 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 423 AARCH64_INSN_IMM_MOVKZ); 424 break; 425 case R_AARCH64_MOVW_UABS_G1_NC: 426 overflow_check = false; 427 fallthrough; 428 case R_AARCH64_MOVW_UABS_G1: 429 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 430 AARCH64_INSN_IMM_MOVKZ); 431 break; 432 case R_AARCH64_MOVW_UABS_G2_NC: 433 overflow_check = false; 434 fallthrough; 435 case R_AARCH64_MOVW_UABS_G2: 436 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 437 AARCH64_INSN_IMM_MOVKZ); 438 break; 439 case R_AARCH64_MOVW_UABS_G3: 440 /* We're using the top bits so we can't overflow. */ 441 overflow_check = false; 442 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, 443 AARCH64_INSN_IMM_MOVKZ); 444 break; 445 case R_AARCH64_MOVW_SABS_G0: 446 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 447 AARCH64_INSN_IMM_MOVNZ); 448 break; 449 case R_AARCH64_MOVW_SABS_G1: 450 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 451 AARCH64_INSN_IMM_MOVNZ); 452 break; 453 case R_AARCH64_MOVW_SABS_G2: 454 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 455 AARCH64_INSN_IMM_MOVNZ); 456 break; 457 case R_AARCH64_MOVW_PREL_G0_NC: 458 overflow_check = false; 459 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 460 AARCH64_INSN_IMM_MOVKZ); 461 break; 462 case R_AARCH64_MOVW_PREL_G0: 463 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 464 AARCH64_INSN_IMM_MOVNZ); 465 break; 466 case R_AARCH64_MOVW_PREL_G1_NC: 467 overflow_check = false; 468 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 469 AARCH64_INSN_IMM_MOVKZ); 470 break; 471 case R_AARCH64_MOVW_PREL_G1: 472 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 473 AARCH64_INSN_IMM_MOVNZ); 474 break; 475 case R_AARCH64_MOVW_PREL_G2_NC: 476 overflow_check = false; 477 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 478 AARCH64_INSN_IMM_MOVKZ); 479 break; 480 case R_AARCH64_MOVW_PREL_G2: 481 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 482 AARCH64_INSN_IMM_MOVNZ); 483 break; 484 case R_AARCH64_MOVW_PREL_G3: 485 /* We're using the top bits so we can't overflow. */ 486 overflow_check = false; 487 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48, 488 AARCH64_INSN_IMM_MOVNZ); 489 break; 490 491 /* Immediate instruction relocations. */ 492 case R_AARCH64_LD_PREL_LO19: 493 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 494 AARCH64_INSN_IMM_19); 495 break; 496 case R_AARCH64_ADR_PREL_LO21: 497 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21, 498 AARCH64_INSN_IMM_ADR); 499 break; 500 case R_AARCH64_ADR_PREL_PG_HI21_NC: 501 overflow_check = false; 502 fallthrough; 503 case R_AARCH64_ADR_PREL_PG_HI21: 504 ovf = reloc_insn_adrp(me, sechdrs, loc, val); 505 if (ovf && ovf != -ERANGE) 506 return ovf; 507 break; 508 case R_AARCH64_ADD_ABS_LO12_NC: 509 case R_AARCH64_LDST8_ABS_LO12_NC: 510 overflow_check = false; 511 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12, 512 AARCH64_INSN_IMM_12); 513 break; 514 case R_AARCH64_LDST16_ABS_LO12_NC: 515 overflow_check = false; 516 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11, 517 AARCH64_INSN_IMM_12); 518 break; 519 case R_AARCH64_LDST32_ABS_LO12_NC: 520 overflow_check = false; 521 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10, 522 AARCH64_INSN_IMM_12); 523 break; 524 case R_AARCH64_LDST64_ABS_LO12_NC: 525 overflow_check = false; 526 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, 527 AARCH64_INSN_IMM_12); 528 break; 529 case R_AARCH64_LDST128_ABS_LO12_NC: 530 overflow_check = false; 531 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8, 532 AARCH64_INSN_IMM_12); 533 break; 534 case R_AARCH64_TSTBR14: 535 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14, 536 AARCH64_INSN_IMM_14); 537 break; 538 case R_AARCH64_CONDBR19: 539 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 540 AARCH64_INSN_IMM_19); 541 break; 542 case R_AARCH64_JUMP26: 543 case R_AARCH64_CALL26: 544 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, 545 AARCH64_INSN_IMM_26); 546 if (ovf == -ERANGE) { 547 val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym); 548 if (!val) 549 return -ENOEXEC; 550 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 551 26, AARCH64_INSN_IMM_26); 552 } 553 break; 554 555 default: 556 pr_err("module %s: unsupported RELA relocation: %llu\n", 557 me->name, ELF64_R_TYPE(rel[i].r_info)); 558 return -ENOEXEC; 559 } 560 561 if (overflow_check && ovf == -ERANGE) 562 goto overflow; 563 564 } 565 566 return 0; 567 568 overflow: 569 pr_err("module %s: overflow in relocation type %d val %Lx\n", 570 me->name, (int)ELF64_R_TYPE(rel[i].r_info), val); 571 return -ENOEXEC; 572 } 573 574 static inline void __init_plt(struct plt_entry *plt, unsigned long addr) 575 { 576 *plt = get_plt_entry(addr, plt); 577 } 578 579 static int module_init_ftrace_plt(const Elf_Ehdr *hdr, 580 const Elf_Shdr *sechdrs, 581 struct module *mod) 582 { 583 #if defined(CONFIG_DYNAMIC_FTRACE) 584 const Elf_Shdr *s; 585 struct plt_entry *plts; 586 587 s = find_section(hdr, sechdrs, ".text.ftrace_trampoline"); 588 if (!s) 589 return -ENOEXEC; 590 591 plts = (void *)s->sh_addr; 592 593 __init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR); 594 595 mod->arch.ftrace_trampolines = plts; 596 #endif 597 return 0; 598 } 599 600 int module_finalize(const Elf_Ehdr *hdr, 601 const Elf_Shdr *sechdrs, 602 struct module *me) 603 { 604 const Elf_Shdr *s; 605 s = find_section(hdr, sechdrs, ".altinstructions"); 606 if (s) 607 apply_alternatives_module((void *)s->sh_addr, s->sh_size); 608 609 if (scs_is_dynamic()) { 610 s = find_section(hdr, sechdrs, ".init.eh_frame"); 611 if (s) 612 __pi_scs_patch((void *)s->sh_addr, s->sh_size); 613 } 614 615 return module_init_ftrace_plt(hdr, sechdrs, me); 616 } 617