1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <[email protected]> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/mman.h> 25 #include <sys/stat.h> 26 #include <getopt.h> 27 #include <elf.h> 28 #include <fcntl.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <stdbool.h> 32 #include <string.h> 33 #include <unistd.h> 34 #include <errno.h> 35 #include <pthread.h> 36 37 #include <tools/be_byteshift.h> 38 #include <tools/le_byteshift.h> 39 40 #ifndef EM_ARCOMPACT 41 #define EM_ARCOMPACT 93 42 #endif 43 44 #ifndef EM_XTENSA 45 #define EM_XTENSA 94 46 #endif 47 48 #ifndef EM_AARCH64 49 #define EM_AARCH64 183 50 #endif 51 52 #ifndef EM_MICROBLAZE 53 #define EM_MICROBLAZE 189 54 #endif 55 56 #ifndef EM_ARCV2 57 #define EM_ARCV2 195 58 #endif 59 60 #ifndef EM_RISCV 61 #define EM_RISCV 243 62 #endif 63 64 #ifndef EM_LOONGARCH 65 #define EM_LOONGARCH 258 66 #endif 67 68 typedef union { 69 Elf32_Ehdr e32; 70 Elf64_Ehdr e64; 71 } Elf_Ehdr; 72 73 typedef union { 74 Elf32_Shdr e32; 75 Elf64_Shdr e64; 76 } Elf_Shdr; 77 78 typedef union { 79 Elf32_Sym e32; 80 Elf64_Sym e64; 81 } Elf_Sym; 82 83 typedef union { 84 Elf32_Rela e32; 85 Elf64_Rela e64; 86 } Elf_Rela; 87 88 static uint32_t (*r)(const uint32_t *); 89 static uint16_t (*r2)(const uint16_t *); 90 static uint64_t (*r8)(const uint64_t *); 91 static void (*w)(uint32_t, uint32_t *); 92 static void (*w8)(uint64_t, uint64_t *); 93 typedef void (*table_sort_t)(char *, int); 94 95 static struct elf_funcs { 96 int (*compare_extable)(const void *a, const void *b); 97 uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr); 98 uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr); 99 uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr); 100 uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr); 101 uint64_t (*shdr_addr)(Elf_Shdr *shdr); 102 uint64_t (*shdr_offset)(Elf_Shdr *shdr); 103 uint64_t (*shdr_size)(Elf_Shdr *shdr); 104 uint64_t (*shdr_entsize)(Elf_Shdr *shdr); 105 uint32_t (*shdr_link)(Elf_Shdr *shdr); 106 uint32_t (*shdr_name)(Elf_Shdr *shdr); 107 uint32_t (*shdr_type)(Elf_Shdr *shdr); 108 uint8_t (*sym_type)(Elf_Sym *sym); 109 uint32_t (*sym_name)(Elf_Sym *sym); 110 uint64_t (*sym_value)(Elf_Sym *sym); 111 uint16_t (*sym_shndx)(Elf_Sym *sym); 112 uint64_t (*rela_offset)(Elf_Rela *rela); 113 uint64_t (*rela_info)(Elf_Rela *rela); 114 uint64_t (*rela_addend)(Elf_Rela *rela); 115 void (*rela_write_addend)(Elf_Rela *rela, uint64_t val); 116 } e; 117 118 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) 119 { 120 return r8(&ehdr->e64.e_shoff); 121 } 122 123 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr) 124 { 125 return r(&ehdr->e32.e_shoff); 126 } 127 128 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr) 129 { 130 return e.ehdr_shoff(ehdr); 131 } 132 133 #define EHDR_HALF(fn_name) \ 134 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \ 135 { \ 136 return r2(&ehdr->e64.e_##fn_name); \ 137 } \ 138 \ 139 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \ 140 { \ 141 return r2(&ehdr->e32.e_##fn_name); \ 142 } \ 143 \ 144 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr) \ 145 { \ 146 return e.ehdr_##fn_name(ehdr); \ 147 } 148 149 EHDR_HALF(shentsize) 150 EHDR_HALF(shstrndx) 151 EHDR_HALF(shnum) 152 153 #define SHDR_WORD(fn_name) \ 154 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 155 { \ 156 return r(&shdr->e64.sh_##fn_name); \ 157 } \ 158 \ 159 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 160 { \ 161 return r(&shdr->e32.sh_##fn_name); \ 162 } \ 163 \ 164 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 165 { \ 166 return e.shdr_##fn_name(shdr); \ 167 } 168 169 #define SHDR_ADDR(fn_name) \ 170 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \ 171 { \ 172 return r8(&shdr->e64.sh_##fn_name); \ 173 } \ 174 \ 175 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \ 176 { \ 177 return r(&shdr->e32.sh_##fn_name); \ 178 } \ 179 \ 180 static uint64_t shdr_##fn_name(Elf_Shdr *shdr) \ 181 { \ 182 return e.shdr_##fn_name(shdr); \ 183 } 184 185 #define SHDR_WORD(fn_name) \ 186 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 187 { \ 188 return r(&shdr->e64.sh_##fn_name); \ 189 } \ 190 \ 191 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 192 { \ 193 return r(&shdr->e32.sh_##fn_name); \ 194 } \ 195 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 196 { \ 197 return e.shdr_##fn_name(shdr); \ 198 } 199 200 SHDR_ADDR(addr) 201 SHDR_ADDR(offset) 202 SHDR_ADDR(size) 203 SHDR_ADDR(entsize) 204 205 SHDR_WORD(link) 206 SHDR_WORD(name) 207 SHDR_WORD(type) 208 209 #define SYM_ADDR(fn_name) \ 210 static uint64_t sym64_##fn_name(Elf_Sym *sym) \ 211 { \ 212 return r8(&sym->e64.st_##fn_name); \ 213 } \ 214 \ 215 static uint64_t sym32_##fn_name(Elf_Sym *sym) \ 216 { \ 217 return r(&sym->e32.st_##fn_name); \ 218 } \ 219 \ 220 static uint64_t sym_##fn_name(Elf_Sym *sym) \ 221 { \ 222 return e.sym_##fn_name(sym); \ 223 } 224 225 #define SYM_WORD(fn_name) \ 226 static uint32_t sym64_##fn_name(Elf_Sym *sym) \ 227 { \ 228 return r(&sym->e64.st_##fn_name); \ 229 } \ 230 \ 231 static uint32_t sym32_##fn_name(Elf_Sym *sym) \ 232 { \ 233 return r(&sym->e32.st_##fn_name); \ 234 } \ 235 \ 236 static uint32_t sym_##fn_name(Elf_Sym *sym) \ 237 { \ 238 return e.sym_##fn_name(sym); \ 239 } 240 241 #define SYM_HALF(fn_name) \ 242 static uint16_t sym64_##fn_name(Elf_Sym *sym) \ 243 { \ 244 return r2(&sym->e64.st_##fn_name); \ 245 } \ 246 \ 247 static uint16_t sym32_##fn_name(Elf_Sym *sym) \ 248 { \ 249 return r2(&sym->e32.st_##fn_name); \ 250 } \ 251 \ 252 static uint16_t sym_##fn_name(Elf_Sym *sym) \ 253 { \ 254 return e.sym_##fn_name(sym); \ 255 } 256 257 static uint8_t sym64_type(Elf_Sym *sym) 258 { 259 return ELF64_ST_TYPE(sym->e64.st_info); 260 } 261 262 static uint8_t sym32_type(Elf_Sym *sym) 263 { 264 return ELF32_ST_TYPE(sym->e32.st_info); 265 } 266 267 static uint8_t sym_type(Elf_Sym *sym) 268 { 269 return e.sym_type(sym); 270 } 271 272 SYM_ADDR(value) 273 SYM_WORD(name) 274 SYM_HALF(shndx) 275 276 #define __maybe_unused __attribute__((__unused__)) 277 278 #define RELA_ADDR(fn_name) \ 279 static uint64_t rela64_##fn_name(Elf_Rela *rela) \ 280 { \ 281 return r8((uint64_t *)&rela->e64.r_##fn_name); \ 282 } \ 283 \ 284 static uint64_t rela32_##fn_name(Elf_Rela *rela) \ 285 { \ 286 return r((uint32_t *)&rela->e32.r_##fn_name); \ 287 } \ 288 \ 289 static uint64_t __maybe_unused rela_##fn_name(Elf_Rela *rela) \ 290 { \ 291 return e.rela_##fn_name(rela); \ 292 } 293 294 RELA_ADDR(offset) 295 RELA_ADDR(info) 296 RELA_ADDR(addend) 297 298 static void rela64_write_addend(Elf_Rela *rela, uint64_t val) 299 { 300 w8(val, (uint64_t *)&rela->e64.r_addend); 301 } 302 303 static void rela32_write_addend(Elf_Rela *rela, uint64_t val) 304 { 305 w(val, (uint32_t *)&rela->e32.r_addend); 306 } 307 308 /* 309 * Get the whole file as a programming convenience in order to avoid 310 * malloc+lseek+read+free of many pieces. If successful, then mmap 311 * avoids copying unused pieces; else just read the whole file. 312 * Open for both read and write. 313 */ 314 static void *mmap_file(char const *fname, size_t *size) 315 { 316 int fd; 317 struct stat sb; 318 void *addr = NULL; 319 320 fd = open(fname, O_RDWR); 321 if (fd < 0) { 322 perror(fname); 323 return NULL; 324 } 325 if (fstat(fd, &sb) < 0) { 326 perror(fname); 327 goto out; 328 } 329 if (!S_ISREG(sb.st_mode)) { 330 fprintf(stderr, "not a regular file: %s\n", fname); 331 goto out; 332 } 333 334 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 335 if (addr == MAP_FAILED) { 336 fprintf(stderr, "Could not mmap file: %s\n", fname); 337 goto out; 338 } 339 340 *size = sb.st_size; 341 342 out: 343 close(fd); 344 return addr; 345 } 346 347 static uint32_t rbe(const uint32_t *x) 348 { 349 return get_unaligned_be32(x); 350 } 351 352 static uint16_t r2be(const uint16_t *x) 353 { 354 return get_unaligned_be16(x); 355 } 356 357 static uint64_t r8be(const uint64_t *x) 358 { 359 return get_unaligned_be64(x); 360 } 361 362 static uint32_t rle(const uint32_t *x) 363 { 364 return get_unaligned_le32(x); 365 } 366 367 static uint16_t r2le(const uint16_t *x) 368 { 369 return get_unaligned_le16(x); 370 } 371 372 static uint64_t r8le(const uint64_t *x) 373 { 374 return get_unaligned_le64(x); 375 } 376 377 static void wbe(uint32_t val, uint32_t *x) 378 { 379 put_unaligned_be32(val, x); 380 } 381 382 static void wle(uint32_t val, uint32_t *x) 383 { 384 put_unaligned_le32(val, x); 385 } 386 387 static void w8be(uint64_t val, uint64_t *x) 388 { 389 put_unaligned_be64(val, x); 390 } 391 392 static void w8le(uint64_t val, uint64_t *x) 393 { 394 put_unaligned_le64(val, x); 395 } 396 397 /* 398 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 399 * the way to -256..-1, to avoid conflicting with real section 400 * indices. 401 */ 402 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 403 404 static inline int is_shndx_special(unsigned int i) 405 { 406 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 407 } 408 409 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 410 static inline unsigned int get_secindex(unsigned int shndx, 411 unsigned int sym_offs, 412 const Elf32_Word *symtab_shndx_start) 413 { 414 if (is_shndx_special(shndx)) 415 return SPECIAL(shndx); 416 if (shndx != SHN_XINDEX) 417 return shndx; 418 return r(&symtab_shndx_start[sym_offs]); 419 } 420 421 static int compare_extable_32(const void *a, const void *b) 422 { 423 Elf32_Addr av = r(a); 424 Elf32_Addr bv = r(b); 425 426 if (av < bv) 427 return -1; 428 return av > bv; 429 } 430 431 static int compare_extable_64(const void *a, const void *b) 432 { 433 Elf64_Addr av = r8(a); 434 Elf64_Addr bv = r8(b); 435 436 if (av < bv) 437 return -1; 438 return av > bv; 439 } 440 441 static int compare_extable(const void *a, const void *b) 442 { 443 return e.compare_extable(a, b); 444 } 445 446 static inline void *get_index(void *start, int entsize, int index) 447 { 448 return start + (entsize * index); 449 } 450 451 static int extable_ent_size; 452 static int long_size; 453 454 #define ERRSTR_MAXSZ 256 455 456 #ifdef UNWINDER_ORC_ENABLED 457 /* ORC unwinder only support X86_64 */ 458 #include <asm/orc_types.h> 459 460 static char g_err[ERRSTR_MAXSZ]; 461 static int *g_orc_ip_table; 462 static struct orc_entry *g_orc_table; 463 464 static pthread_t orc_sort_thread; 465 466 static inline unsigned long orc_ip(const int *ip) 467 { 468 return (unsigned long)ip + *ip; 469 } 470 471 static int orc_sort_cmp(const void *_a, const void *_b) 472 { 473 struct orc_entry *orc_a, *orc_b; 474 const int *a = g_orc_ip_table + *(int *)_a; 475 const int *b = g_orc_ip_table + *(int *)_b; 476 unsigned long a_val = orc_ip(a); 477 unsigned long b_val = orc_ip(b); 478 479 if (a_val > b_val) 480 return 1; 481 if (a_val < b_val) 482 return -1; 483 484 /* 485 * The "weak" section terminator entries need to always be on the left 486 * to ensure the lookup code skips them in favor of real entries. 487 * These terminator entries exist to handle any gaps created by 488 * whitelisted .o files which didn't get objtool generation. 489 */ 490 orc_a = g_orc_table + (a - g_orc_ip_table); 491 orc_b = g_orc_table + (b - g_orc_ip_table); 492 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 493 return 0; 494 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 495 } 496 497 static void *sort_orctable(void *arg) 498 { 499 int i; 500 int *idxs = NULL; 501 int *tmp_orc_ip_table = NULL; 502 struct orc_entry *tmp_orc_table = NULL; 503 unsigned int *orc_ip_size = (unsigned int *)arg; 504 unsigned int num_entries = *orc_ip_size / sizeof(int); 505 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 506 507 idxs = (int *)malloc(*orc_ip_size); 508 if (!idxs) { 509 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 510 strerror(errno)); 511 pthread_exit(g_err); 512 } 513 514 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 515 if (!tmp_orc_ip_table) { 516 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 517 strerror(errno)); 518 pthread_exit(g_err); 519 } 520 521 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 522 if (!tmp_orc_table) { 523 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 524 strerror(errno)); 525 pthread_exit(g_err); 526 } 527 528 /* initialize indices array, convert ip_table to absolute address */ 529 for (i = 0; i < num_entries; i++) { 530 idxs[i] = i; 531 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 532 } 533 memcpy(tmp_orc_table, g_orc_table, orc_size); 534 535 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 536 537 for (i = 0; i < num_entries; i++) { 538 if (idxs[i] == i) 539 continue; 540 541 /* convert back to relative address */ 542 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 543 g_orc_table[i] = tmp_orc_table[idxs[i]]; 544 } 545 546 free(idxs); 547 free(tmp_orc_ip_table); 548 free(tmp_orc_table); 549 pthread_exit(NULL); 550 } 551 #endif 552 553 #ifdef MCOUNT_SORT_ENABLED 554 555 static int compare_values_64(const void *a, const void *b) 556 { 557 uint64_t av = *(uint64_t *)a; 558 uint64_t bv = *(uint64_t *)b; 559 560 if (av < bv) 561 return -1; 562 return av > bv; 563 } 564 565 static int compare_values_32(const void *a, const void *b) 566 { 567 uint32_t av = *(uint32_t *)a; 568 uint32_t bv = *(uint32_t *)b; 569 570 if (av < bv) 571 return -1; 572 return av > bv; 573 } 574 575 static int (*compare_values)(const void *a, const void *b); 576 577 /* Only used for sorting mcount table */ 578 static void rela_write_addend(Elf_Rela *rela, uint64_t val) 579 { 580 e.rela_write_addend(rela, val); 581 } 582 583 static pthread_t mcount_sort_thread; 584 static bool sort_reloc; 585 586 static long rela_type; 587 588 static char m_err[ERRSTR_MAXSZ]; 589 590 struct elf_mcount_loc { 591 Elf_Ehdr *ehdr; 592 Elf_Shdr *init_data_sec; 593 uint64_t start_mcount_loc; 594 uint64_t stop_mcount_loc; 595 }; 596 597 /* Fill the array with the content of the relocs */ 598 static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 599 { 600 Elf_Shdr *shdr_start; 601 Elf_Rela *rel; 602 unsigned int shnum; 603 unsigned int count = 0; 604 int shentsize; 605 void *array_end = ptr + size; 606 607 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 608 shentsize = ehdr_shentsize(ehdr); 609 610 shnum = ehdr_shnum(ehdr); 611 if (shnum == SHN_UNDEF) 612 shnum = shdr_size(shdr_start); 613 614 for (int i = 0; i < shnum; i++) { 615 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 616 void *end; 617 618 if (shdr_type(shdr) != SHT_RELA) 619 continue; 620 621 rel = (void *)ehdr + shdr_offset(shdr); 622 end = (void *)rel + shdr_size(shdr); 623 624 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 625 uint64_t offset = rela_offset(rel); 626 627 if (offset >= start_loc && offset < start_loc + size) { 628 if (ptr + long_size > array_end) { 629 snprintf(m_err, ERRSTR_MAXSZ, 630 "Too many relocations"); 631 return -1; 632 } 633 634 /* Make sure this has the correct type */ 635 if (rela_info(rel) != rela_type) { 636 snprintf(m_err, ERRSTR_MAXSZ, 637 "rela has type %lx but expected %lx\n", 638 (long)rela_info(rel), rela_type); 639 return -1; 640 } 641 642 if (long_size == 4) 643 *(uint32_t *)ptr = rela_addend(rel); 644 else 645 *(uint64_t *)ptr = rela_addend(rel); 646 ptr += long_size; 647 count++; 648 } 649 } 650 } 651 return count; 652 } 653 654 /* Put the sorted vals back into the relocation elements */ 655 static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 656 { 657 Elf_Shdr *shdr_start; 658 Elf_Rela *rel; 659 unsigned int shnum; 660 int shentsize; 661 662 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 663 shentsize = ehdr_shentsize(ehdr); 664 665 shnum = ehdr_shnum(ehdr); 666 if (shnum == SHN_UNDEF) 667 shnum = shdr_size(shdr_start); 668 669 for (int i = 0; i < shnum; i++) { 670 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 671 void *end; 672 673 if (shdr_type(shdr) != SHT_RELA) 674 continue; 675 676 rel = (void *)ehdr + shdr_offset(shdr); 677 end = (void *)rel + shdr_size(shdr); 678 679 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 680 uint64_t offset = rela_offset(rel); 681 682 if (offset >= start_loc && offset < start_loc + size) { 683 if (long_size == 4) 684 rela_write_addend(rel, *(uint32_t *)ptr); 685 else 686 rela_write_addend(rel, *(uint64_t *)ptr); 687 ptr += long_size; 688 } 689 } 690 } 691 } 692 693 static int fill_addrs(void *ptr, uint64_t size, void *addrs) 694 { 695 void *end = ptr + size; 696 int count = 0; 697 698 for (; ptr < end; ptr += long_size, addrs += long_size, count++) { 699 if (long_size == 4) 700 *(uint32_t *)ptr = r(addrs); 701 else 702 *(uint64_t *)ptr = r8(addrs); 703 } 704 return count; 705 } 706 707 static void replace_addrs(void *ptr, uint64_t size, void *addrs) 708 { 709 void *end = ptr + size; 710 711 for (; ptr < end; ptr += long_size, addrs += long_size) { 712 if (long_size == 4) 713 w(*(uint32_t *)ptr, addrs); 714 else 715 w8(*(uint64_t *)ptr, addrs); 716 } 717 } 718 719 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 720 static void *sort_mcount_loc(void *arg) 721 { 722 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 723 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 724 + shdr_offset(emloc->init_data_sec); 725 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc; 726 unsigned char *start_loc = (void *)emloc->ehdr + offset; 727 Elf_Ehdr *ehdr = emloc->ehdr; 728 void *e_msg = NULL; 729 void *vals; 730 int count; 731 732 vals = malloc(long_size * size); 733 if (!vals) { 734 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 735 pthread_exit(m_err); 736 } 737 738 if (sort_reloc) 739 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc); 740 else 741 count = fill_addrs(vals, size, start_loc); 742 743 if (count < 0) { 744 e_msg = m_err; 745 goto out; 746 } 747 748 if (count != size / long_size) { 749 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n", 750 (int)(size / long_size), count); 751 e_msg = m_err; 752 goto out; 753 } 754 755 compare_values = long_size == 4 ? compare_values_32 : compare_values_64; 756 757 qsort(vals, count, long_size, compare_values); 758 759 if (sort_reloc) 760 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc); 761 else 762 replace_addrs(vals, size, start_loc); 763 764 out: 765 free(vals); 766 767 pthread_exit(e_msg); 768 } 769 770 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 771 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 772 const char *strtab) 773 { 774 Elf_Sym *sym, *end_sym; 775 int symentsize = shdr_entsize(symtab_sec); 776 int found = 0; 777 778 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 779 end_sym = (void *)sym + shdr_size(symtab_sec); 780 781 while (sym < end_sym) { 782 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 783 emloc->start_mcount_loc = sym_value(sym); 784 if (++found == 2) 785 break; 786 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 787 emloc->stop_mcount_loc = sym_value(sym); 788 if (++found == 2) 789 break; 790 } 791 sym = (void *)sym + symentsize; 792 } 793 794 if (!emloc->start_mcount_loc) { 795 fprintf(stderr, "get start_mcount_loc error!"); 796 return; 797 } 798 799 if (!emloc->stop_mcount_loc) { 800 fprintf(stderr, "get stop_mcount_loc error!"); 801 return; 802 } 803 } 804 #endif 805 806 static int do_sort(Elf_Ehdr *ehdr, 807 char const *const fname, 808 table_sort_t custom_sort) 809 { 810 int rc = -1; 811 Elf_Shdr *shdr_start; 812 Elf_Shdr *strtab_sec = NULL; 813 Elf_Shdr *symtab_sec = NULL; 814 Elf_Shdr *extab_sec = NULL; 815 Elf_Shdr *string_sec; 816 Elf_Sym *sym; 817 const Elf_Sym *symtab; 818 Elf32_Word *symtab_shndx = NULL; 819 Elf_Sym *sort_needed_sym = NULL; 820 Elf_Shdr *sort_needed_sec; 821 uint32_t *sort_needed_loc; 822 void *sym_start; 823 void *sym_end; 824 const char *secstrings; 825 const char *strtab; 826 char *extab_image; 827 int sort_need_index; 828 int symentsize; 829 int shentsize; 830 int idx; 831 int i; 832 unsigned int shnum; 833 unsigned int shstrndx; 834 #ifdef MCOUNT_SORT_ENABLED 835 struct elf_mcount_loc mstruct = {0}; 836 #endif 837 #ifdef UNWINDER_ORC_ENABLED 838 unsigned int orc_ip_size = 0; 839 unsigned int orc_size = 0; 840 unsigned int orc_num_entries = 0; 841 #endif 842 843 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 844 shentsize = ehdr_shentsize(ehdr); 845 846 shstrndx = ehdr_shstrndx(ehdr); 847 if (shstrndx == SHN_XINDEX) 848 shstrndx = shdr_link(shdr_start); 849 string_sec = get_index(shdr_start, shentsize, shstrndx); 850 secstrings = (const char *)ehdr + shdr_offset(string_sec); 851 852 shnum = ehdr_shnum(ehdr); 853 if (shnum == SHN_UNDEF) 854 shnum = shdr_size(shdr_start); 855 856 for (i = 0; i < shnum; i++) { 857 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 858 859 idx = shdr_name(shdr); 860 if (!strcmp(secstrings + idx, "__ex_table")) 861 extab_sec = shdr; 862 if (!strcmp(secstrings + idx, ".symtab")) 863 symtab_sec = shdr; 864 if (!strcmp(secstrings + idx, ".strtab")) 865 strtab_sec = shdr; 866 867 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 868 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 869 shdr_offset(shdr)); 870 871 #ifdef MCOUNT_SORT_ENABLED 872 /* locate the .init.data section in vmlinux */ 873 if (!strcmp(secstrings + idx, ".init.data")) 874 mstruct.init_data_sec = shdr; 875 #endif 876 877 #ifdef UNWINDER_ORC_ENABLED 878 /* locate the ORC unwind tables */ 879 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 880 orc_ip_size = shdr_size(shdr); 881 g_orc_ip_table = (int *)((void *)ehdr + 882 shdr_offset(shdr)); 883 } 884 if (!strcmp(secstrings + idx, ".orc_unwind")) { 885 orc_size = shdr_size(shdr); 886 g_orc_table = (struct orc_entry *)((void *)ehdr + 887 shdr_offset(shdr)); 888 } 889 #endif 890 } /* for loop */ 891 892 #ifdef UNWINDER_ORC_ENABLED 893 if (!g_orc_ip_table || !g_orc_table) { 894 fprintf(stderr, 895 "incomplete ORC unwind tables in file: %s\n", fname); 896 goto out; 897 } 898 899 orc_num_entries = orc_ip_size / sizeof(int); 900 if (orc_ip_size % sizeof(int) != 0 || 901 orc_size % sizeof(struct orc_entry) != 0 || 902 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 903 fprintf(stderr, 904 "inconsistent ORC unwind table entries in file: %s\n", 905 fname); 906 goto out; 907 } 908 909 /* create thread to sort ORC unwind tables concurrently */ 910 if (pthread_create(&orc_sort_thread, NULL, 911 sort_orctable, &orc_ip_size)) { 912 fprintf(stderr, 913 "pthread_create orc_sort_thread failed '%s': %s\n", 914 strerror(errno), fname); 915 goto out; 916 } 917 #endif 918 if (!extab_sec) { 919 fprintf(stderr, "no __ex_table in file: %s\n", fname); 920 goto out; 921 } 922 923 if (!symtab_sec) { 924 fprintf(stderr, "no .symtab in file: %s\n", fname); 925 goto out; 926 } 927 928 if (!strtab_sec) { 929 fprintf(stderr, "no .strtab in file: %s\n", fname); 930 goto out; 931 } 932 933 extab_image = (void *)ehdr + shdr_offset(extab_sec); 934 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 935 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 936 937 #ifdef MCOUNT_SORT_ENABLED 938 mstruct.ehdr = ehdr; 939 get_mcount_loc(&mstruct, symtab_sec, strtab); 940 941 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 942 fprintf(stderr, 943 "incomplete mcount's sort in file: %s\n", 944 fname); 945 goto out; 946 } 947 948 /* create thread to sort mcount_loc concurrently */ 949 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 950 fprintf(stderr, 951 "pthread_create mcount_sort_thread failed '%s': %s\n", 952 strerror(errno), fname); 953 goto out; 954 } 955 #endif 956 957 if (custom_sort) { 958 custom_sort(extab_image, shdr_size(extab_sec)); 959 } else { 960 int num_entries = shdr_size(extab_sec) / extable_ent_size; 961 qsort(extab_image, num_entries, 962 extable_ent_size, compare_extable); 963 } 964 965 /* find the flag main_extable_sort_needed */ 966 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 967 sym_end = sym_start + shdr_size(symtab_sec); 968 symentsize = shdr_entsize(symtab_sec); 969 970 for (sym = sym_start; (void *)sym + symentsize < sym_end; 971 sym = (void *)sym + symentsize) { 972 if (sym_type(sym) != STT_OBJECT) 973 continue; 974 if (!strcmp(strtab + sym_name(sym), 975 "main_extable_sort_needed")) { 976 sort_needed_sym = sym; 977 break; 978 } 979 } 980 981 if (!sort_needed_sym) { 982 fprintf(stderr, 983 "no main_extable_sort_needed symbol in file: %s\n", 984 fname); 985 goto out; 986 } 987 988 sort_need_index = get_secindex(sym_shndx(sym), 989 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 990 symtab_shndx); 991 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 992 sort_needed_loc = (void *)ehdr + 993 shdr_offset(sort_needed_sec) + 994 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 995 996 /* extable has been sorted, clear the flag */ 997 w(0, sort_needed_loc); 998 rc = 0; 999 1000 out: 1001 #ifdef UNWINDER_ORC_ENABLED 1002 if (orc_sort_thread) { 1003 void *retval = NULL; 1004 /* wait for ORC tables sort done */ 1005 rc = pthread_join(orc_sort_thread, &retval); 1006 if (rc) { 1007 fprintf(stderr, 1008 "pthread_join failed '%s': %s\n", 1009 strerror(errno), fname); 1010 } else if (retval) { 1011 rc = -1; 1012 fprintf(stderr, 1013 "failed to sort ORC tables '%s': %s\n", 1014 (char *)retval, fname); 1015 } 1016 } 1017 #endif 1018 1019 #ifdef MCOUNT_SORT_ENABLED 1020 if (mcount_sort_thread) { 1021 void *retval = NULL; 1022 /* wait for mcount sort done */ 1023 rc = pthread_join(mcount_sort_thread, &retval); 1024 if (rc) { 1025 fprintf(stderr, 1026 "pthread_join failed '%s': %s\n", 1027 strerror(errno), fname); 1028 } else if (retval) { 1029 rc = -1; 1030 fprintf(stderr, 1031 "failed to sort mcount '%s': %s\n", 1032 (char *)retval, fname); 1033 } 1034 } 1035 #endif 1036 return rc; 1037 } 1038 1039 static int compare_relative_table(const void *a, const void *b) 1040 { 1041 int32_t av = (int32_t)r(a); 1042 int32_t bv = (int32_t)r(b); 1043 1044 if (av < bv) 1045 return -1; 1046 if (av > bv) 1047 return 1; 1048 return 0; 1049 } 1050 1051 static void sort_relative_table(char *extab_image, int image_size) 1052 { 1053 int i = 0; 1054 1055 /* 1056 * Do the same thing the runtime sort does, first normalize to 1057 * being relative to the start of the section. 1058 */ 1059 while (i < image_size) { 1060 uint32_t *loc = (uint32_t *)(extab_image + i); 1061 w(r(loc) + i, loc); 1062 i += 4; 1063 } 1064 1065 qsort(extab_image, image_size / 8, 8, compare_relative_table); 1066 1067 /* Now denormalize. */ 1068 i = 0; 1069 while (i < image_size) { 1070 uint32_t *loc = (uint32_t *)(extab_image + i); 1071 w(r(loc) - i, loc); 1072 i += 4; 1073 } 1074 } 1075 1076 static void sort_relative_table_with_data(char *extab_image, int image_size) 1077 { 1078 int i = 0; 1079 1080 while (i < image_size) { 1081 uint32_t *loc = (uint32_t *)(extab_image + i); 1082 1083 w(r(loc) + i, loc); 1084 w(r(loc + 1) + i + 4, loc + 1); 1085 /* Don't touch the fixup type or data */ 1086 1087 i += sizeof(uint32_t) * 3; 1088 } 1089 1090 qsort(extab_image, image_size / 12, 12, compare_relative_table); 1091 1092 i = 0; 1093 while (i < image_size) { 1094 uint32_t *loc = (uint32_t *)(extab_image + i); 1095 1096 w(r(loc) - i, loc); 1097 w(r(loc + 1) - (i + 4), loc + 1); 1098 /* Don't touch the fixup type or data */ 1099 1100 i += sizeof(uint32_t) * 3; 1101 } 1102 } 1103 1104 static int do_file(char const *const fname, void *addr) 1105 { 1106 Elf_Ehdr *ehdr = addr; 1107 table_sort_t custom_sort = NULL; 1108 1109 switch (ehdr->e32.e_ident[EI_DATA]) { 1110 case ELFDATA2LSB: 1111 r = rle; 1112 r2 = r2le; 1113 r8 = r8le; 1114 w = wle; 1115 w8 = w8le; 1116 break; 1117 case ELFDATA2MSB: 1118 r = rbe; 1119 r2 = r2be; 1120 r8 = r8be; 1121 w = wbe; 1122 w8 = w8be; 1123 break; 1124 default: 1125 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 1126 ehdr->e32.e_ident[EI_DATA], fname); 1127 return -1; 1128 } 1129 1130 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 || 1131 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) || 1132 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) { 1133 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); 1134 return -1; 1135 } 1136 1137 switch (r2(&ehdr->e32.e_machine)) { 1138 case EM_AARCH64: 1139 #ifdef MCOUNT_SORT_ENABLED 1140 sort_reloc = true; 1141 rela_type = 0x403; 1142 #endif 1143 /* fallthrough */ 1144 case EM_386: 1145 case EM_LOONGARCH: 1146 case EM_RISCV: 1147 case EM_S390: 1148 case EM_X86_64: 1149 custom_sort = sort_relative_table_with_data; 1150 break; 1151 case EM_PARISC: 1152 case EM_PPC: 1153 case EM_PPC64: 1154 custom_sort = sort_relative_table; 1155 break; 1156 case EM_ARCOMPACT: 1157 case EM_ARCV2: 1158 case EM_ARM: 1159 case EM_MICROBLAZE: 1160 case EM_MIPS: 1161 case EM_XTENSA: 1162 break; 1163 default: 1164 fprintf(stderr, "unrecognized e_machine %d %s\n", 1165 r2(&ehdr->e32.e_machine), fname); 1166 return -1; 1167 } 1168 1169 switch (ehdr->e32.e_ident[EI_CLASS]) { 1170 case ELFCLASS32: { 1171 struct elf_funcs efuncs = { 1172 .compare_extable = compare_extable_32, 1173 .ehdr_shoff = ehdr32_shoff, 1174 .ehdr_shentsize = ehdr32_shentsize, 1175 .ehdr_shstrndx = ehdr32_shstrndx, 1176 .ehdr_shnum = ehdr32_shnum, 1177 .shdr_addr = shdr32_addr, 1178 .shdr_offset = shdr32_offset, 1179 .shdr_link = shdr32_link, 1180 .shdr_size = shdr32_size, 1181 .shdr_name = shdr32_name, 1182 .shdr_type = shdr32_type, 1183 .shdr_entsize = shdr32_entsize, 1184 .sym_type = sym32_type, 1185 .sym_name = sym32_name, 1186 .sym_value = sym32_value, 1187 .sym_shndx = sym32_shndx, 1188 .rela_offset = rela32_offset, 1189 .rela_info = rela32_info, 1190 .rela_addend = rela32_addend, 1191 .rela_write_addend = rela32_write_addend, 1192 }; 1193 1194 e = efuncs; 1195 long_size = 4; 1196 extable_ent_size = 8; 1197 1198 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 1199 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 1200 fprintf(stderr, 1201 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 1202 return -1; 1203 } 1204 1205 } 1206 break; 1207 case ELFCLASS64: { 1208 struct elf_funcs efuncs = { 1209 .compare_extable = compare_extable_64, 1210 .ehdr_shoff = ehdr64_shoff, 1211 .ehdr_shentsize = ehdr64_shentsize, 1212 .ehdr_shstrndx = ehdr64_shstrndx, 1213 .ehdr_shnum = ehdr64_shnum, 1214 .shdr_addr = shdr64_addr, 1215 .shdr_offset = shdr64_offset, 1216 .shdr_link = shdr64_link, 1217 .shdr_size = shdr64_size, 1218 .shdr_name = shdr64_name, 1219 .shdr_type = shdr64_type, 1220 .shdr_entsize = shdr64_entsize, 1221 .sym_type = sym64_type, 1222 .sym_name = sym64_name, 1223 .sym_value = sym64_value, 1224 .sym_shndx = sym64_shndx, 1225 .rela_offset = rela64_offset, 1226 .rela_info = rela64_info, 1227 .rela_addend = rela64_addend, 1228 .rela_write_addend = rela64_write_addend, 1229 }; 1230 1231 e = efuncs; 1232 long_size = 8; 1233 extable_ent_size = 16; 1234 1235 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 1236 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 1237 fprintf(stderr, 1238 "unrecognized ET_EXEC/ET_DYN file: %s\n", 1239 fname); 1240 return -1; 1241 } 1242 1243 } 1244 break; 1245 default: 1246 fprintf(stderr, "unrecognized ELF class %d %s\n", 1247 ehdr->e32.e_ident[EI_CLASS], fname); 1248 return -1; 1249 } 1250 1251 return do_sort(ehdr, fname, custom_sort); 1252 } 1253 1254 int main(int argc, char *argv[]) 1255 { 1256 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 1257 size_t size = 0; 1258 void *addr = NULL; 1259 1260 if (argc < 2) { 1261 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1262 return 0; 1263 } 1264 1265 /* Process each file in turn, allowing deep failure. */ 1266 for (i = 1; i < argc; i++) { 1267 addr = mmap_file(argv[i], &size); 1268 if (!addr) { 1269 ++n_error; 1270 continue; 1271 } 1272 1273 if (do_file(argv[i], addr)) 1274 ++n_error; 1275 1276 munmap(addr, size); 1277 } 1278 1279 return !!n_error; 1280 } 1281