1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <[email protected]> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/mman.h> 25 #include <sys/stat.h> 26 #include <getopt.h> 27 #include <elf.h> 28 #include <fcntl.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <stdbool.h> 32 #include <string.h> 33 #include <unistd.h> 34 #include <errno.h> 35 #include <pthread.h> 36 37 #include <tools/be_byteshift.h> 38 #include <tools/le_byteshift.h> 39 40 #ifndef EM_ARCOMPACT 41 #define EM_ARCOMPACT 93 42 #endif 43 44 #ifndef EM_XTENSA 45 #define EM_XTENSA 94 46 #endif 47 48 #ifndef EM_AARCH64 49 #define EM_AARCH64 183 50 #endif 51 52 #ifndef EM_MICROBLAZE 53 #define EM_MICROBLAZE 189 54 #endif 55 56 #ifndef EM_ARCV2 57 #define EM_ARCV2 195 58 #endif 59 60 #ifndef EM_RISCV 61 #define EM_RISCV 243 62 #endif 63 64 #ifndef EM_LOONGARCH 65 #define EM_LOONGARCH 258 66 #endif 67 68 typedef union { 69 Elf32_Ehdr e32; 70 Elf64_Ehdr e64; 71 } Elf_Ehdr; 72 73 typedef union { 74 Elf32_Shdr e32; 75 Elf64_Shdr e64; 76 } Elf_Shdr; 77 78 typedef union { 79 Elf32_Sym e32; 80 Elf64_Sym e64; 81 } Elf_Sym; 82 83 typedef union { 84 Elf32_Rela e32; 85 Elf64_Rela e64; 86 } Elf_Rela; 87 88 static uint32_t (*r)(const uint32_t *); 89 static uint16_t (*r2)(const uint16_t *); 90 static uint64_t (*r8)(const uint64_t *); 91 static void (*w)(uint32_t, uint32_t *); 92 static void (*w8)(uint64_t, uint64_t *); 93 typedef void (*table_sort_t)(char *, int); 94 95 static struct elf_funcs { 96 int (*compare_extable)(const void *a, const void *b); 97 uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr); 98 uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr); 99 uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr); 100 uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr); 101 uint64_t (*shdr_addr)(Elf_Shdr *shdr); 102 uint64_t (*shdr_offset)(Elf_Shdr *shdr); 103 uint64_t (*shdr_size)(Elf_Shdr *shdr); 104 uint64_t (*shdr_entsize)(Elf_Shdr *shdr); 105 uint32_t (*shdr_link)(Elf_Shdr *shdr); 106 uint32_t (*shdr_name)(Elf_Shdr *shdr); 107 uint32_t (*shdr_type)(Elf_Shdr *shdr); 108 uint8_t (*sym_type)(Elf_Sym *sym); 109 uint32_t (*sym_name)(Elf_Sym *sym); 110 uint64_t (*sym_value)(Elf_Sym *sym); 111 uint16_t (*sym_shndx)(Elf_Sym *sym); 112 uint64_t (*rela_offset)(Elf_Rela *rela); 113 uint64_t (*rela_info)(Elf_Rela *rela); 114 uint64_t (*rela_addend)(Elf_Rela *rela); 115 void (*rela_write_addend)(Elf_Rela *rela, uint64_t val); 116 } e; 117 118 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) 119 { 120 return r8(&ehdr->e64.e_shoff); 121 } 122 123 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr) 124 { 125 return r(&ehdr->e32.e_shoff); 126 } 127 128 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr) 129 { 130 return e.ehdr_shoff(ehdr); 131 } 132 133 #define EHDR_HALF(fn_name) \ 134 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \ 135 { \ 136 return r2(&ehdr->e64.e_##fn_name); \ 137 } \ 138 \ 139 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \ 140 { \ 141 return r2(&ehdr->e32.e_##fn_name); \ 142 } \ 143 \ 144 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr) \ 145 { \ 146 return e.ehdr_##fn_name(ehdr); \ 147 } 148 149 EHDR_HALF(shentsize) 150 EHDR_HALF(shstrndx) 151 EHDR_HALF(shnum) 152 153 #define SHDR_WORD(fn_name) \ 154 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 155 { \ 156 return r(&shdr->e64.sh_##fn_name); \ 157 } \ 158 \ 159 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 160 { \ 161 return r(&shdr->e32.sh_##fn_name); \ 162 } \ 163 \ 164 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 165 { \ 166 return e.shdr_##fn_name(shdr); \ 167 } 168 169 #define SHDR_ADDR(fn_name) \ 170 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \ 171 { \ 172 return r8(&shdr->e64.sh_##fn_name); \ 173 } \ 174 \ 175 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \ 176 { \ 177 return r(&shdr->e32.sh_##fn_name); \ 178 } \ 179 \ 180 static uint64_t shdr_##fn_name(Elf_Shdr *shdr) \ 181 { \ 182 return e.shdr_##fn_name(shdr); \ 183 } 184 185 #define SHDR_WORD(fn_name) \ 186 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 187 { \ 188 return r(&shdr->e64.sh_##fn_name); \ 189 } \ 190 \ 191 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 192 { \ 193 return r(&shdr->e32.sh_##fn_name); \ 194 } \ 195 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 196 { \ 197 return e.shdr_##fn_name(shdr); \ 198 } 199 200 SHDR_ADDR(addr) 201 SHDR_ADDR(offset) 202 SHDR_ADDR(size) 203 SHDR_ADDR(entsize) 204 205 SHDR_WORD(link) 206 SHDR_WORD(name) 207 SHDR_WORD(type) 208 209 #define SYM_ADDR(fn_name) \ 210 static uint64_t sym64_##fn_name(Elf_Sym *sym) \ 211 { \ 212 return r8(&sym->e64.st_##fn_name); \ 213 } \ 214 \ 215 static uint64_t sym32_##fn_name(Elf_Sym *sym) \ 216 { \ 217 return r(&sym->e32.st_##fn_name); \ 218 } \ 219 \ 220 static uint64_t sym_##fn_name(Elf_Sym *sym) \ 221 { \ 222 return e.sym_##fn_name(sym); \ 223 } 224 225 #define SYM_WORD(fn_name) \ 226 static uint32_t sym64_##fn_name(Elf_Sym *sym) \ 227 { \ 228 return r(&sym->e64.st_##fn_name); \ 229 } \ 230 \ 231 static uint32_t sym32_##fn_name(Elf_Sym *sym) \ 232 { \ 233 return r(&sym->e32.st_##fn_name); \ 234 } \ 235 \ 236 static uint32_t sym_##fn_name(Elf_Sym *sym) \ 237 { \ 238 return e.sym_##fn_name(sym); \ 239 } 240 241 #define SYM_HALF(fn_name) \ 242 static uint16_t sym64_##fn_name(Elf_Sym *sym) \ 243 { \ 244 return r2(&sym->e64.st_##fn_name); \ 245 } \ 246 \ 247 static uint16_t sym32_##fn_name(Elf_Sym *sym) \ 248 { \ 249 return r2(&sym->e32.st_##fn_name); \ 250 } \ 251 \ 252 static uint16_t sym_##fn_name(Elf_Sym *sym) \ 253 { \ 254 return e.sym_##fn_name(sym); \ 255 } 256 257 static uint8_t sym64_type(Elf_Sym *sym) 258 { 259 return ELF64_ST_TYPE(sym->e64.st_info); 260 } 261 262 static uint8_t sym32_type(Elf_Sym *sym) 263 { 264 return ELF32_ST_TYPE(sym->e32.st_info); 265 } 266 267 static uint8_t sym_type(Elf_Sym *sym) 268 { 269 return e.sym_type(sym); 270 } 271 272 SYM_ADDR(value) 273 SYM_WORD(name) 274 SYM_HALF(shndx) 275 276 #define __maybe_unused __attribute__((__unused__)) 277 278 #define RELA_ADDR(fn_name) \ 279 static uint64_t rela64_##fn_name(Elf_Rela *rela) \ 280 { \ 281 return r8((uint64_t *)&rela->e64.r_##fn_name); \ 282 } \ 283 \ 284 static uint64_t rela32_##fn_name(Elf_Rela *rela) \ 285 { \ 286 return r((uint32_t *)&rela->e32.r_##fn_name); \ 287 } \ 288 \ 289 static uint64_t __maybe_unused rela_##fn_name(Elf_Rela *rela) \ 290 { \ 291 return e.rela_##fn_name(rela); \ 292 } 293 294 RELA_ADDR(offset) 295 RELA_ADDR(info) 296 RELA_ADDR(addend) 297 298 static void rela64_write_addend(Elf_Rela *rela, uint64_t val) 299 { 300 w8(val, (uint64_t *)&rela->e64.r_addend); 301 } 302 303 static void rela32_write_addend(Elf_Rela *rela, uint64_t val) 304 { 305 w(val, (uint32_t *)&rela->e32.r_addend); 306 } 307 308 /* 309 * Get the whole file as a programming convenience in order to avoid 310 * malloc+lseek+read+free of many pieces. If successful, then mmap 311 * avoids copying unused pieces; else just read the whole file. 312 * Open for both read and write. 313 */ 314 static void *mmap_file(char const *fname, size_t *size) 315 { 316 int fd; 317 struct stat sb; 318 void *addr = NULL; 319 320 fd = open(fname, O_RDWR); 321 if (fd < 0) { 322 perror(fname); 323 return NULL; 324 } 325 if (fstat(fd, &sb) < 0) { 326 perror(fname); 327 goto out; 328 } 329 if (!S_ISREG(sb.st_mode)) { 330 fprintf(stderr, "not a regular file: %s\n", fname); 331 goto out; 332 } 333 334 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 335 if (addr == MAP_FAILED) { 336 fprintf(stderr, "Could not mmap file: %s\n", fname); 337 goto out; 338 } 339 340 *size = sb.st_size; 341 342 out: 343 close(fd); 344 return addr; 345 } 346 347 static uint32_t rbe(const uint32_t *x) 348 { 349 return get_unaligned_be32(x); 350 } 351 352 static uint16_t r2be(const uint16_t *x) 353 { 354 return get_unaligned_be16(x); 355 } 356 357 static uint64_t r8be(const uint64_t *x) 358 { 359 return get_unaligned_be64(x); 360 } 361 362 static uint32_t rle(const uint32_t *x) 363 { 364 return get_unaligned_le32(x); 365 } 366 367 static uint16_t r2le(const uint16_t *x) 368 { 369 return get_unaligned_le16(x); 370 } 371 372 static uint64_t r8le(const uint64_t *x) 373 { 374 return get_unaligned_le64(x); 375 } 376 377 static void wbe(uint32_t val, uint32_t *x) 378 { 379 put_unaligned_be32(val, x); 380 } 381 382 static void wle(uint32_t val, uint32_t *x) 383 { 384 put_unaligned_le32(val, x); 385 } 386 387 static void w8be(uint64_t val, uint64_t *x) 388 { 389 put_unaligned_be64(val, x); 390 } 391 392 static void w8le(uint64_t val, uint64_t *x) 393 { 394 put_unaligned_le64(val, x); 395 } 396 397 /* 398 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 399 * the way to -256..-1, to avoid conflicting with real section 400 * indices. 401 */ 402 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 403 404 static inline int is_shndx_special(unsigned int i) 405 { 406 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 407 } 408 409 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 410 static inline unsigned int get_secindex(unsigned int shndx, 411 unsigned int sym_offs, 412 const Elf32_Word *symtab_shndx_start) 413 { 414 if (is_shndx_special(shndx)) 415 return SPECIAL(shndx); 416 if (shndx != SHN_XINDEX) 417 return shndx; 418 return r(&symtab_shndx_start[sym_offs]); 419 } 420 421 static int compare_extable_32(const void *a, const void *b) 422 { 423 Elf32_Addr av = r(a); 424 Elf32_Addr bv = r(b); 425 426 if (av < bv) 427 return -1; 428 return av > bv; 429 } 430 431 static int compare_extable_64(const void *a, const void *b) 432 { 433 Elf64_Addr av = r8(a); 434 Elf64_Addr bv = r8(b); 435 436 if (av < bv) 437 return -1; 438 return av > bv; 439 } 440 441 static int compare_extable(const void *a, const void *b) 442 { 443 return e.compare_extable(a, b); 444 } 445 446 static inline void *get_index(void *start, int entsize, int index) 447 { 448 return start + (entsize * index); 449 } 450 451 static int extable_ent_size; 452 static int long_size; 453 454 #define ERRSTR_MAXSZ 256 455 456 #ifdef UNWINDER_ORC_ENABLED 457 /* ORC unwinder only support X86_64 */ 458 #include <asm/orc_types.h> 459 460 static char g_err[ERRSTR_MAXSZ]; 461 static int *g_orc_ip_table; 462 static struct orc_entry *g_orc_table; 463 464 static pthread_t orc_sort_thread; 465 466 static inline unsigned long orc_ip(const int *ip) 467 { 468 return (unsigned long)ip + *ip; 469 } 470 471 static int orc_sort_cmp(const void *_a, const void *_b) 472 { 473 struct orc_entry *orc_a, *orc_b; 474 const int *a = g_orc_ip_table + *(int *)_a; 475 const int *b = g_orc_ip_table + *(int *)_b; 476 unsigned long a_val = orc_ip(a); 477 unsigned long b_val = orc_ip(b); 478 479 if (a_val > b_val) 480 return 1; 481 if (a_val < b_val) 482 return -1; 483 484 /* 485 * The "weak" section terminator entries need to always be on the left 486 * to ensure the lookup code skips them in favor of real entries. 487 * These terminator entries exist to handle any gaps created by 488 * whitelisted .o files which didn't get objtool generation. 489 */ 490 orc_a = g_orc_table + (a - g_orc_ip_table); 491 orc_b = g_orc_table + (b - g_orc_ip_table); 492 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 493 return 0; 494 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 495 } 496 497 static void *sort_orctable(void *arg) 498 { 499 int i; 500 int *idxs = NULL; 501 int *tmp_orc_ip_table = NULL; 502 struct orc_entry *tmp_orc_table = NULL; 503 unsigned int *orc_ip_size = (unsigned int *)arg; 504 unsigned int num_entries = *orc_ip_size / sizeof(int); 505 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 506 507 idxs = (int *)malloc(*orc_ip_size); 508 if (!idxs) { 509 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 510 strerror(errno)); 511 pthread_exit(g_err); 512 } 513 514 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 515 if (!tmp_orc_ip_table) { 516 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 517 strerror(errno)); 518 pthread_exit(g_err); 519 } 520 521 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 522 if (!tmp_orc_table) { 523 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 524 strerror(errno)); 525 pthread_exit(g_err); 526 } 527 528 /* initialize indices array, convert ip_table to absolute address */ 529 for (i = 0; i < num_entries; i++) { 530 idxs[i] = i; 531 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 532 } 533 memcpy(tmp_orc_table, g_orc_table, orc_size); 534 535 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 536 537 for (i = 0; i < num_entries; i++) { 538 if (idxs[i] == i) 539 continue; 540 541 /* convert back to relative address */ 542 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 543 g_orc_table[i] = tmp_orc_table[idxs[i]]; 544 } 545 546 free(idxs); 547 free(tmp_orc_ip_table); 548 free(tmp_orc_table); 549 pthread_exit(NULL); 550 } 551 #endif 552 553 #ifdef MCOUNT_SORT_ENABLED 554 555 static int compare_values_64(const void *a, const void *b) 556 { 557 uint64_t av = *(uint64_t *)a; 558 uint64_t bv = *(uint64_t *)b; 559 560 if (av < bv) 561 return -1; 562 return av > bv; 563 } 564 565 static int compare_values_32(const void *a, const void *b) 566 { 567 uint32_t av = *(uint32_t *)a; 568 uint32_t bv = *(uint32_t *)b; 569 570 if (av < bv) 571 return -1; 572 return av > bv; 573 } 574 575 static int (*compare_values)(const void *a, const void *b); 576 577 /* Only used for sorting mcount table */ 578 static void rela_write_addend(Elf_Rela *rela, uint64_t val) 579 { 580 e.rela_write_addend(rela, val); 581 } 582 583 struct func_info { 584 uint64_t addr; 585 uint64_t size; 586 }; 587 588 /* List of functions created by: nm -S vmlinux */ 589 static struct func_info *function_list; 590 static int function_list_size; 591 592 /* Allocate functions in 1k blocks */ 593 #define FUNC_BLK_SIZE 1024 594 #define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1) 595 596 static int add_field(uint64_t addr, uint64_t size) 597 { 598 struct func_info *fi; 599 int fsize = function_list_size; 600 601 if (!(fsize & FUNC_BLK_MASK)) { 602 fsize += FUNC_BLK_SIZE; 603 fi = realloc(function_list, fsize * sizeof(struct func_info)); 604 if (!fi) 605 return -1; 606 function_list = fi; 607 } 608 fi = &function_list[function_list_size++]; 609 fi->addr = addr; 610 fi->size = size; 611 return 0; 612 } 613 614 /* Only return match if the address lies inside the function size */ 615 static int cmp_func_addr(const void *K, const void *A) 616 { 617 uint64_t key = *(const uint64_t *)K; 618 const struct func_info *a = A; 619 620 if (key < a->addr) 621 return -1; 622 return key >= a->addr + a->size; 623 } 624 625 /* Find the function in function list that is bounded by the function size */ 626 static int find_func(uint64_t key) 627 { 628 return bsearch(&key, function_list, function_list_size, 629 sizeof(struct func_info), cmp_func_addr) != NULL; 630 } 631 632 static int cmp_funcs(const void *A, const void *B) 633 { 634 const struct func_info *a = A; 635 const struct func_info *b = B; 636 637 if (a->addr < b->addr) 638 return -1; 639 return a->addr > b->addr; 640 } 641 642 static int parse_symbols(const char *fname) 643 { 644 FILE *fp; 645 char addr_str[20]; /* Only need 17, but round up to next int size */ 646 char size_str[20]; 647 char type; 648 649 fp = fopen(fname, "r"); 650 if (!fp) { 651 perror(fname); 652 return -1; 653 } 654 655 while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) { 656 uint64_t addr; 657 uint64_t size; 658 659 /* Only care about functions */ 660 if (type != 't' && type != 'T' && type != 'W') 661 continue; 662 663 addr = strtoull(addr_str, NULL, 16); 664 size = strtoull(size_str, NULL, 16); 665 if (add_field(addr, size) < 0) 666 return -1; 667 } 668 fclose(fp); 669 670 qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs); 671 672 return 0; 673 } 674 675 static pthread_t mcount_sort_thread; 676 static bool sort_reloc; 677 678 static long rela_type; 679 680 static char m_err[ERRSTR_MAXSZ]; 681 682 struct elf_mcount_loc { 683 Elf_Ehdr *ehdr; 684 Elf_Shdr *init_data_sec; 685 uint64_t start_mcount_loc; 686 uint64_t stop_mcount_loc; 687 }; 688 689 /* Fill the array with the content of the relocs */ 690 static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 691 { 692 Elf_Shdr *shdr_start; 693 Elf_Rela *rel; 694 unsigned int shnum; 695 unsigned int count = 0; 696 int shentsize; 697 void *array_end = ptr + size; 698 699 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 700 shentsize = ehdr_shentsize(ehdr); 701 702 shnum = ehdr_shnum(ehdr); 703 if (shnum == SHN_UNDEF) 704 shnum = shdr_size(shdr_start); 705 706 for (int i = 0; i < shnum; i++) { 707 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 708 void *end; 709 710 if (shdr_type(shdr) != SHT_RELA) 711 continue; 712 713 rel = (void *)ehdr + shdr_offset(shdr); 714 end = (void *)rel + shdr_size(shdr); 715 716 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 717 uint64_t offset = rela_offset(rel); 718 719 if (offset >= start_loc && offset < start_loc + size) { 720 if (ptr + long_size > array_end) { 721 snprintf(m_err, ERRSTR_MAXSZ, 722 "Too many relocations"); 723 return -1; 724 } 725 726 /* Make sure this has the correct type */ 727 if (rela_info(rel) != rela_type) { 728 snprintf(m_err, ERRSTR_MAXSZ, 729 "rela has type %lx but expected %lx\n", 730 (long)rela_info(rel), rela_type); 731 return -1; 732 } 733 734 if (long_size == 4) 735 *(uint32_t *)ptr = rela_addend(rel); 736 else 737 *(uint64_t *)ptr = rela_addend(rel); 738 ptr += long_size; 739 count++; 740 } 741 } 742 } 743 return count; 744 } 745 746 /* Put the sorted vals back into the relocation elements */ 747 static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 748 { 749 Elf_Shdr *shdr_start; 750 Elf_Rela *rel; 751 unsigned int shnum; 752 int shentsize; 753 754 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 755 shentsize = ehdr_shentsize(ehdr); 756 757 shnum = ehdr_shnum(ehdr); 758 if (shnum == SHN_UNDEF) 759 shnum = shdr_size(shdr_start); 760 761 for (int i = 0; i < shnum; i++) { 762 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 763 void *end; 764 765 if (shdr_type(shdr) != SHT_RELA) 766 continue; 767 768 rel = (void *)ehdr + shdr_offset(shdr); 769 end = (void *)rel + shdr_size(shdr); 770 771 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 772 uint64_t offset = rela_offset(rel); 773 774 if (offset >= start_loc && offset < start_loc + size) { 775 if (long_size == 4) 776 rela_write_addend(rel, *(uint32_t *)ptr); 777 else 778 rela_write_addend(rel, *(uint64_t *)ptr); 779 ptr += long_size; 780 } 781 } 782 } 783 } 784 785 static int fill_addrs(void *ptr, uint64_t size, void *addrs) 786 { 787 void *end = ptr + size; 788 int count = 0; 789 790 for (; ptr < end; ptr += long_size, addrs += long_size, count++) { 791 if (long_size == 4) 792 *(uint32_t *)ptr = r(addrs); 793 else 794 *(uint64_t *)ptr = r8(addrs); 795 } 796 return count; 797 } 798 799 static void replace_addrs(void *ptr, uint64_t size, void *addrs) 800 { 801 void *end = ptr + size; 802 803 for (; ptr < end; ptr += long_size, addrs += long_size) { 804 if (long_size == 4) 805 w(*(uint32_t *)ptr, addrs); 806 else 807 w8(*(uint64_t *)ptr, addrs); 808 } 809 } 810 811 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 812 static void *sort_mcount_loc(void *arg) 813 { 814 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 815 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 816 + shdr_offset(emloc->init_data_sec); 817 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc; 818 unsigned char *start_loc = (void *)emloc->ehdr + offset; 819 Elf_Ehdr *ehdr = emloc->ehdr; 820 void *e_msg = NULL; 821 void *vals; 822 int count; 823 824 vals = malloc(long_size * size); 825 if (!vals) { 826 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 827 pthread_exit(m_err); 828 } 829 830 if (sort_reloc) { 831 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc); 832 /* gcc may use relocs to save the addresses, but clang does not. */ 833 if (!count) { 834 count = fill_addrs(vals, size, start_loc); 835 sort_reloc = 0; 836 } 837 } else 838 count = fill_addrs(vals, size, start_loc); 839 840 if (count < 0) { 841 e_msg = m_err; 842 goto out; 843 } 844 845 if (count != size / long_size) { 846 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n", 847 (int)(size / long_size), count); 848 e_msg = m_err; 849 goto out; 850 } 851 852 /* zero out any locations not found by function list */ 853 if (function_list_size) { 854 for (void *ptr = vals; ptr < vals + size; ptr += long_size) { 855 uint64_t key; 856 857 key = long_size == 4 ? r((uint32_t *)ptr) : r8((uint64_t *)ptr); 858 if (!find_func(key)) { 859 if (long_size == 4) 860 *(uint32_t *)ptr = 0; 861 else 862 *(uint64_t *)ptr = 0; 863 } 864 } 865 } 866 867 compare_values = long_size == 4 ? compare_values_32 : compare_values_64; 868 869 qsort(vals, count, long_size, compare_values); 870 871 if (sort_reloc) 872 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc); 873 else 874 replace_addrs(vals, size, start_loc); 875 876 out: 877 free(vals); 878 879 pthread_exit(e_msg); 880 } 881 882 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 883 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 884 const char *strtab) 885 { 886 Elf_Sym *sym, *end_sym; 887 int symentsize = shdr_entsize(symtab_sec); 888 int found = 0; 889 890 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 891 end_sym = (void *)sym + shdr_size(symtab_sec); 892 893 while (sym < end_sym) { 894 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 895 emloc->start_mcount_loc = sym_value(sym); 896 if (++found == 2) 897 break; 898 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 899 emloc->stop_mcount_loc = sym_value(sym); 900 if (++found == 2) 901 break; 902 } 903 sym = (void *)sym + symentsize; 904 } 905 906 if (!emloc->start_mcount_loc) { 907 fprintf(stderr, "get start_mcount_loc error!"); 908 return; 909 } 910 911 if (!emloc->stop_mcount_loc) { 912 fprintf(stderr, "get stop_mcount_loc error!"); 913 return; 914 } 915 } 916 #else /* MCOUNT_SORT_ENABLED */ 917 static inline int parse_symbols(const char *fname) { return 0; } 918 #endif 919 920 static int do_sort(Elf_Ehdr *ehdr, 921 char const *const fname, 922 table_sort_t custom_sort) 923 { 924 int rc = -1; 925 Elf_Shdr *shdr_start; 926 Elf_Shdr *strtab_sec = NULL; 927 Elf_Shdr *symtab_sec = NULL; 928 Elf_Shdr *extab_sec = NULL; 929 Elf_Shdr *string_sec; 930 Elf_Sym *sym; 931 const Elf_Sym *symtab; 932 Elf32_Word *symtab_shndx = NULL; 933 Elf_Sym *sort_needed_sym = NULL; 934 Elf_Shdr *sort_needed_sec; 935 uint32_t *sort_needed_loc; 936 void *sym_start; 937 void *sym_end; 938 const char *secstrings; 939 const char *strtab; 940 char *extab_image; 941 int sort_need_index; 942 int symentsize; 943 int shentsize; 944 int idx; 945 int i; 946 unsigned int shnum; 947 unsigned int shstrndx; 948 #ifdef MCOUNT_SORT_ENABLED 949 struct elf_mcount_loc mstruct = {0}; 950 #endif 951 #ifdef UNWINDER_ORC_ENABLED 952 unsigned int orc_ip_size = 0; 953 unsigned int orc_size = 0; 954 unsigned int orc_num_entries = 0; 955 #endif 956 957 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 958 shentsize = ehdr_shentsize(ehdr); 959 960 shstrndx = ehdr_shstrndx(ehdr); 961 if (shstrndx == SHN_XINDEX) 962 shstrndx = shdr_link(shdr_start); 963 string_sec = get_index(shdr_start, shentsize, shstrndx); 964 secstrings = (const char *)ehdr + shdr_offset(string_sec); 965 966 shnum = ehdr_shnum(ehdr); 967 if (shnum == SHN_UNDEF) 968 shnum = shdr_size(shdr_start); 969 970 for (i = 0; i < shnum; i++) { 971 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 972 973 idx = shdr_name(shdr); 974 if (!strcmp(secstrings + idx, "__ex_table")) 975 extab_sec = shdr; 976 if (!strcmp(secstrings + idx, ".symtab")) 977 symtab_sec = shdr; 978 if (!strcmp(secstrings + idx, ".strtab")) 979 strtab_sec = shdr; 980 981 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 982 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 983 shdr_offset(shdr)); 984 985 #ifdef MCOUNT_SORT_ENABLED 986 /* locate the .init.data section in vmlinux */ 987 if (!strcmp(secstrings + idx, ".init.data")) 988 mstruct.init_data_sec = shdr; 989 #endif 990 991 #ifdef UNWINDER_ORC_ENABLED 992 /* locate the ORC unwind tables */ 993 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 994 orc_ip_size = shdr_size(shdr); 995 g_orc_ip_table = (int *)((void *)ehdr + 996 shdr_offset(shdr)); 997 } 998 if (!strcmp(secstrings + idx, ".orc_unwind")) { 999 orc_size = shdr_size(shdr); 1000 g_orc_table = (struct orc_entry *)((void *)ehdr + 1001 shdr_offset(shdr)); 1002 } 1003 #endif 1004 } /* for loop */ 1005 1006 #ifdef UNWINDER_ORC_ENABLED 1007 if (!g_orc_ip_table || !g_orc_table) { 1008 fprintf(stderr, 1009 "incomplete ORC unwind tables in file: %s\n", fname); 1010 goto out; 1011 } 1012 1013 orc_num_entries = orc_ip_size / sizeof(int); 1014 if (orc_ip_size % sizeof(int) != 0 || 1015 orc_size % sizeof(struct orc_entry) != 0 || 1016 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 1017 fprintf(stderr, 1018 "inconsistent ORC unwind table entries in file: %s\n", 1019 fname); 1020 goto out; 1021 } 1022 1023 /* create thread to sort ORC unwind tables concurrently */ 1024 if (pthread_create(&orc_sort_thread, NULL, 1025 sort_orctable, &orc_ip_size)) { 1026 fprintf(stderr, 1027 "pthread_create orc_sort_thread failed '%s': %s\n", 1028 strerror(errno), fname); 1029 goto out; 1030 } 1031 #endif 1032 if (!extab_sec) { 1033 fprintf(stderr, "no __ex_table in file: %s\n", fname); 1034 goto out; 1035 } 1036 1037 if (!symtab_sec) { 1038 fprintf(stderr, "no .symtab in file: %s\n", fname); 1039 goto out; 1040 } 1041 1042 if (!strtab_sec) { 1043 fprintf(stderr, "no .strtab in file: %s\n", fname); 1044 goto out; 1045 } 1046 1047 extab_image = (void *)ehdr + shdr_offset(extab_sec); 1048 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 1049 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 1050 1051 #ifdef MCOUNT_SORT_ENABLED 1052 mstruct.ehdr = ehdr; 1053 get_mcount_loc(&mstruct, symtab_sec, strtab); 1054 1055 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 1056 fprintf(stderr, 1057 "incomplete mcount's sort in file: %s\n", 1058 fname); 1059 goto out; 1060 } 1061 1062 /* create thread to sort mcount_loc concurrently */ 1063 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 1064 fprintf(stderr, 1065 "pthread_create mcount_sort_thread failed '%s': %s\n", 1066 strerror(errno), fname); 1067 goto out; 1068 } 1069 #endif 1070 1071 if (custom_sort) { 1072 custom_sort(extab_image, shdr_size(extab_sec)); 1073 } else { 1074 int num_entries = shdr_size(extab_sec) / extable_ent_size; 1075 qsort(extab_image, num_entries, 1076 extable_ent_size, compare_extable); 1077 } 1078 1079 /* find the flag main_extable_sort_needed */ 1080 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 1081 sym_end = sym_start + shdr_size(symtab_sec); 1082 symentsize = shdr_entsize(symtab_sec); 1083 1084 for (sym = sym_start; (void *)sym + symentsize < sym_end; 1085 sym = (void *)sym + symentsize) { 1086 if (sym_type(sym) != STT_OBJECT) 1087 continue; 1088 if (!strcmp(strtab + sym_name(sym), 1089 "main_extable_sort_needed")) { 1090 sort_needed_sym = sym; 1091 break; 1092 } 1093 } 1094 1095 if (!sort_needed_sym) { 1096 fprintf(stderr, 1097 "no main_extable_sort_needed symbol in file: %s\n", 1098 fname); 1099 goto out; 1100 } 1101 1102 sort_need_index = get_secindex(sym_shndx(sym), 1103 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 1104 symtab_shndx); 1105 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 1106 sort_needed_loc = (void *)ehdr + 1107 shdr_offset(sort_needed_sec) + 1108 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 1109 1110 /* extable has been sorted, clear the flag */ 1111 w(0, sort_needed_loc); 1112 rc = 0; 1113 1114 out: 1115 #ifdef UNWINDER_ORC_ENABLED 1116 if (orc_sort_thread) { 1117 void *retval = NULL; 1118 /* wait for ORC tables sort done */ 1119 rc = pthread_join(orc_sort_thread, &retval); 1120 if (rc) { 1121 fprintf(stderr, 1122 "pthread_join failed '%s': %s\n", 1123 strerror(errno), fname); 1124 } else if (retval) { 1125 rc = -1; 1126 fprintf(stderr, 1127 "failed to sort ORC tables '%s': %s\n", 1128 (char *)retval, fname); 1129 } 1130 } 1131 #endif 1132 1133 #ifdef MCOUNT_SORT_ENABLED 1134 if (mcount_sort_thread) { 1135 void *retval = NULL; 1136 /* wait for mcount sort done */ 1137 rc = pthread_join(mcount_sort_thread, &retval); 1138 if (rc) { 1139 fprintf(stderr, 1140 "pthread_join failed '%s': %s\n", 1141 strerror(errno), fname); 1142 } else if (retval) { 1143 rc = -1; 1144 fprintf(stderr, 1145 "failed to sort mcount '%s': %s\n", 1146 (char *)retval, fname); 1147 } 1148 } 1149 #endif 1150 return rc; 1151 } 1152 1153 static int compare_relative_table(const void *a, const void *b) 1154 { 1155 int32_t av = (int32_t)r(a); 1156 int32_t bv = (int32_t)r(b); 1157 1158 if (av < bv) 1159 return -1; 1160 if (av > bv) 1161 return 1; 1162 return 0; 1163 } 1164 1165 static void sort_relative_table(char *extab_image, int image_size) 1166 { 1167 int i = 0; 1168 1169 /* 1170 * Do the same thing the runtime sort does, first normalize to 1171 * being relative to the start of the section. 1172 */ 1173 while (i < image_size) { 1174 uint32_t *loc = (uint32_t *)(extab_image + i); 1175 w(r(loc) + i, loc); 1176 i += 4; 1177 } 1178 1179 qsort(extab_image, image_size / 8, 8, compare_relative_table); 1180 1181 /* Now denormalize. */ 1182 i = 0; 1183 while (i < image_size) { 1184 uint32_t *loc = (uint32_t *)(extab_image + i); 1185 w(r(loc) - i, loc); 1186 i += 4; 1187 } 1188 } 1189 1190 static void sort_relative_table_with_data(char *extab_image, int image_size) 1191 { 1192 int i = 0; 1193 1194 while (i < image_size) { 1195 uint32_t *loc = (uint32_t *)(extab_image + i); 1196 1197 w(r(loc) + i, loc); 1198 w(r(loc + 1) + i + 4, loc + 1); 1199 /* Don't touch the fixup type or data */ 1200 1201 i += sizeof(uint32_t) * 3; 1202 } 1203 1204 qsort(extab_image, image_size / 12, 12, compare_relative_table); 1205 1206 i = 0; 1207 while (i < image_size) { 1208 uint32_t *loc = (uint32_t *)(extab_image + i); 1209 1210 w(r(loc) - i, loc); 1211 w(r(loc + 1) - (i + 4), loc + 1); 1212 /* Don't touch the fixup type or data */ 1213 1214 i += sizeof(uint32_t) * 3; 1215 } 1216 } 1217 1218 static int do_file(char const *const fname, void *addr) 1219 { 1220 Elf_Ehdr *ehdr = addr; 1221 table_sort_t custom_sort = NULL; 1222 1223 switch (ehdr->e32.e_ident[EI_DATA]) { 1224 case ELFDATA2LSB: 1225 r = rle; 1226 r2 = r2le; 1227 r8 = r8le; 1228 w = wle; 1229 w8 = w8le; 1230 break; 1231 case ELFDATA2MSB: 1232 r = rbe; 1233 r2 = r2be; 1234 r8 = r8be; 1235 w = wbe; 1236 w8 = w8be; 1237 break; 1238 default: 1239 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 1240 ehdr->e32.e_ident[EI_DATA], fname); 1241 return -1; 1242 } 1243 1244 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 || 1245 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) || 1246 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) { 1247 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); 1248 return -1; 1249 } 1250 1251 switch (r2(&ehdr->e32.e_machine)) { 1252 case EM_AARCH64: 1253 #ifdef MCOUNT_SORT_ENABLED 1254 sort_reloc = true; 1255 rela_type = 0x403; 1256 #endif 1257 /* fallthrough */ 1258 case EM_386: 1259 case EM_LOONGARCH: 1260 case EM_RISCV: 1261 case EM_S390: 1262 case EM_X86_64: 1263 custom_sort = sort_relative_table_with_data; 1264 break; 1265 case EM_PARISC: 1266 case EM_PPC: 1267 case EM_PPC64: 1268 custom_sort = sort_relative_table; 1269 break; 1270 case EM_ARCOMPACT: 1271 case EM_ARCV2: 1272 case EM_ARM: 1273 case EM_MICROBLAZE: 1274 case EM_MIPS: 1275 case EM_XTENSA: 1276 break; 1277 default: 1278 fprintf(stderr, "unrecognized e_machine %d %s\n", 1279 r2(&ehdr->e32.e_machine), fname); 1280 return -1; 1281 } 1282 1283 switch (ehdr->e32.e_ident[EI_CLASS]) { 1284 case ELFCLASS32: { 1285 struct elf_funcs efuncs = { 1286 .compare_extable = compare_extable_32, 1287 .ehdr_shoff = ehdr32_shoff, 1288 .ehdr_shentsize = ehdr32_shentsize, 1289 .ehdr_shstrndx = ehdr32_shstrndx, 1290 .ehdr_shnum = ehdr32_shnum, 1291 .shdr_addr = shdr32_addr, 1292 .shdr_offset = shdr32_offset, 1293 .shdr_link = shdr32_link, 1294 .shdr_size = shdr32_size, 1295 .shdr_name = shdr32_name, 1296 .shdr_type = shdr32_type, 1297 .shdr_entsize = shdr32_entsize, 1298 .sym_type = sym32_type, 1299 .sym_name = sym32_name, 1300 .sym_value = sym32_value, 1301 .sym_shndx = sym32_shndx, 1302 .rela_offset = rela32_offset, 1303 .rela_info = rela32_info, 1304 .rela_addend = rela32_addend, 1305 .rela_write_addend = rela32_write_addend, 1306 }; 1307 1308 e = efuncs; 1309 long_size = 4; 1310 extable_ent_size = 8; 1311 1312 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 1313 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 1314 fprintf(stderr, 1315 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 1316 return -1; 1317 } 1318 1319 } 1320 break; 1321 case ELFCLASS64: { 1322 struct elf_funcs efuncs = { 1323 .compare_extable = compare_extable_64, 1324 .ehdr_shoff = ehdr64_shoff, 1325 .ehdr_shentsize = ehdr64_shentsize, 1326 .ehdr_shstrndx = ehdr64_shstrndx, 1327 .ehdr_shnum = ehdr64_shnum, 1328 .shdr_addr = shdr64_addr, 1329 .shdr_offset = shdr64_offset, 1330 .shdr_link = shdr64_link, 1331 .shdr_size = shdr64_size, 1332 .shdr_name = shdr64_name, 1333 .shdr_type = shdr64_type, 1334 .shdr_entsize = shdr64_entsize, 1335 .sym_type = sym64_type, 1336 .sym_name = sym64_name, 1337 .sym_value = sym64_value, 1338 .sym_shndx = sym64_shndx, 1339 .rela_offset = rela64_offset, 1340 .rela_info = rela64_info, 1341 .rela_addend = rela64_addend, 1342 .rela_write_addend = rela64_write_addend, 1343 }; 1344 1345 e = efuncs; 1346 long_size = 8; 1347 extable_ent_size = 16; 1348 1349 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 1350 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 1351 fprintf(stderr, 1352 "unrecognized ET_EXEC/ET_DYN file: %s\n", 1353 fname); 1354 return -1; 1355 } 1356 1357 } 1358 break; 1359 default: 1360 fprintf(stderr, "unrecognized ELF class %d %s\n", 1361 ehdr->e32.e_ident[EI_CLASS], fname); 1362 return -1; 1363 } 1364 1365 return do_sort(ehdr, fname, custom_sort); 1366 } 1367 1368 int main(int argc, char *argv[]) 1369 { 1370 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 1371 size_t size = 0; 1372 void *addr = NULL; 1373 int c; 1374 1375 while ((c = getopt(argc, argv, "s:")) >= 0) { 1376 switch (c) { 1377 case 's': 1378 if (parse_symbols(optarg) < 0) { 1379 fprintf(stderr, "Could not parse %s\n", optarg); 1380 return -1; 1381 } 1382 break; 1383 default: 1384 fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n"); 1385 return 0; 1386 } 1387 } 1388 1389 if ((argc - optind) < 1) { 1390 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1391 return 0; 1392 } 1393 1394 /* Process each file in turn, allowing deep failure. */ 1395 for (i = optind; i < argc; i++) { 1396 addr = mmap_file(argv[i], &size); 1397 if (!addr) { 1398 ++n_error; 1399 continue; 1400 } 1401 1402 if (do_file(argv[i], addr)) 1403 ++n_error; 1404 1405 munmap(addr, size); 1406 } 1407 1408 return !!n_error; 1409 } 1410