1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <[email protected]> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/mman.h> 25 #include <sys/stat.h> 26 #include <getopt.h> 27 #include <elf.h> 28 #include <fcntl.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <stdbool.h> 32 #include <string.h> 33 #include <unistd.h> 34 #include <errno.h> 35 #include <pthread.h> 36 37 #include <tools/be_byteshift.h> 38 #include <tools/le_byteshift.h> 39 40 #ifndef EM_ARCOMPACT 41 #define EM_ARCOMPACT 93 42 #endif 43 44 #ifndef EM_XTENSA 45 #define EM_XTENSA 94 46 #endif 47 48 #ifndef EM_AARCH64 49 #define EM_AARCH64 183 50 #endif 51 52 #ifndef EM_MICROBLAZE 53 #define EM_MICROBLAZE 189 54 #endif 55 56 #ifndef EM_ARCV2 57 #define EM_ARCV2 195 58 #endif 59 60 #ifndef EM_RISCV 61 #define EM_RISCV 243 62 #endif 63 64 #ifndef EM_LOONGARCH 65 #define EM_LOONGARCH 258 66 #endif 67 68 typedef union { 69 Elf32_Ehdr e32; 70 Elf64_Ehdr e64; 71 } Elf_Ehdr; 72 73 typedef union { 74 Elf32_Shdr e32; 75 Elf64_Shdr e64; 76 } Elf_Shdr; 77 78 typedef union { 79 Elf32_Sym e32; 80 Elf64_Sym e64; 81 } Elf_Sym; 82 83 typedef union { 84 Elf32_Rela e32; 85 Elf64_Rela e64; 86 } Elf_Rela; 87 88 static uint32_t (*r)(const uint32_t *); 89 static uint16_t (*r2)(const uint16_t *); 90 static uint64_t (*r8)(const uint64_t *); 91 static void (*w)(uint32_t, uint32_t *); 92 static void (*w8)(uint64_t, uint64_t *); 93 typedef void (*table_sort_t)(char *, int); 94 95 static struct elf_funcs { 96 int (*compare_extable)(const void *a, const void *b); 97 uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr); 98 uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr); 99 uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr); 100 uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr); 101 uint64_t (*shdr_addr)(Elf_Shdr *shdr); 102 uint64_t (*shdr_offset)(Elf_Shdr *shdr); 103 uint64_t (*shdr_size)(Elf_Shdr *shdr); 104 uint64_t (*shdr_entsize)(Elf_Shdr *shdr); 105 uint32_t (*shdr_link)(Elf_Shdr *shdr); 106 uint32_t (*shdr_name)(Elf_Shdr *shdr); 107 uint32_t (*shdr_type)(Elf_Shdr *shdr); 108 uint8_t (*sym_type)(Elf_Sym *sym); 109 uint32_t (*sym_name)(Elf_Sym *sym); 110 uint64_t (*sym_value)(Elf_Sym *sym); 111 uint16_t (*sym_shndx)(Elf_Sym *sym); 112 uint64_t (*rela_offset)(Elf_Rela *rela); 113 uint64_t (*rela_info)(Elf_Rela *rela); 114 uint64_t (*rela_addend)(Elf_Rela *rela); 115 void (*rela_write_addend)(Elf_Rela *rela, uint64_t val); 116 } e; 117 118 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) 119 { 120 return r8(&ehdr->e64.e_shoff); 121 } 122 123 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr) 124 { 125 return r(&ehdr->e32.e_shoff); 126 } 127 128 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr) 129 { 130 return e.ehdr_shoff(ehdr); 131 } 132 133 #define EHDR_HALF(fn_name) \ 134 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \ 135 { \ 136 return r2(&ehdr->e64.e_##fn_name); \ 137 } \ 138 \ 139 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \ 140 { \ 141 return r2(&ehdr->e32.e_##fn_name); \ 142 } \ 143 \ 144 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr) \ 145 { \ 146 return e.ehdr_##fn_name(ehdr); \ 147 } 148 149 EHDR_HALF(shentsize) 150 EHDR_HALF(shstrndx) 151 EHDR_HALF(shnum) 152 153 #define SHDR_WORD(fn_name) \ 154 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 155 { \ 156 return r(&shdr->e64.sh_##fn_name); \ 157 } \ 158 \ 159 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 160 { \ 161 return r(&shdr->e32.sh_##fn_name); \ 162 } \ 163 \ 164 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 165 { \ 166 return e.shdr_##fn_name(shdr); \ 167 } 168 169 #define SHDR_ADDR(fn_name) \ 170 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \ 171 { \ 172 return r8(&shdr->e64.sh_##fn_name); \ 173 } \ 174 \ 175 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \ 176 { \ 177 return r(&shdr->e32.sh_##fn_name); \ 178 } \ 179 \ 180 static uint64_t shdr_##fn_name(Elf_Shdr *shdr) \ 181 { \ 182 return e.shdr_##fn_name(shdr); \ 183 } 184 185 #define SHDR_WORD(fn_name) \ 186 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 187 { \ 188 return r(&shdr->e64.sh_##fn_name); \ 189 } \ 190 \ 191 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 192 { \ 193 return r(&shdr->e32.sh_##fn_name); \ 194 } \ 195 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 196 { \ 197 return e.shdr_##fn_name(shdr); \ 198 } 199 200 SHDR_ADDR(addr) 201 SHDR_ADDR(offset) 202 SHDR_ADDR(size) 203 SHDR_ADDR(entsize) 204 205 SHDR_WORD(link) 206 SHDR_WORD(name) 207 SHDR_WORD(type) 208 209 #define SYM_ADDR(fn_name) \ 210 static uint64_t sym64_##fn_name(Elf_Sym *sym) \ 211 { \ 212 return r8(&sym->e64.st_##fn_name); \ 213 } \ 214 \ 215 static uint64_t sym32_##fn_name(Elf_Sym *sym) \ 216 { \ 217 return r(&sym->e32.st_##fn_name); \ 218 } \ 219 \ 220 static uint64_t sym_##fn_name(Elf_Sym *sym) \ 221 { \ 222 return e.sym_##fn_name(sym); \ 223 } 224 225 #define SYM_WORD(fn_name) \ 226 static uint32_t sym64_##fn_name(Elf_Sym *sym) \ 227 { \ 228 return r(&sym->e64.st_##fn_name); \ 229 } \ 230 \ 231 static uint32_t sym32_##fn_name(Elf_Sym *sym) \ 232 { \ 233 return r(&sym->e32.st_##fn_name); \ 234 } \ 235 \ 236 static uint32_t sym_##fn_name(Elf_Sym *sym) \ 237 { \ 238 return e.sym_##fn_name(sym); \ 239 } 240 241 #define SYM_HALF(fn_name) \ 242 static uint16_t sym64_##fn_name(Elf_Sym *sym) \ 243 { \ 244 return r2(&sym->e64.st_##fn_name); \ 245 } \ 246 \ 247 static uint16_t sym32_##fn_name(Elf_Sym *sym) \ 248 { \ 249 return r2(&sym->e32.st_##fn_name); \ 250 } \ 251 \ 252 static uint16_t sym_##fn_name(Elf_Sym *sym) \ 253 { \ 254 return e.sym_##fn_name(sym); \ 255 } 256 257 static uint8_t sym64_type(Elf_Sym *sym) 258 { 259 return ELF64_ST_TYPE(sym->e64.st_info); 260 } 261 262 static uint8_t sym32_type(Elf_Sym *sym) 263 { 264 return ELF32_ST_TYPE(sym->e32.st_info); 265 } 266 267 static uint8_t sym_type(Elf_Sym *sym) 268 { 269 return e.sym_type(sym); 270 } 271 272 SYM_ADDR(value) 273 SYM_WORD(name) 274 SYM_HALF(shndx) 275 276 #define __maybe_unused __attribute__((__unused__)) 277 278 #define RELA_ADDR(fn_name) \ 279 static uint64_t rela64_##fn_name(Elf_Rela *rela) \ 280 { \ 281 return r8((uint64_t *)&rela->e64.r_##fn_name); \ 282 } \ 283 \ 284 static uint64_t rela32_##fn_name(Elf_Rela *rela) \ 285 { \ 286 return r((uint32_t *)&rela->e32.r_##fn_name); \ 287 } \ 288 \ 289 static uint64_t __maybe_unused rela_##fn_name(Elf_Rela *rela) \ 290 { \ 291 return e.rela_##fn_name(rela); \ 292 } 293 294 RELA_ADDR(offset) 295 RELA_ADDR(info) 296 RELA_ADDR(addend) 297 298 static void rela64_write_addend(Elf_Rela *rela, uint64_t val) 299 { 300 w8(val, (uint64_t *)&rela->e64.r_addend); 301 } 302 303 static void rela32_write_addend(Elf_Rela *rela, uint64_t val) 304 { 305 w(val, (uint32_t *)&rela->e32.r_addend); 306 } 307 308 /* 309 * Get the whole file as a programming convenience in order to avoid 310 * malloc+lseek+read+free of many pieces. If successful, then mmap 311 * avoids copying unused pieces; else just read the whole file. 312 * Open for both read and write. 313 */ 314 static void *mmap_file(char const *fname, size_t *size) 315 { 316 int fd; 317 struct stat sb; 318 void *addr = NULL; 319 320 fd = open(fname, O_RDWR); 321 if (fd < 0) { 322 perror(fname); 323 return NULL; 324 } 325 if (fstat(fd, &sb) < 0) { 326 perror(fname); 327 goto out; 328 } 329 if (!S_ISREG(sb.st_mode)) { 330 fprintf(stderr, "not a regular file: %s\n", fname); 331 goto out; 332 } 333 334 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 335 if (addr == MAP_FAILED) { 336 fprintf(stderr, "Could not mmap file: %s\n", fname); 337 goto out; 338 } 339 340 *size = sb.st_size; 341 342 out: 343 close(fd); 344 return addr; 345 } 346 347 static uint32_t rbe(const uint32_t *x) 348 { 349 return get_unaligned_be32(x); 350 } 351 352 static uint16_t r2be(const uint16_t *x) 353 { 354 return get_unaligned_be16(x); 355 } 356 357 static uint64_t r8be(const uint64_t *x) 358 { 359 return get_unaligned_be64(x); 360 } 361 362 static uint32_t rle(const uint32_t *x) 363 { 364 return get_unaligned_le32(x); 365 } 366 367 static uint16_t r2le(const uint16_t *x) 368 { 369 return get_unaligned_le16(x); 370 } 371 372 static uint64_t r8le(const uint64_t *x) 373 { 374 return get_unaligned_le64(x); 375 } 376 377 static void wbe(uint32_t val, uint32_t *x) 378 { 379 put_unaligned_be32(val, x); 380 } 381 382 static void wle(uint32_t val, uint32_t *x) 383 { 384 put_unaligned_le32(val, x); 385 } 386 387 static void w8be(uint64_t val, uint64_t *x) 388 { 389 put_unaligned_be64(val, x); 390 } 391 392 static void w8le(uint64_t val, uint64_t *x) 393 { 394 put_unaligned_le64(val, x); 395 } 396 397 /* 398 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 399 * the way to -256..-1, to avoid conflicting with real section 400 * indices. 401 */ 402 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 403 404 static inline int is_shndx_special(unsigned int i) 405 { 406 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 407 } 408 409 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 410 static inline unsigned int get_secindex(unsigned int shndx, 411 unsigned int sym_offs, 412 const Elf32_Word *symtab_shndx_start) 413 { 414 if (is_shndx_special(shndx)) 415 return SPECIAL(shndx); 416 if (shndx != SHN_XINDEX) 417 return shndx; 418 return r(&symtab_shndx_start[sym_offs]); 419 } 420 421 static int compare_extable_32(const void *a, const void *b) 422 { 423 Elf32_Addr av = r(a); 424 Elf32_Addr bv = r(b); 425 426 if (av < bv) 427 return -1; 428 return av > bv; 429 } 430 431 static int compare_extable_64(const void *a, const void *b) 432 { 433 Elf64_Addr av = r8(a); 434 Elf64_Addr bv = r8(b); 435 436 if (av < bv) 437 return -1; 438 return av > bv; 439 } 440 441 static int compare_extable(const void *a, const void *b) 442 { 443 return e.compare_extable(a, b); 444 } 445 446 static inline void *get_index(void *start, int entsize, int index) 447 { 448 return start + (entsize * index); 449 } 450 451 static int extable_ent_size; 452 static int long_size; 453 454 #define ERRSTR_MAXSZ 256 455 456 #ifdef UNWINDER_ORC_ENABLED 457 /* ORC unwinder only support X86_64 */ 458 #include <asm/orc_types.h> 459 460 static char g_err[ERRSTR_MAXSZ]; 461 static int *g_orc_ip_table; 462 static struct orc_entry *g_orc_table; 463 464 static pthread_t orc_sort_thread; 465 466 static inline unsigned long orc_ip(const int *ip) 467 { 468 return (unsigned long)ip + *ip; 469 } 470 471 static int orc_sort_cmp(const void *_a, const void *_b) 472 { 473 struct orc_entry *orc_a, *orc_b; 474 const int *a = g_orc_ip_table + *(int *)_a; 475 const int *b = g_orc_ip_table + *(int *)_b; 476 unsigned long a_val = orc_ip(a); 477 unsigned long b_val = orc_ip(b); 478 479 if (a_val > b_val) 480 return 1; 481 if (a_val < b_val) 482 return -1; 483 484 /* 485 * The "weak" section terminator entries need to always be on the left 486 * to ensure the lookup code skips them in favor of real entries. 487 * These terminator entries exist to handle any gaps created by 488 * whitelisted .o files which didn't get objtool generation. 489 */ 490 orc_a = g_orc_table + (a - g_orc_ip_table); 491 orc_b = g_orc_table + (b - g_orc_ip_table); 492 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 493 return 0; 494 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 495 } 496 497 static void *sort_orctable(void *arg) 498 { 499 int i; 500 int *idxs = NULL; 501 int *tmp_orc_ip_table = NULL; 502 struct orc_entry *tmp_orc_table = NULL; 503 unsigned int *orc_ip_size = (unsigned int *)arg; 504 unsigned int num_entries = *orc_ip_size / sizeof(int); 505 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 506 507 idxs = (int *)malloc(*orc_ip_size); 508 if (!idxs) { 509 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 510 strerror(errno)); 511 pthread_exit(g_err); 512 } 513 514 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 515 if (!tmp_orc_ip_table) { 516 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 517 strerror(errno)); 518 pthread_exit(g_err); 519 } 520 521 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 522 if (!tmp_orc_table) { 523 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 524 strerror(errno)); 525 pthread_exit(g_err); 526 } 527 528 /* initialize indices array, convert ip_table to absolute address */ 529 for (i = 0; i < num_entries; i++) { 530 idxs[i] = i; 531 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 532 } 533 memcpy(tmp_orc_table, g_orc_table, orc_size); 534 535 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 536 537 for (i = 0; i < num_entries; i++) { 538 if (idxs[i] == i) 539 continue; 540 541 /* convert back to relative address */ 542 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 543 g_orc_table[i] = tmp_orc_table[idxs[i]]; 544 } 545 546 free(idxs); 547 free(tmp_orc_ip_table); 548 free(tmp_orc_table); 549 pthread_exit(NULL); 550 } 551 #endif 552 553 #ifdef MCOUNT_SORT_ENABLED 554 555 static int compare_values_64(const void *a, const void *b) 556 { 557 uint64_t av = *(uint64_t *)a; 558 uint64_t bv = *(uint64_t *)b; 559 560 if (av < bv) 561 return -1; 562 return av > bv; 563 } 564 565 static int compare_values_32(const void *a, const void *b) 566 { 567 uint32_t av = *(uint32_t *)a; 568 uint32_t bv = *(uint32_t *)b; 569 570 if (av < bv) 571 return -1; 572 return av > bv; 573 } 574 575 static int (*compare_values)(const void *a, const void *b); 576 577 /* Only used for sorting mcount table */ 578 static void rela_write_addend(Elf_Rela *rela, uint64_t val) 579 { 580 e.rela_write_addend(rela, val); 581 } 582 583 struct func_info { 584 uint64_t addr; 585 uint64_t size; 586 }; 587 588 /* List of functions created by: nm -S vmlinux */ 589 static struct func_info *function_list; 590 static int function_list_size; 591 592 /* Allocate functions in 1k blocks */ 593 #define FUNC_BLK_SIZE 1024 594 #define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1) 595 596 static int add_field(uint64_t addr, uint64_t size) 597 { 598 struct func_info *fi; 599 int fsize = function_list_size; 600 601 if (!(fsize & FUNC_BLK_MASK)) { 602 fsize += FUNC_BLK_SIZE; 603 fi = realloc(function_list, fsize * sizeof(struct func_info)); 604 if (!fi) 605 return -1; 606 function_list = fi; 607 } 608 fi = &function_list[function_list_size++]; 609 fi->addr = addr; 610 fi->size = size; 611 return 0; 612 } 613 614 /* Only return match if the address lies inside the function size */ 615 static int cmp_func_addr(const void *K, const void *A) 616 { 617 uint64_t key = *(const uint64_t *)K; 618 const struct func_info *a = A; 619 620 if (key < a->addr) 621 return -1; 622 return key >= a->addr + a->size; 623 } 624 625 /* Find the function in function list that is bounded by the function size */ 626 static int find_func(uint64_t key) 627 { 628 return bsearch(&key, function_list, function_list_size, 629 sizeof(struct func_info), cmp_func_addr) != NULL; 630 } 631 632 static int cmp_funcs(const void *A, const void *B) 633 { 634 const struct func_info *a = A; 635 const struct func_info *b = B; 636 637 if (a->addr < b->addr) 638 return -1; 639 return a->addr > b->addr; 640 } 641 642 static int parse_symbols(const char *fname) 643 { 644 FILE *fp; 645 char addr_str[20]; /* Only need 17, but round up to next int size */ 646 char size_str[20]; 647 char type; 648 649 fp = fopen(fname, "r"); 650 if (!fp) { 651 perror(fname); 652 return -1; 653 } 654 655 while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) { 656 uint64_t addr; 657 uint64_t size; 658 659 /* Only care about functions */ 660 if (type != 't' && type != 'T' && type != 'W') 661 continue; 662 663 addr = strtoull(addr_str, NULL, 16); 664 size = strtoull(size_str, NULL, 16); 665 if (add_field(addr, size) < 0) 666 return -1; 667 } 668 fclose(fp); 669 670 qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs); 671 672 return 0; 673 } 674 675 static pthread_t mcount_sort_thread; 676 static bool sort_reloc; 677 678 static long rela_type; 679 680 static char m_err[ERRSTR_MAXSZ]; 681 682 struct elf_mcount_loc { 683 Elf_Ehdr *ehdr; 684 Elf_Shdr *init_data_sec; 685 uint64_t start_mcount_loc; 686 uint64_t stop_mcount_loc; 687 }; 688 689 /* Fill the array with the content of the relocs */ 690 static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 691 { 692 Elf_Shdr *shdr_start; 693 Elf_Rela *rel; 694 unsigned int shnum; 695 unsigned int count = 0; 696 int shentsize; 697 void *array_end = ptr + size; 698 699 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 700 shentsize = ehdr_shentsize(ehdr); 701 702 shnum = ehdr_shnum(ehdr); 703 if (shnum == SHN_UNDEF) 704 shnum = shdr_size(shdr_start); 705 706 for (int i = 0; i < shnum; i++) { 707 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 708 void *end; 709 710 if (shdr_type(shdr) != SHT_RELA) 711 continue; 712 713 rel = (void *)ehdr + shdr_offset(shdr); 714 end = (void *)rel + shdr_size(shdr); 715 716 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 717 uint64_t offset = rela_offset(rel); 718 719 if (offset >= start_loc && offset < start_loc + size) { 720 if (ptr + long_size > array_end) { 721 snprintf(m_err, ERRSTR_MAXSZ, 722 "Too many relocations"); 723 return -1; 724 } 725 726 /* Make sure this has the correct type */ 727 if (rela_info(rel) != rela_type) { 728 snprintf(m_err, ERRSTR_MAXSZ, 729 "rela has type %lx but expected %lx\n", 730 (long)rela_info(rel), rela_type); 731 return -1; 732 } 733 734 if (long_size == 4) 735 *(uint32_t *)ptr = rela_addend(rel); 736 else 737 *(uint64_t *)ptr = rela_addend(rel); 738 ptr += long_size; 739 count++; 740 } 741 } 742 } 743 return count; 744 } 745 746 /* Put the sorted vals back into the relocation elements */ 747 static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 748 { 749 Elf_Shdr *shdr_start; 750 Elf_Rela *rel; 751 unsigned int shnum; 752 int shentsize; 753 754 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 755 shentsize = ehdr_shentsize(ehdr); 756 757 shnum = ehdr_shnum(ehdr); 758 if (shnum == SHN_UNDEF) 759 shnum = shdr_size(shdr_start); 760 761 for (int i = 0; i < shnum; i++) { 762 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 763 void *end; 764 765 if (shdr_type(shdr) != SHT_RELA) 766 continue; 767 768 rel = (void *)ehdr + shdr_offset(shdr); 769 end = (void *)rel + shdr_size(shdr); 770 771 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 772 uint64_t offset = rela_offset(rel); 773 774 if (offset >= start_loc && offset < start_loc + size) { 775 if (long_size == 4) 776 rela_write_addend(rel, *(uint32_t *)ptr); 777 else 778 rela_write_addend(rel, *(uint64_t *)ptr); 779 ptr += long_size; 780 } 781 } 782 } 783 } 784 785 static int fill_addrs(void *ptr, uint64_t size, void *addrs) 786 { 787 void *end = ptr + size; 788 int count = 0; 789 790 for (; ptr < end; ptr += long_size, addrs += long_size, count++) { 791 if (long_size == 4) 792 *(uint32_t *)ptr = r(addrs); 793 else 794 *(uint64_t *)ptr = r8(addrs); 795 } 796 return count; 797 } 798 799 static void replace_addrs(void *ptr, uint64_t size, void *addrs) 800 { 801 void *end = ptr + size; 802 803 for (; ptr < end; ptr += long_size, addrs += long_size) { 804 if (long_size == 4) 805 w(*(uint32_t *)ptr, addrs); 806 else 807 w8(*(uint64_t *)ptr, addrs); 808 } 809 } 810 811 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 812 static void *sort_mcount_loc(void *arg) 813 { 814 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 815 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 816 + shdr_offset(emloc->init_data_sec); 817 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc; 818 unsigned char *start_loc = (void *)emloc->ehdr + offset; 819 Elf_Ehdr *ehdr = emloc->ehdr; 820 void *e_msg = NULL; 821 void *vals; 822 int count; 823 824 vals = malloc(long_size * size); 825 if (!vals) { 826 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 827 pthread_exit(m_err); 828 } 829 830 if (sort_reloc) 831 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc); 832 else 833 count = fill_addrs(vals, size, start_loc); 834 835 if (count < 0) { 836 e_msg = m_err; 837 goto out; 838 } 839 840 if (count != size / long_size) { 841 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n", 842 (int)(size / long_size), count); 843 e_msg = m_err; 844 goto out; 845 } 846 847 /* zero out any locations not found by function list */ 848 if (function_list_size) { 849 for (void *ptr = vals; ptr < vals + size; ptr += long_size) { 850 uint64_t key; 851 852 key = long_size == 4 ? r((uint32_t *)ptr) : r8((uint64_t *)ptr); 853 if (!find_func(key)) { 854 if (long_size == 4) 855 *(uint32_t *)ptr = 0; 856 else 857 *(uint64_t *)ptr = 0; 858 } 859 } 860 } 861 862 compare_values = long_size == 4 ? compare_values_32 : compare_values_64; 863 864 qsort(vals, count, long_size, compare_values); 865 866 if (sort_reloc) 867 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc); 868 else 869 replace_addrs(vals, size, start_loc); 870 871 out: 872 free(vals); 873 874 pthread_exit(e_msg); 875 } 876 877 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 878 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 879 const char *strtab) 880 { 881 Elf_Sym *sym, *end_sym; 882 int symentsize = shdr_entsize(symtab_sec); 883 int found = 0; 884 885 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 886 end_sym = (void *)sym + shdr_size(symtab_sec); 887 888 while (sym < end_sym) { 889 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 890 emloc->start_mcount_loc = sym_value(sym); 891 if (++found == 2) 892 break; 893 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 894 emloc->stop_mcount_loc = sym_value(sym); 895 if (++found == 2) 896 break; 897 } 898 sym = (void *)sym + symentsize; 899 } 900 901 if (!emloc->start_mcount_loc) { 902 fprintf(stderr, "get start_mcount_loc error!"); 903 return; 904 } 905 906 if (!emloc->stop_mcount_loc) { 907 fprintf(stderr, "get stop_mcount_loc error!"); 908 return; 909 } 910 } 911 #else /* MCOUNT_SORT_ENABLED */ 912 static inline int parse_symbols(const char *fname) { return 0; } 913 #endif 914 915 static int do_sort(Elf_Ehdr *ehdr, 916 char const *const fname, 917 table_sort_t custom_sort) 918 { 919 int rc = -1; 920 Elf_Shdr *shdr_start; 921 Elf_Shdr *strtab_sec = NULL; 922 Elf_Shdr *symtab_sec = NULL; 923 Elf_Shdr *extab_sec = NULL; 924 Elf_Shdr *string_sec; 925 Elf_Sym *sym; 926 const Elf_Sym *symtab; 927 Elf32_Word *symtab_shndx = NULL; 928 Elf_Sym *sort_needed_sym = NULL; 929 Elf_Shdr *sort_needed_sec; 930 uint32_t *sort_needed_loc; 931 void *sym_start; 932 void *sym_end; 933 const char *secstrings; 934 const char *strtab; 935 char *extab_image; 936 int sort_need_index; 937 int symentsize; 938 int shentsize; 939 int idx; 940 int i; 941 unsigned int shnum; 942 unsigned int shstrndx; 943 #ifdef MCOUNT_SORT_ENABLED 944 struct elf_mcount_loc mstruct = {0}; 945 #endif 946 #ifdef UNWINDER_ORC_ENABLED 947 unsigned int orc_ip_size = 0; 948 unsigned int orc_size = 0; 949 unsigned int orc_num_entries = 0; 950 #endif 951 952 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 953 shentsize = ehdr_shentsize(ehdr); 954 955 shstrndx = ehdr_shstrndx(ehdr); 956 if (shstrndx == SHN_XINDEX) 957 shstrndx = shdr_link(shdr_start); 958 string_sec = get_index(shdr_start, shentsize, shstrndx); 959 secstrings = (const char *)ehdr + shdr_offset(string_sec); 960 961 shnum = ehdr_shnum(ehdr); 962 if (shnum == SHN_UNDEF) 963 shnum = shdr_size(shdr_start); 964 965 for (i = 0; i < shnum; i++) { 966 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 967 968 idx = shdr_name(shdr); 969 if (!strcmp(secstrings + idx, "__ex_table")) 970 extab_sec = shdr; 971 if (!strcmp(secstrings + idx, ".symtab")) 972 symtab_sec = shdr; 973 if (!strcmp(secstrings + idx, ".strtab")) 974 strtab_sec = shdr; 975 976 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 977 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 978 shdr_offset(shdr)); 979 980 #ifdef MCOUNT_SORT_ENABLED 981 /* locate the .init.data section in vmlinux */ 982 if (!strcmp(secstrings + idx, ".init.data")) 983 mstruct.init_data_sec = shdr; 984 #endif 985 986 #ifdef UNWINDER_ORC_ENABLED 987 /* locate the ORC unwind tables */ 988 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 989 orc_ip_size = shdr_size(shdr); 990 g_orc_ip_table = (int *)((void *)ehdr + 991 shdr_offset(shdr)); 992 } 993 if (!strcmp(secstrings + idx, ".orc_unwind")) { 994 orc_size = shdr_size(shdr); 995 g_orc_table = (struct orc_entry *)((void *)ehdr + 996 shdr_offset(shdr)); 997 } 998 #endif 999 } /* for loop */ 1000 1001 #ifdef UNWINDER_ORC_ENABLED 1002 if (!g_orc_ip_table || !g_orc_table) { 1003 fprintf(stderr, 1004 "incomplete ORC unwind tables in file: %s\n", fname); 1005 goto out; 1006 } 1007 1008 orc_num_entries = orc_ip_size / sizeof(int); 1009 if (orc_ip_size % sizeof(int) != 0 || 1010 orc_size % sizeof(struct orc_entry) != 0 || 1011 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 1012 fprintf(stderr, 1013 "inconsistent ORC unwind table entries in file: %s\n", 1014 fname); 1015 goto out; 1016 } 1017 1018 /* create thread to sort ORC unwind tables concurrently */ 1019 if (pthread_create(&orc_sort_thread, NULL, 1020 sort_orctable, &orc_ip_size)) { 1021 fprintf(stderr, 1022 "pthread_create orc_sort_thread failed '%s': %s\n", 1023 strerror(errno), fname); 1024 goto out; 1025 } 1026 #endif 1027 if (!extab_sec) { 1028 fprintf(stderr, "no __ex_table in file: %s\n", fname); 1029 goto out; 1030 } 1031 1032 if (!symtab_sec) { 1033 fprintf(stderr, "no .symtab in file: %s\n", fname); 1034 goto out; 1035 } 1036 1037 if (!strtab_sec) { 1038 fprintf(stderr, "no .strtab in file: %s\n", fname); 1039 goto out; 1040 } 1041 1042 extab_image = (void *)ehdr + shdr_offset(extab_sec); 1043 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 1044 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 1045 1046 #ifdef MCOUNT_SORT_ENABLED 1047 mstruct.ehdr = ehdr; 1048 get_mcount_loc(&mstruct, symtab_sec, strtab); 1049 1050 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 1051 fprintf(stderr, 1052 "incomplete mcount's sort in file: %s\n", 1053 fname); 1054 goto out; 1055 } 1056 1057 /* create thread to sort mcount_loc concurrently */ 1058 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 1059 fprintf(stderr, 1060 "pthread_create mcount_sort_thread failed '%s': %s\n", 1061 strerror(errno), fname); 1062 goto out; 1063 } 1064 #endif 1065 1066 if (custom_sort) { 1067 custom_sort(extab_image, shdr_size(extab_sec)); 1068 } else { 1069 int num_entries = shdr_size(extab_sec) / extable_ent_size; 1070 qsort(extab_image, num_entries, 1071 extable_ent_size, compare_extable); 1072 } 1073 1074 /* find the flag main_extable_sort_needed */ 1075 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 1076 sym_end = sym_start + shdr_size(symtab_sec); 1077 symentsize = shdr_entsize(symtab_sec); 1078 1079 for (sym = sym_start; (void *)sym + symentsize < sym_end; 1080 sym = (void *)sym + symentsize) { 1081 if (sym_type(sym) != STT_OBJECT) 1082 continue; 1083 if (!strcmp(strtab + sym_name(sym), 1084 "main_extable_sort_needed")) { 1085 sort_needed_sym = sym; 1086 break; 1087 } 1088 } 1089 1090 if (!sort_needed_sym) { 1091 fprintf(stderr, 1092 "no main_extable_sort_needed symbol in file: %s\n", 1093 fname); 1094 goto out; 1095 } 1096 1097 sort_need_index = get_secindex(sym_shndx(sym), 1098 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 1099 symtab_shndx); 1100 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 1101 sort_needed_loc = (void *)ehdr + 1102 shdr_offset(sort_needed_sec) + 1103 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 1104 1105 /* extable has been sorted, clear the flag */ 1106 w(0, sort_needed_loc); 1107 rc = 0; 1108 1109 out: 1110 #ifdef UNWINDER_ORC_ENABLED 1111 if (orc_sort_thread) { 1112 void *retval = NULL; 1113 /* wait for ORC tables sort done */ 1114 rc = pthread_join(orc_sort_thread, &retval); 1115 if (rc) { 1116 fprintf(stderr, 1117 "pthread_join failed '%s': %s\n", 1118 strerror(errno), fname); 1119 } else if (retval) { 1120 rc = -1; 1121 fprintf(stderr, 1122 "failed to sort ORC tables '%s': %s\n", 1123 (char *)retval, fname); 1124 } 1125 } 1126 #endif 1127 1128 #ifdef MCOUNT_SORT_ENABLED 1129 if (mcount_sort_thread) { 1130 void *retval = NULL; 1131 /* wait for mcount sort done */ 1132 rc = pthread_join(mcount_sort_thread, &retval); 1133 if (rc) { 1134 fprintf(stderr, 1135 "pthread_join failed '%s': %s\n", 1136 strerror(errno), fname); 1137 } else if (retval) { 1138 rc = -1; 1139 fprintf(stderr, 1140 "failed to sort mcount '%s': %s\n", 1141 (char *)retval, fname); 1142 } 1143 } 1144 #endif 1145 return rc; 1146 } 1147 1148 static int compare_relative_table(const void *a, const void *b) 1149 { 1150 int32_t av = (int32_t)r(a); 1151 int32_t bv = (int32_t)r(b); 1152 1153 if (av < bv) 1154 return -1; 1155 if (av > bv) 1156 return 1; 1157 return 0; 1158 } 1159 1160 static void sort_relative_table(char *extab_image, int image_size) 1161 { 1162 int i = 0; 1163 1164 /* 1165 * Do the same thing the runtime sort does, first normalize to 1166 * being relative to the start of the section. 1167 */ 1168 while (i < image_size) { 1169 uint32_t *loc = (uint32_t *)(extab_image + i); 1170 w(r(loc) + i, loc); 1171 i += 4; 1172 } 1173 1174 qsort(extab_image, image_size / 8, 8, compare_relative_table); 1175 1176 /* Now denormalize. */ 1177 i = 0; 1178 while (i < image_size) { 1179 uint32_t *loc = (uint32_t *)(extab_image + i); 1180 w(r(loc) - i, loc); 1181 i += 4; 1182 } 1183 } 1184 1185 static void sort_relative_table_with_data(char *extab_image, int image_size) 1186 { 1187 int i = 0; 1188 1189 while (i < image_size) { 1190 uint32_t *loc = (uint32_t *)(extab_image + i); 1191 1192 w(r(loc) + i, loc); 1193 w(r(loc + 1) + i + 4, loc + 1); 1194 /* Don't touch the fixup type or data */ 1195 1196 i += sizeof(uint32_t) * 3; 1197 } 1198 1199 qsort(extab_image, image_size / 12, 12, compare_relative_table); 1200 1201 i = 0; 1202 while (i < image_size) { 1203 uint32_t *loc = (uint32_t *)(extab_image + i); 1204 1205 w(r(loc) - i, loc); 1206 w(r(loc + 1) - (i + 4), loc + 1); 1207 /* Don't touch the fixup type or data */ 1208 1209 i += sizeof(uint32_t) * 3; 1210 } 1211 } 1212 1213 static int do_file(char const *const fname, void *addr) 1214 { 1215 Elf_Ehdr *ehdr = addr; 1216 table_sort_t custom_sort = NULL; 1217 1218 switch (ehdr->e32.e_ident[EI_DATA]) { 1219 case ELFDATA2LSB: 1220 r = rle; 1221 r2 = r2le; 1222 r8 = r8le; 1223 w = wle; 1224 w8 = w8le; 1225 break; 1226 case ELFDATA2MSB: 1227 r = rbe; 1228 r2 = r2be; 1229 r8 = r8be; 1230 w = wbe; 1231 w8 = w8be; 1232 break; 1233 default: 1234 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 1235 ehdr->e32.e_ident[EI_DATA], fname); 1236 return -1; 1237 } 1238 1239 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 || 1240 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) || 1241 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) { 1242 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); 1243 return -1; 1244 } 1245 1246 switch (r2(&ehdr->e32.e_machine)) { 1247 case EM_AARCH64: 1248 #ifdef MCOUNT_SORT_ENABLED 1249 sort_reloc = true; 1250 rela_type = 0x403; 1251 #endif 1252 /* fallthrough */ 1253 case EM_386: 1254 case EM_LOONGARCH: 1255 case EM_RISCV: 1256 case EM_S390: 1257 case EM_X86_64: 1258 custom_sort = sort_relative_table_with_data; 1259 break; 1260 case EM_PARISC: 1261 case EM_PPC: 1262 case EM_PPC64: 1263 custom_sort = sort_relative_table; 1264 break; 1265 case EM_ARCOMPACT: 1266 case EM_ARCV2: 1267 case EM_ARM: 1268 case EM_MICROBLAZE: 1269 case EM_MIPS: 1270 case EM_XTENSA: 1271 break; 1272 default: 1273 fprintf(stderr, "unrecognized e_machine %d %s\n", 1274 r2(&ehdr->e32.e_machine), fname); 1275 return -1; 1276 } 1277 1278 switch (ehdr->e32.e_ident[EI_CLASS]) { 1279 case ELFCLASS32: { 1280 struct elf_funcs efuncs = { 1281 .compare_extable = compare_extable_32, 1282 .ehdr_shoff = ehdr32_shoff, 1283 .ehdr_shentsize = ehdr32_shentsize, 1284 .ehdr_shstrndx = ehdr32_shstrndx, 1285 .ehdr_shnum = ehdr32_shnum, 1286 .shdr_addr = shdr32_addr, 1287 .shdr_offset = shdr32_offset, 1288 .shdr_link = shdr32_link, 1289 .shdr_size = shdr32_size, 1290 .shdr_name = shdr32_name, 1291 .shdr_type = shdr32_type, 1292 .shdr_entsize = shdr32_entsize, 1293 .sym_type = sym32_type, 1294 .sym_name = sym32_name, 1295 .sym_value = sym32_value, 1296 .sym_shndx = sym32_shndx, 1297 .rela_offset = rela32_offset, 1298 .rela_info = rela32_info, 1299 .rela_addend = rela32_addend, 1300 .rela_write_addend = rela32_write_addend, 1301 }; 1302 1303 e = efuncs; 1304 long_size = 4; 1305 extable_ent_size = 8; 1306 1307 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 1308 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 1309 fprintf(stderr, 1310 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 1311 return -1; 1312 } 1313 1314 } 1315 break; 1316 case ELFCLASS64: { 1317 struct elf_funcs efuncs = { 1318 .compare_extable = compare_extable_64, 1319 .ehdr_shoff = ehdr64_shoff, 1320 .ehdr_shentsize = ehdr64_shentsize, 1321 .ehdr_shstrndx = ehdr64_shstrndx, 1322 .ehdr_shnum = ehdr64_shnum, 1323 .shdr_addr = shdr64_addr, 1324 .shdr_offset = shdr64_offset, 1325 .shdr_link = shdr64_link, 1326 .shdr_size = shdr64_size, 1327 .shdr_name = shdr64_name, 1328 .shdr_type = shdr64_type, 1329 .shdr_entsize = shdr64_entsize, 1330 .sym_type = sym64_type, 1331 .sym_name = sym64_name, 1332 .sym_value = sym64_value, 1333 .sym_shndx = sym64_shndx, 1334 .rela_offset = rela64_offset, 1335 .rela_info = rela64_info, 1336 .rela_addend = rela64_addend, 1337 .rela_write_addend = rela64_write_addend, 1338 }; 1339 1340 e = efuncs; 1341 long_size = 8; 1342 extable_ent_size = 16; 1343 1344 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 1345 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 1346 fprintf(stderr, 1347 "unrecognized ET_EXEC/ET_DYN file: %s\n", 1348 fname); 1349 return -1; 1350 } 1351 1352 } 1353 break; 1354 default: 1355 fprintf(stderr, "unrecognized ELF class %d %s\n", 1356 ehdr->e32.e_ident[EI_CLASS], fname); 1357 return -1; 1358 } 1359 1360 return do_sort(ehdr, fname, custom_sort); 1361 } 1362 1363 int main(int argc, char *argv[]) 1364 { 1365 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 1366 size_t size = 0; 1367 void *addr = NULL; 1368 int c; 1369 1370 while ((c = getopt(argc, argv, "s:")) >= 0) { 1371 switch (c) { 1372 case 's': 1373 if (parse_symbols(optarg) < 0) { 1374 fprintf(stderr, "Could not parse %s\n", optarg); 1375 return -1; 1376 } 1377 break; 1378 default: 1379 fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n"); 1380 return 0; 1381 } 1382 } 1383 1384 if ((argc - optind) < 1) { 1385 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1386 return 0; 1387 } 1388 1389 /* Process each file in turn, allowing deep failure. */ 1390 for (i = optind; i < argc; i++) { 1391 addr = mmap_file(argv[i], &size); 1392 if (!addr) { 1393 ++n_error; 1394 continue; 1395 } 1396 1397 if (do_file(argv[i], addr)) 1398 ++n_error; 1399 1400 munmap(addr, size); 1401 } 1402 1403 return !!n_error; 1404 } 1405