1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <[email protected]> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/mman.h> 25 #include <sys/stat.h> 26 #include <getopt.h> 27 #include <elf.h> 28 #include <fcntl.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <unistd.h> 33 #include <errno.h> 34 #include <pthread.h> 35 36 #include <tools/be_byteshift.h> 37 #include <tools/le_byteshift.h> 38 39 #ifndef EM_ARCOMPACT 40 #define EM_ARCOMPACT 93 41 #endif 42 43 #ifndef EM_XTENSA 44 #define EM_XTENSA 94 45 #endif 46 47 #ifndef EM_AARCH64 48 #define EM_AARCH64 183 49 #endif 50 51 #ifndef EM_MICROBLAZE 52 #define EM_MICROBLAZE 189 53 #endif 54 55 #ifndef EM_ARCV2 56 #define EM_ARCV2 195 57 #endif 58 59 #ifndef EM_RISCV 60 #define EM_RISCV 243 61 #endif 62 63 #ifndef EM_LOONGARCH 64 #define EM_LOONGARCH 258 65 #endif 66 67 typedef union { 68 Elf32_Ehdr e32; 69 Elf64_Ehdr e64; 70 } Elf_Ehdr; 71 72 typedef union { 73 Elf32_Shdr e32; 74 Elf64_Shdr e64; 75 } Elf_Shdr; 76 77 typedef union { 78 Elf32_Sym e32; 79 Elf64_Sym e64; 80 } Elf_Sym; 81 82 static uint32_t (*r)(const uint32_t *); 83 static uint16_t (*r2)(const uint16_t *); 84 static uint64_t (*r8)(const uint64_t *); 85 static void (*w)(uint32_t, uint32_t *); 86 typedef void (*table_sort_t)(char *, int); 87 88 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) 89 { 90 return r8(&ehdr->e64.e_shoff); 91 } 92 93 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr) 94 { 95 return r(&ehdr->e32.e_shoff); 96 } 97 98 #define EHDR_HALF(fn_name) \ 99 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \ 100 { \ 101 return r2(&ehdr->e64.e_##fn_name); \ 102 } \ 103 \ 104 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \ 105 { \ 106 return r2(&ehdr->e32.e_##fn_name); \ 107 } 108 109 EHDR_HALF(shentsize) 110 EHDR_HALF(shstrndx) 111 EHDR_HALF(shnum) 112 113 #define SHDR_WORD(fn_name) \ 114 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 115 { \ 116 return r(&shdr->e64.sh_##fn_name); \ 117 } \ 118 \ 119 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 120 { \ 121 return r(&shdr->e32.sh_##fn_name); \ 122 } 123 124 #define SHDR_ADDR(fn_name) \ 125 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \ 126 { \ 127 return r8(&shdr->e64.sh_##fn_name); \ 128 } \ 129 \ 130 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \ 131 { \ 132 return r(&shdr->e32.sh_##fn_name); \ 133 } 134 135 #define SHDR_WORD(fn_name) \ 136 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 137 { \ 138 return r(&shdr->e64.sh_##fn_name); \ 139 } \ 140 \ 141 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 142 { \ 143 return r(&shdr->e32.sh_##fn_name); \ 144 } 145 146 SHDR_ADDR(addr) 147 SHDR_ADDR(offset) 148 SHDR_ADDR(size) 149 SHDR_ADDR(entsize) 150 151 SHDR_WORD(link) 152 SHDR_WORD(name) 153 SHDR_WORD(type) 154 155 #define SYM_ADDR(fn_name) \ 156 static uint64_t sym64_##fn_name(Elf_Sym *sym) \ 157 { \ 158 return r8(&sym->e64.st_##fn_name); \ 159 } \ 160 \ 161 static uint64_t sym32_##fn_name(Elf_Sym *sym) \ 162 { \ 163 return r(&sym->e32.st_##fn_name); \ 164 } 165 166 #define SYM_WORD(fn_name) \ 167 static uint32_t sym64_##fn_name(Elf_Sym *sym) \ 168 { \ 169 return r(&sym->e64.st_##fn_name); \ 170 } \ 171 \ 172 static uint32_t sym32_##fn_name(Elf_Sym *sym) \ 173 { \ 174 return r(&sym->e32.st_##fn_name); \ 175 } 176 177 #define SYM_HALF(fn_name) \ 178 static uint16_t sym64_##fn_name(Elf_Sym *sym) \ 179 { \ 180 return r2(&sym->e64.st_##fn_name); \ 181 } \ 182 \ 183 static uint16_t sym32_##fn_name(Elf_Sym *sym) \ 184 { \ 185 return r2(&sym->e32.st_##fn_name); \ 186 } 187 188 static uint8_t sym64_type(Elf_Sym *sym) 189 { 190 return ELF64_ST_TYPE(sym->e64.st_info); 191 } 192 193 static uint8_t sym32_type(Elf_Sym *sym) 194 { 195 return ELF32_ST_TYPE(sym->e32.st_info); 196 } 197 198 SYM_ADDR(value) 199 SYM_WORD(name) 200 SYM_HALF(shndx) 201 202 /* 203 * Get the whole file as a programming convenience in order to avoid 204 * malloc+lseek+read+free of many pieces. If successful, then mmap 205 * avoids copying unused pieces; else just read the whole file. 206 * Open for both read and write. 207 */ 208 static void *mmap_file(char const *fname, size_t *size) 209 { 210 int fd; 211 struct stat sb; 212 void *addr = NULL; 213 214 fd = open(fname, O_RDWR); 215 if (fd < 0) { 216 perror(fname); 217 return NULL; 218 } 219 if (fstat(fd, &sb) < 0) { 220 perror(fname); 221 goto out; 222 } 223 if (!S_ISREG(sb.st_mode)) { 224 fprintf(stderr, "not a regular file: %s\n", fname); 225 goto out; 226 } 227 228 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 229 if (addr == MAP_FAILED) { 230 fprintf(stderr, "Could not mmap file: %s\n", fname); 231 goto out; 232 } 233 234 *size = sb.st_size; 235 236 out: 237 close(fd); 238 return addr; 239 } 240 241 static uint32_t rbe(const uint32_t *x) 242 { 243 return get_unaligned_be32(x); 244 } 245 246 static uint16_t r2be(const uint16_t *x) 247 { 248 return get_unaligned_be16(x); 249 } 250 251 static uint64_t r8be(const uint64_t *x) 252 { 253 return get_unaligned_be64(x); 254 } 255 256 static uint32_t rle(const uint32_t *x) 257 { 258 return get_unaligned_le32(x); 259 } 260 261 static uint16_t r2le(const uint16_t *x) 262 { 263 return get_unaligned_le16(x); 264 } 265 266 static uint64_t r8le(const uint64_t *x) 267 { 268 return get_unaligned_le64(x); 269 } 270 271 static void wbe(uint32_t val, uint32_t *x) 272 { 273 put_unaligned_be32(val, x); 274 } 275 276 static void wle(uint32_t val, uint32_t *x) 277 { 278 put_unaligned_le32(val, x); 279 } 280 281 /* 282 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 283 * the way to -256..-1, to avoid conflicting with real section 284 * indices. 285 */ 286 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 287 288 static inline int is_shndx_special(unsigned int i) 289 { 290 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 291 } 292 293 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 294 static inline unsigned int get_secindex(unsigned int shndx, 295 unsigned int sym_offs, 296 const Elf32_Word *symtab_shndx_start) 297 { 298 if (is_shndx_special(shndx)) 299 return SPECIAL(shndx); 300 if (shndx != SHN_XINDEX) 301 return shndx; 302 return r(&symtab_shndx_start[sym_offs]); 303 } 304 305 static int compare_extable_32(const void *a, const void *b) 306 { 307 Elf32_Addr av = r(a); 308 Elf32_Addr bv = r(b); 309 310 if (av < bv) 311 return -1; 312 return av > bv; 313 } 314 315 static int compare_extable_64(const void *a, const void *b) 316 { 317 Elf64_Addr av = r8(a); 318 Elf64_Addr bv = r8(b); 319 320 if (av < bv) 321 return -1; 322 return av > bv; 323 } 324 325 static inline void *get_index(void *start, int entsize, int index) 326 { 327 return start + (entsize * index); 328 } 329 330 331 static int (*compare_extable)(const void *a, const void *b); 332 static uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr); 333 static uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr); 334 static uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr); 335 static uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr); 336 static uint64_t (*shdr_addr)(Elf_Shdr *shdr); 337 static uint64_t (*shdr_offset)(Elf_Shdr *shdr); 338 static uint64_t (*shdr_size)(Elf_Shdr *shdr); 339 static uint64_t (*shdr_entsize)(Elf_Shdr *shdr); 340 static uint32_t (*shdr_link)(Elf_Shdr *shdr); 341 static uint32_t (*shdr_name)(Elf_Shdr *shdr); 342 static uint32_t (*shdr_type)(Elf_Shdr *shdr); 343 static uint8_t (*sym_type)(Elf_Sym *sym); 344 static uint32_t (*sym_name)(Elf_Sym *sym); 345 static uint64_t (*sym_value)(Elf_Sym *sym); 346 static uint16_t (*sym_shndx)(Elf_Sym *sym); 347 348 static int extable_ent_size; 349 static int long_size; 350 351 352 #ifdef UNWINDER_ORC_ENABLED 353 /* ORC unwinder only support X86_64 */ 354 #include <asm/orc_types.h> 355 356 #define ERRSTR_MAXSZ 256 357 358 static char g_err[ERRSTR_MAXSZ]; 359 static int *g_orc_ip_table; 360 static struct orc_entry *g_orc_table; 361 362 static pthread_t orc_sort_thread; 363 364 static inline unsigned long orc_ip(const int *ip) 365 { 366 return (unsigned long)ip + *ip; 367 } 368 369 static int orc_sort_cmp(const void *_a, const void *_b) 370 { 371 struct orc_entry *orc_a, *orc_b; 372 const int *a = g_orc_ip_table + *(int *)_a; 373 const int *b = g_orc_ip_table + *(int *)_b; 374 unsigned long a_val = orc_ip(a); 375 unsigned long b_val = orc_ip(b); 376 377 if (a_val > b_val) 378 return 1; 379 if (a_val < b_val) 380 return -1; 381 382 /* 383 * The "weak" section terminator entries need to always be on the left 384 * to ensure the lookup code skips them in favor of real entries. 385 * These terminator entries exist to handle any gaps created by 386 * whitelisted .o files which didn't get objtool generation. 387 */ 388 orc_a = g_orc_table + (a - g_orc_ip_table); 389 orc_b = g_orc_table + (b - g_orc_ip_table); 390 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 391 return 0; 392 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 393 } 394 395 static void *sort_orctable(void *arg) 396 { 397 int i; 398 int *idxs = NULL; 399 int *tmp_orc_ip_table = NULL; 400 struct orc_entry *tmp_orc_table = NULL; 401 unsigned int *orc_ip_size = (unsigned int *)arg; 402 unsigned int num_entries = *orc_ip_size / sizeof(int); 403 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 404 405 idxs = (int *)malloc(*orc_ip_size); 406 if (!idxs) { 407 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 408 strerror(errno)); 409 pthread_exit(g_err); 410 } 411 412 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 413 if (!tmp_orc_ip_table) { 414 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 415 strerror(errno)); 416 pthread_exit(g_err); 417 } 418 419 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 420 if (!tmp_orc_table) { 421 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 422 strerror(errno)); 423 pthread_exit(g_err); 424 } 425 426 /* initialize indices array, convert ip_table to absolute address */ 427 for (i = 0; i < num_entries; i++) { 428 idxs[i] = i; 429 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 430 } 431 memcpy(tmp_orc_table, g_orc_table, orc_size); 432 433 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 434 435 for (i = 0; i < num_entries; i++) { 436 if (idxs[i] == i) 437 continue; 438 439 /* convert back to relative address */ 440 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 441 g_orc_table[i] = tmp_orc_table[idxs[i]]; 442 } 443 444 free(idxs); 445 free(tmp_orc_ip_table); 446 free(tmp_orc_table); 447 pthread_exit(NULL); 448 } 449 #endif 450 451 #ifdef MCOUNT_SORT_ENABLED 452 static pthread_t mcount_sort_thread; 453 454 struct elf_mcount_loc { 455 Elf_Ehdr *ehdr; 456 Elf_Shdr *init_data_sec; 457 uint64_t start_mcount_loc; 458 uint64_t stop_mcount_loc; 459 }; 460 461 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 462 static void *sort_mcount_loc(void *arg) 463 { 464 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 465 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 466 + shdr_offset(emloc->init_data_sec); 467 uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 468 unsigned char *start_loc = (void *)emloc->ehdr + offset; 469 470 qsort(start_loc, count/long_size, long_size, compare_extable); 471 return NULL; 472 } 473 474 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 475 static void get_mcount_loc(uint64_t *_start, uint64_t *_stop) 476 { 477 FILE *file_start, *file_stop; 478 char start_buff[20]; 479 char stop_buff[20]; 480 int len = 0; 481 482 file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r"); 483 if (!file_start) { 484 fprintf(stderr, "get start_mcount_loc error!"); 485 return; 486 } 487 488 file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r"); 489 if (!file_stop) { 490 fprintf(stderr, "get stop_mcount_loc error!"); 491 pclose(file_start); 492 return; 493 } 494 495 while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { 496 len = strlen(start_buff); 497 start_buff[len - 1] = '\0'; 498 } 499 *_start = strtoul(start_buff, NULL, 16); 500 501 while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { 502 len = strlen(stop_buff); 503 stop_buff[len - 1] = '\0'; 504 } 505 *_stop = strtoul(stop_buff, NULL, 16); 506 507 pclose(file_start); 508 pclose(file_stop); 509 } 510 #endif 511 static int do_sort(Elf_Ehdr *ehdr, 512 char const *const fname, 513 table_sort_t custom_sort) 514 { 515 int rc = -1; 516 Elf_Shdr *shdr_start; 517 Elf_Shdr *strtab_sec = NULL; 518 Elf_Shdr *symtab_sec = NULL; 519 Elf_Shdr *extab_sec = NULL; 520 Elf_Shdr *string_sec; 521 Elf_Sym *sym; 522 const Elf_Sym *symtab; 523 Elf32_Word *symtab_shndx = NULL; 524 Elf_Sym *sort_needed_sym = NULL; 525 Elf_Shdr *sort_needed_sec; 526 uint32_t *sort_needed_loc; 527 void *sym_start; 528 void *sym_end; 529 const char *secstrings; 530 const char *strtab; 531 char *extab_image; 532 int sort_need_index; 533 int symentsize; 534 int shentsize; 535 int idx; 536 int i; 537 unsigned int shnum; 538 unsigned int shstrndx; 539 #ifdef MCOUNT_SORT_ENABLED 540 struct elf_mcount_loc mstruct = {0}; 541 uint64_t _start_mcount_loc = 0; 542 uint64_t _stop_mcount_loc = 0; 543 #endif 544 #ifdef UNWINDER_ORC_ENABLED 545 unsigned int orc_ip_size = 0; 546 unsigned int orc_size = 0; 547 unsigned int orc_num_entries = 0; 548 #endif 549 550 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 551 shentsize = ehdr_shentsize(ehdr); 552 553 shstrndx = ehdr_shstrndx(ehdr); 554 if (shstrndx == SHN_XINDEX) 555 shstrndx = shdr_link(shdr_start); 556 string_sec = get_index(shdr_start, shentsize, shstrndx); 557 secstrings = (const char *)ehdr + shdr_offset(string_sec); 558 559 shnum = ehdr_shnum(ehdr); 560 if (shnum == SHN_UNDEF) 561 shnum = shdr_size(shdr_start); 562 563 for (i = 0; i < shnum; i++) { 564 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 565 566 idx = shdr_name(shdr); 567 if (!strcmp(secstrings + idx, "__ex_table")) 568 extab_sec = shdr; 569 if (!strcmp(secstrings + idx, ".symtab")) 570 symtab_sec = shdr; 571 if (!strcmp(secstrings + idx, ".strtab")) 572 strtab_sec = shdr; 573 574 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 575 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 576 shdr_offset(shdr)); 577 578 #ifdef MCOUNT_SORT_ENABLED 579 /* locate the .init.data section in vmlinux */ 580 if (!strcmp(secstrings + idx, ".init.data")) { 581 get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); 582 mstruct.ehdr = ehdr; 583 mstruct.init_data_sec = shdr; 584 mstruct.start_mcount_loc = _start_mcount_loc; 585 mstruct.stop_mcount_loc = _stop_mcount_loc; 586 } 587 #endif 588 589 #ifdef UNWINDER_ORC_ENABLED 590 /* locate the ORC unwind tables */ 591 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 592 orc_ip_size = shdr_size(shdr); 593 g_orc_ip_table = (int *)((void *)ehdr + 594 shdr_offset(shdr)); 595 } 596 if (!strcmp(secstrings + idx, ".orc_unwind")) { 597 orc_size = shdr_size(shdr); 598 g_orc_table = (struct orc_entry *)((void *)ehdr + 599 shdr_offset(shdr)); 600 } 601 #endif 602 } /* for loop */ 603 604 #ifdef UNWINDER_ORC_ENABLED 605 if (!g_orc_ip_table || !g_orc_table) { 606 fprintf(stderr, 607 "incomplete ORC unwind tables in file: %s\n", fname); 608 goto out; 609 } 610 611 orc_num_entries = orc_ip_size / sizeof(int); 612 if (orc_ip_size % sizeof(int) != 0 || 613 orc_size % sizeof(struct orc_entry) != 0 || 614 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 615 fprintf(stderr, 616 "inconsistent ORC unwind table entries in file: %s\n", 617 fname); 618 goto out; 619 } 620 621 /* create thread to sort ORC unwind tables concurrently */ 622 if (pthread_create(&orc_sort_thread, NULL, 623 sort_orctable, &orc_ip_size)) { 624 fprintf(stderr, 625 "pthread_create orc_sort_thread failed '%s': %s\n", 626 strerror(errno), fname); 627 goto out; 628 } 629 #endif 630 631 #ifdef MCOUNT_SORT_ENABLED 632 if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { 633 fprintf(stderr, 634 "incomplete mcount's sort in file: %s\n", 635 fname); 636 goto out; 637 } 638 639 /* create thread to sort mcount_loc concurrently */ 640 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 641 fprintf(stderr, 642 "pthread_create mcount_sort_thread failed '%s': %s\n", 643 strerror(errno), fname); 644 goto out; 645 } 646 #endif 647 if (!extab_sec) { 648 fprintf(stderr, "no __ex_table in file: %s\n", fname); 649 goto out; 650 } 651 652 if (!symtab_sec) { 653 fprintf(stderr, "no .symtab in file: %s\n", fname); 654 goto out; 655 } 656 657 if (!strtab_sec) { 658 fprintf(stderr, "no .strtab in file: %s\n", fname); 659 goto out; 660 } 661 662 extab_image = (void *)ehdr + shdr_offset(extab_sec); 663 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 664 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 665 666 if (custom_sort) { 667 custom_sort(extab_image, shdr_size(extab_sec)); 668 } else { 669 int num_entries = shdr_size(extab_sec) / extable_ent_size; 670 qsort(extab_image, num_entries, 671 extable_ent_size, compare_extable); 672 } 673 674 /* find the flag main_extable_sort_needed */ 675 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 676 sym_end = sym_start + shdr_size(symtab_sec); 677 symentsize = shdr_entsize(symtab_sec); 678 679 for (sym = sym_start; (void *)sym + symentsize < sym_end; 680 sym = (void *)sym + symentsize) { 681 if (sym_type(sym) != STT_OBJECT) 682 continue; 683 if (!strcmp(strtab + sym_name(sym), 684 "main_extable_sort_needed")) { 685 sort_needed_sym = sym; 686 break; 687 } 688 } 689 690 if (!sort_needed_sym) { 691 fprintf(stderr, 692 "no main_extable_sort_needed symbol in file: %s\n", 693 fname); 694 goto out; 695 } 696 697 sort_need_index = get_secindex(sym_shndx(sym), 698 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 699 symtab_shndx); 700 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 701 sort_needed_loc = (void *)ehdr + 702 shdr_offset(sort_needed_sec) + 703 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 704 705 /* extable has been sorted, clear the flag */ 706 w(0, sort_needed_loc); 707 rc = 0; 708 709 out: 710 #ifdef UNWINDER_ORC_ENABLED 711 if (orc_sort_thread) { 712 void *retval = NULL; 713 /* wait for ORC tables sort done */ 714 rc = pthread_join(orc_sort_thread, &retval); 715 if (rc) { 716 fprintf(stderr, 717 "pthread_join failed '%s': %s\n", 718 strerror(errno), fname); 719 } else if (retval) { 720 rc = -1; 721 fprintf(stderr, 722 "failed to sort ORC tables '%s': %s\n", 723 (char *)retval, fname); 724 } 725 } 726 #endif 727 728 #ifdef MCOUNT_SORT_ENABLED 729 if (mcount_sort_thread) { 730 void *retval = NULL; 731 /* wait for mcount sort done */ 732 rc = pthread_join(mcount_sort_thread, &retval); 733 if (rc) { 734 fprintf(stderr, 735 "pthread_join failed '%s': %s\n", 736 strerror(errno), fname); 737 } else if (retval) { 738 rc = -1; 739 fprintf(stderr, 740 "failed to sort mcount '%s': %s\n", 741 (char *)retval, fname); 742 } 743 } 744 #endif 745 return rc; 746 } 747 748 static int compare_relative_table(const void *a, const void *b) 749 { 750 int32_t av = (int32_t)r(a); 751 int32_t bv = (int32_t)r(b); 752 753 if (av < bv) 754 return -1; 755 if (av > bv) 756 return 1; 757 return 0; 758 } 759 760 static void sort_relative_table(char *extab_image, int image_size) 761 { 762 int i = 0; 763 764 /* 765 * Do the same thing the runtime sort does, first normalize to 766 * being relative to the start of the section. 767 */ 768 while (i < image_size) { 769 uint32_t *loc = (uint32_t *)(extab_image + i); 770 w(r(loc) + i, loc); 771 i += 4; 772 } 773 774 qsort(extab_image, image_size / 8, 8, compare_relative_table); 775 776 /* Now denormalize. */ 777 i = 0; 778 while (i < image_size) { 779 uint32_t *loc = (uint32_t *)(extab_image + i); 780 w(r(loc) - i, loc); 781 i += 4; 782 } 783 } 784 785 static void sort_relative_table_with_data(char *extab_image, int image_size) 786 { 787 int i = 0; 788 789 while (i < image_size) { 790 uint32_t *loc = (uint32_t *)(extab_image + i); 791 792 w(r(loc) + i, loc); 793 w(r(loc + 1) + i + 4, loc + 1); 794 /* Don't touch the fixup type or data */ 795 796 i += sizeof(uint32_t) * 3; 797 } 798 799 qsort(extab_image, image_size / 12, 12, compare_relative_table); 800 801 i = 0; 802 while (i < image_size) { 803 uint32_t *loc = (uint32_t *)(extab_image + i); 804 805 w(r(loc) - i, loc); 806 w(r(loc + 1) - (i + 4), loc + 1); 807 /* Don't touch the fixup type or data */ 808 809 i += sizeof(uint32_t) * 3; 810 } 811 } 812 813 static int do_file(char const *const fname, void *addr) 814 { 815 Elf_Ehdr *ehdr = addr; 816 table_sort_t custom_sort = NULL; 817 818 switch (ehdr->e32.e_ident[EI_DATA]) { 819 case ELFDATA2LSB: 820 r = rle; 821 r2 = r2le; 822 r8 = r8le; 823 w = wle; 824 break; 825 case ELFDATA2MSB: 826 r = rbe; 827 r2 = r2be; 828 r8 = r8be; 829 w = wbe; 830 break; 831 default: 832 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 833 ehdr->e32.e_ident[EI_DATA], fname); 834 return -1; 835 } 836 837 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 || 838 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) || 839 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) { 840 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); 841 return -1; 842 } 843 844 switch (r2(&ehdr->e32.e_machine)) { 845 case EM_386: 846 case EM_AARCH64: 847 case EM_LOONGARCH: 848 case EM_RISCV: 849 case EM_S390: 850 case EM_X86_64: 851 custom_sort = sort_relative_table_with_data; 852 break; 853 case EM_PARISC: 854 case EM_PPC: 855 case EM_PPC64: 856 custom_sort = sort_relative_table; 857 break; 858 case EM_ARCOMPACT: 859 case EM_ARCV2: 860 case EM_ARM: 861 case EM_MICROBLAZE: 862 case EM_MIPS: 863 case EM_XTENSA: 864 break; 865 default: 866 fprintf(stderr, "unrecognized e_machine %d %s\n", 867 r2(&ehdr->e32.e_machine), fname); 868 return -1; 869 } 870 871 switch (ehdr->e32.e_ident[EI_CLASS]) { 872 case ELFCLASS32: 873 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 874 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 875 fprintf(stderr, 876 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 877 return -1; 878 } 879 880 compare_extable = compare_extable_32; 881 ehdr_shoff = ehdr32_shoff; 882 ehdr_shentsize = ehdr32_shentsize; 883 ehdr_shstrndx = ehdr32_shstrndx; 884 ehdr_shnum = ehdr32_shnum; 885 shdr_addr = shdr32_addr; 886 shdr_offset = shdr32_offset; 887 shdr_link = shdr32_link; 888 shdr_size = shdr32_size; 889 shdr_name = shdr32_name; 890 shdr_type = shdr32_type; 891 shdr_entsize = shdr32_entsize; 892 sym_type = sym32_type; 893 sym_name = sym32_name; 894 sym_value = sym32_value; 895 sym_shndx = sym32_shndx; 896 long_size = 4; 897 extable_ent_size = 8; 898 break; 899 case ELFCLASS64: 900 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 901 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 902 fprintf(stderr, 903 "unrecognized ET_EXEC/ET_DYN file: %s\n", 904 fname); 905 return -1; 906 } 907 908 compare_extable = compare_extable_64; 909 ehdr_shoff = ehdr64_shoff; 910 ehdr_shentsize = ehdr64_shentsize; 911 ehdr_shstrndx = ehdr64_shstrndx; 912 ehdr_shnum = ehdr64_shnum; 913 shdr_addr = shdr64_addr; 914 shdr_offset = shdr64_offset; 915 shdr_link = shdr64_link; 916 shdr_size = shdr64_size; 917 shdr_name = shdr64_name; 918 shdr_type = shdr64_type; 919 shdr_entsize = shdr64_entsize; 920 sym_type = sym64_type; 921 sym_name = sym64_name; 922 sym_value = sym64_value; 923 sym_shndx = sym64_shndx; 924 long_size = 8; 925 extable_ent_size = 16; 926 927 break; 928 default: 929 fprintf(stderr, "unrecognized ELF class %d %s\n", 930 ehdr->e32.e_ident[EI_CLASS], fname); 931 return -1; 932 } 933 934 return do_sort(ehdr, fname, custom_sort); 935 } 936 937 int main(int argc, char *argv[]) 938 { 939 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 940 size_t size = 0; 941 void *addr = NULL; 942 943 if (argc < 2) { 944 fprintf(stderr, "usage: sorttable vmlinux...\n"); 945 return 0; 946 } 947 948 /* Process each file in turn, allowing deep failure. */ 949 for (i = 1; i < argc; i++) { 950 addr = mmap_file(argv[i], &size); 951 if (!addr) { 952 ++n_error; 953 continue; 954 } 955 956 if (do_file(argv[i], addr)) 957 ++n_error; 958 959 munmap(addr, size); 960 } 961 962 return !!n_error; 963 } 964