1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <[email protected]> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/mman.h> 25 #include <sys/stat.h> 26 #include <getopt.h> 27 #include <elf.h> 28 #include <fcntl.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <stdbool.h> 32 #include <string.h> 33 #include <unistd.h> 34 #include <errno.h> 35 #include <pthread.h> 36 37 #include <tools/be_byteshift.h> 38 #include <tools/le_byteshift.h> 39 40 #ifndef EM_ARCOMPACT 41 #define EM_ARCOMPACT 93 42 #endif 43 44 #ifndef EM_XTENSA 45 #define EM_XTENSA 94 46 #endif 47 48 #ifndef EM_AARCH64 49 #define EM_AARCH64 183 50 #endif 51 52 #ifndef EM_MICROBLAZE 53 #define EM_MICROBLAZE 189 54 #endif 55 56 #ifndef EM_ARCV2 57 #define EM_ARCV2 195 58 #endif 59 60 #ifndef EM_RISCV 61 #define EM_RISCV 243 62 #endif 63 64 #ifndef EM_LOONGARCH 65 #define EM_LOONGARCH 258 66 #endif 67 68 typedef union { 69 Elf32_Ehdr e32; 70 Elf64_Ehdr e64; 71 } Elf_Ehdr; 72 73 typedef union { 74 Elf32_Shdr e32; 75 Elf64_Shdr e64; 76 } Elf_Shdr; 77 78 typedef union { 79 Elf32_Sym e32; 80 Elf64_Sym e64; 81 } Elf_Sym; 82 83 typedef union { 84 Elf32_Rela e32; 85 Elf64_Rela e64; 86 } Elf_Rela; 87 88 static uint32_t (*r)(const uint32_t *); 89 static uint16_t (*r2)(const uint16_t *); 90 static uint64_t (*r8)(const uint64_t *); 91 static void (*w)(uint32_t, uint32_t *); 92 static void (*w8)(uint64_t, uint64_t *); 93 typedef void (*table_sort_t)(char *, int); 94 95 static struct elf_funcs { 96 int (*compare_extable)(const void *a, const void *b); 97 uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr); 98 uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr); 99 uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr); 100 uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr); 101 uint64_t (*shdr_addr)(Elf_Shdr *shdr); 102 uint64_t (*shdr_offset)(Elf_Shdr *shdr); 103 uint64_t (*shdr_size)(Elf_Shdr *shdr); 104 uint64_t (*shdr_entsize)(Elf_Shdr *shdr); 105 uint32_t (*shdr_link)(Elf_Shdr *shdr); 106 uint32_t (*shdr_name)(Elf_Shdr *shdr); 107 uint32_t (*shdr_type)(Elf_Shdr *shdr); 108 uint8_t (*sym_type)(Elf_Sym *sym); 109 uint32_t (*sym_name)(Elf_Sym *sym); 110 uint64_t (*sym_value)(Elf_Sym *sym); 111 uint16_t (*sym_shndx)(Elf_Sym *sym); 112 uint64_t (*rela_offset)(Elf_Rela *rela); 113 uint64_t (*rela_info)(Elf_Rela *rela); 114 uint64_t (*rela_addend)(Elf_Rela *rela); 115 void (*rela_write_addend)(Elf_Rela *rela, uint64_t val); 116 } e; 117 118 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) 119 { 120 return r8(&ehdr->e64.e_shoff); 121 } 122 123 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr) 124 { 125 return r(&ehdr->e32.e_shoff); 126 } 127 128 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr) 129 { 130 return e.ehdr_shoff(ehdr); 131 } 132 133 #define EHDR_HALF(fn_name) \ 134 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \ 135 { \ 136 return r2(&ehdr->e64.e_##fn_name); \ 137 } \ 138 \ 139 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \ 140 { \ 141 return r2(&ehdr->e32.e_##fn_name); \ 142 } \ 143 \ 144 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr) \ 145 { \ 146 return e.ehdr_##fn_name(ehdr); \ 147 } 148 149 EHDR_HALF(shentsize) 150 EHDR_HALF(shstrndx) 151 EHDR_HALF(shnum) 152 153 #define SHDR_WORD(fn_name) \ 154 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 155 { \ 156 return r(&shdr->e64.sh_##fn_name); \ 157 } \ 158 \ 159 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 160 { \ 161 return r(&shdr->e32.sh_##fn_name); \ 162 } \ 163 \ 164 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 165 { \ 166 return e.shdr_##fn_name(shdr); \ 167 } 168 169 #define SHDR_ADDR(fn_name) \ 170 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \ 171 { \ 172 return r8(&shdr->e64.sh_##fn_name); \ 173 } \ 174 \ 175 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \ 176 { \ 177 return r(&shdr->e32.sh_##fn_name); \ 178 } \ 179 \ 180 static uint64_t shdr_##fn_name(Elf_Shdr *shdr) \ 181 { \ 182 return e.shdr_##fn_name(shdr); \ 183 } 184 185 #define SHDR_WORD(fn_name) \ 186 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 187 { \ 188 return r(&shdr->e64.sh_##fn_name); \ 189 } \ 190 \ 191 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 192 { \ 193 return r(&shdr->e32.sh_##fn_name); \ 194 } \ 195 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 196 { \ 197 return e.shdr_##fn_name(shdr); \ 198 } 199 200 SHDR_ADDR(addr) 201 SHDR_ADDR(offset) 202 SHDR_ADDR(size) 203 SHDR_ADDR(entsize) 204 205 SHDR_WORD(link) 206 SHDR_WORD(name) 207 SHDR_WORD(type) 208 209 #define SYM_ADDR(fn_name) \ 210 static uint64_t sym64_##fn_name(Elf_Sym *sym) \ 211 { \ 212 return r8(&sym->e64.st_##fn_name); \ 213 } \ 214 \ 215 static uint64_t sym32_##fn_name(Elf_Sym *sym) \ 216 { \ 217 return r(&sym->e32.st_##fn_name); \ 218 } \ 219 \ 220 static uint64_t sym_##fn_name(Elf_Sym *sym) \ 221 { \ 222 return e.sym_##fn_name(sym); \ 223 } 224 225 #define SYM_WORD(fn_name) \ 226 static uint32_t sym64_##fn_name(Elf_Sym *sym) \ 227 { \ 228 return r(&sym->e64.st_##fn_name); \ 229 } \ 230 \ 231 static uint32_t sym32_##fn_name(Elf_Sym *sym) \ 232 { \ 233 return r(&sym->e32.st_##fn_name); \ 234 } \ 235 \ 236 static uint32_t sym_##fn_name(Elf_Sym *sym) \ 237 { \ 238 return e.sym_##fn_name(sym); \ 239 } 240 241 #define SYM_HALF(fn_name) \ 242 static uint16_t sym64_##fn_name(Elf_Sym *sym) \ 243 { \ 244 return r2(&sym->e64.st_##fn_name); \ 245 } \ 246 \ 247 static uint16_t sym32_##fn_name(Elf_Sym *sym) \ 248 { \ 249 return r2(&sym->e32.st_##fn_name); \ 250 } \ 251 \ 252 static uint16_t sym_##fn_name(Elf_Sym *sym) \ 253 { \ 254 return e.sym_##fn_name(sym); \ 255 } 256 257 static uint8_t sym64_type(Elf_Sym *sym) 258 { 259 return ELF64_ST_TYPE(sym->e64.st_info); 260 } 261 262 static uint8_t sym32_type(Elf_Sym *sym) 263 { 264 return ELF32_ST_TYPE(sym->e32.st_info); 265 } 266 267 static uint8_t sym_type(Elf_Sym *sym) 268 { 269 return e.sym_type(sym); 270 } 271 272 SYM_ADDR(value) 273 SYM_WORD(name) 274 SYM_HALF(shndx) 275 276 #define __maybe_unused __attribute__((__unused__)) 277 278 #define RELA_ADDR(fn_name) \ 279 static uint64_t rela64_##fn_name(Elf_Rela *rela) \ 280 { \ 281 return r8((uint64_t *)&rela->e64.r_##fn_name); \ 282 } \ 283 \ 284 static uint64_t rela32_##fn_name(Elf_Rela *rela) \ 285 { \ 286 return r((uint32_t *)&rela->e32.r_##fn_name); \ 287 } \ 288 \ 289 static uint64_t __maybe_unused rela_##fn_name(Elf_Rela *rela) \ 290 { \ 291 return e.rela_##fn_name(rela); \ 292 } 293 294 RELA_ADDR(offset) 295 RELA_ADDR(info) 296 RELA_ADDR(addend) 297 298 static void rela64_write_addend(Elf_Rela *rela, uint64_t val) 299 { 300 w8(val, (uint64_t *)&rela->e64.r_addend); 301 } 302 303 static void rela32_write_addend(Elf_Rela *rela, uint64_t val) 304 { 305 w(val, (uint32_t *)&rela->e32.r_addend); 306 } 307 308 /* 309 * Get the whole file as a programming convenience in order to avoid 310 * malloc+lseek+read+free of many pieces. If successful, then mmap 311 * avoids copying unused pieces; else just read the whole file. 312 * Open for both read and write. 313 */ 314 static void *mmap_file(char const *fname, size_t *size) 315 { 316 int fd; 317 struct stat sb; 318 void *addr = NULL; 319 320 fd = open(fname, O_RDWR); 321 if (fd < 0) { 322 perror(fname); 323 return NULL; 324 } 325 if (fstat(fd, &sb) < 0) { 326 perror(fname); 327 goto out; 328 } 329 if (!S_ISREG(sb.st_mode)) { 330 fprintf(stderr, "not a regular file: %s\n", fname); 331 goto out; 332 } 333 334 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 335 if (addr == MAP_FAILED) { 336 fprintf(stderr, "Could not mmap file: %s\n", fname); 337 goto out; 338 } 339 340 *size = sb.st_size; 341 342 out: 343 close(fd); 344 return addr; 345 } 346 347 static uint32_t rbe(const uint32_t *x) 348 { 349 return get_unaligned_be32(x); 350 } 351 352 static uint16_t r2be(const uint16_t *x) 353 { 354 return get_unaligned_be16(x); 355 } 356 357 static uint64_t r8be(const uint64_t *x) 358 { 359 return get_unaligned_be64(x); 360 } 361 362 static uint32_t rle(const uint32_t *x) 363 { 364 return get_unaligned_le32(x); 365 } 366 367 static uint16_t r2le(const uint16_t *x) 368 { 369 return get_unaligned_le16(x); 370 } 371 372 static uint64_t r8le(const uint64_t *x) 373 { 374 return get_unaligned_le64(x); 375 } 376 377 static void wbe(uint32_t val, uint32_t *x) 378 { 379 put_unaligned_be32(val, x); 380 } 381 382 static void wle(uint32_t val, uint32_t *x) 383 { 384 put_unaligned_le32(val, x); 385 } 386 387 static void w8be(uint64_t val, uint64_t *x) 388 { 389 put_unaligned_be64(val, x); 390 } 391 392 static void w8le(uint64_t val, uint64_t *x) 393 { 394 put_unaligned_le64(val, x); 395 } 396 397 /* 398 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 399 * the way to -256..-1, to avoid conflicting with real section 400 * indices. 401 */ 402 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 403 404 static inline int is_shndx_special(unsigned int i) 405 { 406 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 407 } 408 409 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 410 static inline unsigned int get_secindex(unsigned int shndx, 411 unsigned int sym_offs, 412 const Elf32_Word *symtab_shndx_start) 413 { 414 if (is_shndx_special(shndx)) 415 return SPECIAL(shndx); 416 if (shndx != SHN_XINDEX) 417 return shndx; 418 return r(&symtab_shndx_start[sym_offs]); 419 } 420 421 static int compare_extable_32(const void *a, const void *b) 422 { 423 Elf32_Addr av = r(a); 424 Elf32_Addr bv = r(b); 425 426 if (av < bv) 427 return -1; 428 return av > bv; 429 } 430 431 static int compare_extable_64(const void *a, const void *b) 432 { 433 Elf64_Addr av = r8(a); 434 Elf64_Addr bv = r8(b); 435 436 if (av < bv) 437 return -1; 438 return av > bv; 439 } 440 441 static int compare_extable(const void *a, const void *b) 442 { 443 return e.compare_extable(a, b); 444 } 445 446 static inline void *get_index(void *start, int entsize, int index) 447 { 448 return start + (entsize * index); 449 } 450 451 static int extable_ent_size; 452 static int long_size; 453 454 #define ERRSTR_MAXSZ 256 455 456 #ifdef UNWINDER_ORC_ENABLED 457 /* ORC unwinder only support X86_64 */ 458 #include <asm/orc_types.h> 459 460 static char g_err[ERRSTR_MAXSZ]; 461 static int *g_orc_ip_table; 462 static struct orc_entry *g_orc_table; 463 464 static pthread_t orc_sort_thread; 465 466 static inline unsigned long orc_ip(const int *ip) 467 { 468 return (unsigned long)ip + *ip; 469 } 470 471 static int orc_sort_cmp(const void *_a, const void *_b) 472 { 473 struct orc_entry *orc_a, *orc_b; 474 const int *a = g_orc_ip_table + *(int *)_a; 475 const int *b = g_orc_ip_table + *(int *)_b; 476 unsigned long a_val = orc_ip(a); 477 unsigned long b_val = orc_ip(b); 478 479 if (a_val > b_val) 480 return 1; 481 if (a_val < b_val) 482 return -1; 483 484 /* 485 * The "weak" section terminator entries need to always be on the left 486 * to ensure the lookup code skips them in favor of real entries. 487 * These terminator entries exist to handle any gaps created by 488 * whitelisted .o files which didn't get objtool generation. 489 */ 490 orc_a = g_orc_table + (a - g_orc_ip_table); 491 orc_b = g_orc_table + (b - g_orc_ip_table); 492 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 493 return 0; 494 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 495 } 496 497 static void *sort_orctable(void *arg) 498 { 499 int i; 500 int *idxs = NULL; 501 int *tmp_orc_ip_table = NULL; 502 struct orc_entry *tmp_orc_table = NULL; 503 unsigned int *orc_ip_size = (unsigned int *)arg; 504 unsigned int num_entries = *orc_ip_size / sizeof(int); 505 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 506 507 idxs = (int *)malloc(*orc_ip_size); 508 if (!idxs) { 509 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 510 strerror(errno)); 511 pthread_exit(g_err); 512 } 513 514 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 515 if (!tmp_orc_ip_table) { 516 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 517 strerror(errno)); 518 pthread_exit(g_err); 519 } 520 521 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 522 if (!tmp_orc_table) { 523 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 524 strerror(errno)); 525 pthread_exit(g_err); 526 } 527 528 /* initialize indices array, convert ip_table to absolute address */ 529 for (i = 0; i < num_entries; i++) { 530 idxs[i] = i; 531 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 532 } 533 memcpy(tmp_orc_table, g_orc_table, orc_size); 534 535 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 536 537 for (i = 0; i < num_entries; i++) { 538 if (idxs[i] == i) 539 continue; 540 541 /* convert back to relative address */ 542 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 543 g_orc_table[i] = tmp_orc_table[idxs[i]]; 544 } 545 546 free(idxs); 547 free(tmp_orc_ip_table); 548 free(tmp_orc_table); 549 pthread_exit(NULL); 550 } 551 #endif 552 553 #ifdef MCOUNT_SORT_ENABLED 554 555 /* Only used for sorting mcount table */ 556 static void rela_write_addend(Elf_Rela *rela, uint64_t val) 557 { 558 e.rela_write_addend(rela, val); 559 } 560 561 static pthread_t mcount_sort_thread; 562 static bool sort_reloc; 563 564 static long rela_type; 565 566 static char m_err[ERRSTR_MAXSZ]; 567 568 struct elf_mcount_loc { 569 Elf_Ehdr *ehdr; 570 Elf_Shdr *init_data_sec; 571 uint64_t start_mcount_loc; 572 uint64_t stop_mcount_loc; 573 }; 574 575 /* Sort the relocations not the address itself */ 576 static void *sort_relocs(Elf_Ehdr *ehdr, uint64_t start_loc, uint64_t size) 577 { 578 Elf_Shdr *shdr_start; 579 Elf_Rela *rel; 580 unsigned int shnum; 581 unsigned int count; 582 int shentsize; 583 void *vals; 584 void *ptr; 585 586 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 587 shentsize = ehdr_shentsize(ehdr); 588 589 vals = malloc(long_size * size); 590 if (!vals) { 591 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 592 pthread_exit(m_err); 593 return NULL; 594 } 595 596 ptr = vals; 597 598 shnum = ehdr_shnum(ehdr); 599 if (shnum == SHN_UNDEF) 600 shnum = shdr_size(shdr_start); 601 602 for (int i = 0; i < shnum; i++) { 603 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 604 void *end; 605 606 if (shdr_type(shdr) != SHT_RELA) 607 continue; 608 609 rel = (void *)ehdr + shdr_offset(shdr); 610 end = (void *)rel + shdr_size(shdr); 611 612 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 613 uint64_t offset = rela_offset(rel); 614 615 if (offset >= start_loc && offset < start_loc + size) { 616 if (ptr + long_size > vals + size) { 617 free(vals); 618 snprintf(m_err, ERRSTR_MAXSZ, 619 "Too many relocations"); 620 pthread_exit(m_err); 621 return NULL; 622 } 623 624 /* Make sure this has the correct type */ 625 if (rela_info(rel) != rela_type) { 626 free(vals); 627 snprintf(m_err, ERRSTR_MAXSZ, 628 "rela has type %lx but expected %lx\n", 629 (long)rela_info(rel), rela_type); 630 pthread_exit(m_err); 631 return NULL; 632 } 633 634 if (long_size == 4) 635 *(uint32_t *)ptr = rela_addend(rel); 636 else 637 *(uint64_t *)ptr = rela_addend(rel); 638 ptr += long_size; 639 } 640 } 641 } 642 count = ptr - vals; 643 qsort(vals, count / long_size, long_size, compare_extable); 644 645 ptr = vals; 646 for (int i = 0; i < shnum; i++) { 647 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 648 void *end; 649 650 if (shdr_type(shdr) != SHT_RELA) 651 continue; 652 653 rel = (void *)ehdr + shdr_offset(shdr); 654 end = (void *)rel + shdr_size(shdr); 655 656 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 657 uint64_t offset = rela_offset(rel); 658 659 if (offset >= start_loc && offset < start_loc + size) { 660 if (long_size == 4) 661 rela_write_addend(rel, *(uint32_t *)ptr); 662 else 663 rela_write_addend(rel, *(uint64_t *)ptr); 664 ptr += long_size; 665 } 666 } 667 } 668 free(vals); 669 return NULL; 670 } 671 672 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 673 static void *sort_mcount_loc(void *arg) 674 { 675 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 676 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 677 + shdr_offset(emloc->init_data_sec); 678 uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 679 unsigned char *start_loc = (void *)emloc->ehdr + offset; 680 681 if (sort_reloc) 682 return sort_relocs(emloc->ehdr, emloc->start_mcount_loc, count); 683 684 qsort(start_loc, count/long_size, long_size, compare_extable); 685 return NULL; 686 } 687 688 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 689 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 690 const char *strtab) 691 { 692 Elf_Sym *sym, *end_sym; 693 int symentsize = shdr_entsize(symtab_sec); 694 int found = 0; 695 696 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 697 end_sym = (void *)sym + shdr_size(symtab_sec); 698 699 while (sym < end_sym) { 700 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 701 emloc->start_mcount_loc = sym_value(sym); 702 if (++found == 2) 703 break; 704 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 705 emloc->stop_mcount_loc = sym_value(sym); 706 if (++found == 2) 707 break; 708 } 709 sym = (void *)sym + symentsize; 710 } 711 712 if (!emloc->start_mcount_loc) { 713 fprintf(stderr, "get start_mcount_loc error!"); 714 return; 715 } 716 717 if (!emloc->stop_mcount_loc) { 718 fprintf(stderr, "get stop_mcount_loc error!"); 719 return; 720 } 721 } 722 #endif 723 724 static int do_sort(Elf_Ehdr *ehdr, 725 char const *const fname, 726 table_sort_t custom_sort) 727 { 728 int rc = -1; 729 Elf_Shdr *shdr_start; 730 Elf_Shdr *strtab_sec = NULL; 731 Elf_Shdr *symtab_sec = NULL; 732 Elf_Shdr *extab_sec = NULL; 733 Elf_Shdr *string_sec; 734 Elf_Sym *sym; 735 const Elf_Sym *symtab; 736 Elf32_Word *symtab_shndx = NULL; 737 Elf_Sym *sort_needed_sym = NULL; 738 Elf_Shdr *sort_needed_sec; 739 uint32_t *sort_needed_loc; 740 void *sym_start; 741 void *sym_end; 742 const char *secstrings; 743 const char *strtab; 744 char *extab_image; 745 int sort_need_index; 746 int symentsize; 747 int shentsize; 748 int idx; 749 int i; 750 unsigned int shnum; 751 unsigned int shstrndx; 752 #ifdef MCOUNT_SORT_ENABLED 753 struct elf_mcount_loc mstruct = {0}; 754 #endif 755 #ifdef UNWINDER_ORC_ENABLED 756 unsigned int orc_ip_size = 0; 757 unsigned int orc_size = 0; 758 unsigned int orc_num_entries = 0; 759 #endif 760 761 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 762 shentsize = ehdr_shentsize(ehdr); 763 764 shstrndx = ehdr_shstrndx(ehdr); 765 if (shstrndx == SHN_XINDEX) 766 shstrndx = shdr_link(shdr_start); 767 string_sec = get_index(shdr_start, shentsize, shstrndx); 768 secstrings = (const char *)ehdr + shdr_offset(string_sec); 769 770 shnum = ehdr_shnum(ehdr); 771 if (shnum == SHN_UNDEF) 772 shnum = shdr_size(shdr_start); 773 774 for (i = 0; i < shnum; i++) { 775 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 776 777 idx = shdr_name(shdr); 778 if (!strcmp(secstrings + idx, "__ex_table")) 779 extab_sec = shdr; 780 if (!strcmp(secstrings + idx, ".symtab")) 781 symtab_sec = shdr; 782 if (!strcmp(secstrings + idx, ".strtab")) 783 strtab_sec = shdr; 784 785 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 786 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 787 shdr_offset(shdr)); 788 789 #ifdef MCOUNT_SORT_ENABLED 790 /* locate the .init.data section in vmlinux */ 791 if (!strcmp(secstrings + idx, ".init.data")) 792 mstruct.init_data_sec = shdr; 793 #endif 794 795 #ifdef UNWINDER_ORC_ENABLED 796 /* locate the ORC unwind tables */ 797 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 798 orc_ip_size = shdr_size(shdr); 799 g_orc_ip_table = (int *)((void *)ehdr + 800 shdr_offset(shdr)); 801 } 802 if (!strcmp(secstrings + idx, ".orc_unwind")) { 803 orc_size = shdr_size(shdr); 804 g_orc_table = (struct orc_entry *)((void *)ehdr + 805 shdr_offset(shdr)); 806 } 807 #endif 808 } /* for loop */ 809 810 #ifdef UNWINDER_ORC_ENABLED 811 if (!g_orc_ip_table || !g_orc_table) { 812 fprintf(stderr, 813 "incomplete ORC unwind tables in file: %s\n", fname); 814 goto out; 815 } 816 817 orc_num_entries = orc_ip_size / sizeof(int); 818 if (orc_ip_size % sizeof(int) != 0 || 819 orc_size % sizeof(struct orc_entry) != 0 || 820 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 821 fprintf(stderr, 822 "inconsistent ORC unwind table entries in file: %s\n", 823 fname); 824 goto out; 825 } 826 827 /* create thread to sort ORC unwind tables concurrently */ 828 if (pthread_create(&orc_sort_thread, NULL, 829 sort_orctable, &orc_ip_size)) { 830 fprintf(stderr, 831 "pthread_create orc_sort_thread failed '%s': %s\n", 832 strerror(errno), fname); 833 goto out; 834 } 835 #endif 836 if (!extab_sec) { 837 fprintf(stderr, "no __ex_table in file: %s\n", fname); 838 goto out; 839 } 840 841 if (!symtab_sec) { 842 fprintf(stderr, "no .symtab in file: %s\n", fname); 843 goto out; 844 } 845 846 if (!strtab_sec) { 847 fprintf(stderr, "no .strtab in file: %s\n", fname); 848 goto out; 849 } 850 851 extab_image = (void *)ehdr + shdr_offset(extab_sec); 852 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 853 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 854 855 #ifdef MCOUNT_SORT_ENABLED 856 mstruct.ehdr = ehdr; 857 get_mcount_loc(&mstruct, symtab_sec, strtab); 858 859 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 860 fprintf(stderr, 861 "incomplete mcount's sort in file: %s\n", 862 fname); 863 goto out; 864 } 865 866 /* create thread to sort mcount_loc concurrently */ 867 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 868 fprintf(stderr, 869 "pthread_create mcount_sort_thread failed '%s': %s\n", 870 strerror(errno), fname); 871 goto out; 872 } 873 #endif 874 875 if (custom_sort) { 876 custom_sort(extab_image, shdr_size(extab_sec)); 877 } else { 878 int num_entries = shdr_size(extab_sec) / extable_ent_size; 879 qsort(extab_image, num_entries, 880 extable_ent_size, compare_extable); 881 } 882 883 /* find the flag main_extable_sort_needed */ 884 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 885 sym_end = sym_start + shdr_size(symtab_sec); 886 symentsize = shdr_entsize(symtab_sec); 887 888 for (sym = sym_start; (void *)sym + symentsize < sym_end; 889 sym = (void *)sym + symentsize) { 890 if (sym_type(sym) != STT_OBJECT) 891 continue; 892 if (!strcmp(strtab + sym_name(sym), 893 "main_extable_sort_needed")) { 894 sort_needed_sym = sym; 895 break; 896 } 897 } 898 899 if (!sort_needed_sym) { 900 fprintf(stderr, 901 "no main_extable_sort_needed symbol in file: %s\n", 902 fname); 903 goto out; 904 } 905 906 sort_need_index = get_secindex(sym_shndx(sym), 907 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 908 symtab_shndx); 909 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 910 sort_needed_loc = (void *)ehdr + 911 shdr_offset(sort_needed_sec) + 912 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 913 914 /* extable has been sorted, clear the flag */ 915 w(0, sort_needed_loc); 916 rc = 0; 917 918 out: 919 #ifdef UNWINDER_ORC_ENABLED 920 if (orc_sort_thread) { 921 void *retval = NULL; 922 /* wait for ORC tables sort done */ 923 rc = pthread_join(orc_sort_thread, &retval); 924 if (rc) { 925 fprintf(stderr, 926 "pthread_join failed '%s': %s\n", 927 strerror(errno), fname); 928 } else if (retval) { 929 rc = -1; 930 fprintf(stderr, 931 "failed to sort ORC tables '%s': %s\n", 932 (char *)retval, fname); 933 } 934 } 935 #endif 936 937 #ifdef MCOUNT_SORT_ENABLED 938 if (mcount_sort_thread) { 939 void *retval = NULL; 940 /* wait for mcount sort done */ 941 rc = pthread_join(mcount_sort_thread, &retval); 942 if (rc) { 943 fprintf(stderr, 944 "pthread_join failed '%s': %s\n", 945 strerror(errno), fname); 946 } else if (retval) { 947 rc = -1; 948 fprintf(stderr, 949 "failed to sort mcount '%s': %s\n", 950 (char *)retval, fname); 951 } 952 } 953 #endif 954 return rc; 955 } 956 957 static int compare_relative_table(const void *a, const void *b) 958 { 959 int32_t av = (int32_t)r(a); 960 int32_t bv = (int32_t)r(b); 961 962 if (av < bv) 963 return -1; 964 if (av > bv) 965 return 1; 966 return 0; 967 } 968 969 static void sort_relative_table(char *extab_image, int image_size) 970 { 971 int i = 0; 972 973 /* 974 * Do the same thing the runtime sort does, first normalize to 975 * being relative to the start of the section. 976 */ 977 while (i < image_size) { 978 uint32_t *loc = (uint32_t *)(extab_image + i); 979 w(r(loc) + i, loc); 980 i += 4; 981 } 982 983 qsort(extab_image, image_size / 8, 8, compare_relative_table); 984 985 /* Now denormalize. */ 986 i = 0; 987 while (i < image_size) { 988 uint32_t *loc = (uint32_t *)(extab_image + i); 989 w(r(loc) - i, loc); 990 i += 4; 991 } 992 } 993 994 static void sort_relative_table_with_data(char *extab_image, int image_size) 995 { 996 int i = 0; 997 998 while (i < image_size) { 999 uint32_t *loc = (uint32_t *)(extab_image + i); 1000 1001 w(r(loc) + i, loc); 1002 w(r(loc + 1) + i + 4, loc + 1); 1003 /* Don't touch the fixup type or data */ 1004 1005 i += sizeof(uint32_t) * 3; 1006 } 1007 1008 qsort(extab_image, image_size / 12, 12, compare_relative_table); 1009 1010 i = 0; 1011 while (i < image_size) { 1012 uint32_t *loc = (uint32_t *)(extab_image + i); 1013 1014 w(r(loc) - i, loc); 1015 w(r(loc + 1) - (i + 4), loc + 1); 1016 /* Don't touch the fixup type or data */ 1017 1018 i += sizeof(uint32_t) * 3; 1019 } 1020 } 1021 1022 static int do_file(char const *const fname, void *addr) 1023 { 1024 Elf_Ehdr *ehdr = addr; 1025 table_sort_t custom_sort = NULL; 1026 1027 switch (ehdr->e32.e_ident[EI_DATA]) { 1028 case ELFDATA2LSB: 1029 r = rle; 1030 r2 = r2le; 1031 r8 = r8le; 1032 w = wle; 1033 w8 = w8le; 1034 break; 1035 case ELFDATA2MSB: 1036 r = rbe; 1037 r2 = r2be; 1038 r8 = r8be; 1039 w = wbe; 1040 w8 = w8be; 1041 break; 1042 default: 1043 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 1044 ehdr->e32.e_ident[EI_DATA], fname); 1045 return -1; 1046 } 1047 1048 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 || 1049 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) || 1050 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) { 1051 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); 1052 return -1; 1053 } 1054 1055 switch (r2(&ehdr->e32.e_machine)) { 1056 case EM_AARCH64: 1057 #ifdef MCOUNT_SORT_ENABLED 1058 sort_reloc = true; 1059 rela_type = 0x403; 1060 #endif 1061 /* fallthrough */ 1062 case EM_386: 1063 case EM_LOONGARCH: 1064 case EM_RISCV: 1065 case EM_S390: 1066 case EM_X86_64: 1067 custom_sort = sort_relative_table_with_data; 1068 break; 1069 case EM_PARISC: 1070 case EM_PPC: 1071 case EM_PPC64: 1072 custom_sort = sort_relative_table; 1073 break; 1074 case EM_ARCOMPACT: 1075 case EM_ARCV2: 1076 case EM_ARM: 1077 case EM_MICROBLAZE: 1078 case EM_MIPS: 1079 case EM_XTENSA: 1080 break; 1081 default: 1082 fprintf(stderr, "unrecognized e_machine %d %s\n", 1083 r2(&ehdr->e32.e_machine), fname); 1084 return -1; 1085 } 1086 1087 switch (ehdr->e32.e_ident[EI_CLASS]) { 1088 case ELFCLASS32: { 1089 struct elf_funcs efuncs = { 1090 .compare_extable = compare_extable_32, 1091 .ehdr_shoff = ehdr32_shoff, 1092 .ehdr_shentsize = ehdr32_shentsize, 1093 .ehdr_shstrndx = ehdr32_shstrndx, 1094 .ehdr_shnum = ehdr32_shnum, 1095 .shdr_addr = shdr32_addr, 1096 .shdr_offset = shdr32_offset, 1097 .shdr_link = shdr32_link, 1098 .shdr_size = shdr32_size, 1099 .shdr_name = shdr32_name, 1100 .shdr_type = shdr32_type, 1101 .shdr_entsize = shdr32_entsize, 1102 .sym_type = sym32_type, 1103 .sym_name = sym32_name, 1104 .sym_value = sym32_value, 1105 .sym_shndx = sym32_shndx, 1106 .rela_offset = rela32_offset, 1107 .rela_info = rela32_info, 1108 .rela_addend = rela32_addend, 1109 .rela_write_addend = rela32_write_addend, 1110 }; 1111 1112 e = efuncs; 1113 long_size = 4; 1114 extable_ent_size = 8; 1115 1116 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 1117 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 1118 fprintf(stderr, 1119 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 1120 return -1; 1121 } 1122 1123 } 1124 break; 1125 case ELFCLASS64: { 1126 struct elf_funcs efuncs = { 1127 .compare_extable = compare_extable_64, 1128 .ehdr_shoff = ehdr64_shoff, 1129 .ehdr_shentsize = ehdr64_shentsize, 1130 .ehdr_shstrndx = ehdr64_shstrndx, 1131 .ehdr_shnum = ehdr64_shnum, 1132 .shdr_addr = shdr64_addr, 1133 .shdr_offset = shdr64_offset, 1134 .shdr_link = shdr64_link, 1135 .shdr_size = shdr64_size, 1136 .shdr_name = shdr64_name, 1137 .shdr_type = shdr64_type, 1138 .shdr_entsize = shdr64_entsize, 1139 .sym_type = sym64_type, 1140 .sym_name = sym64_name, 1141 .sym_value = sym64_value, 1142 .sym_shndx = sym64_shndx, 1143 .rela_offset = rela64_offset, 1144 .rela_info = rela64_info, 1145 .rela_addend = rela64_addend, 1146 .rela_write_addend = rela64_write_addend, 1147 }; 1148 1149 e = efuncs; 1150 long_size = 8; 1151 extable_ent_size = 16; 1152 1153 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 1154 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 1155 fprintf(stderr, 1156 "unrecognized ET_EXEC/ET_DYN file: %s\n", 1157 fname); 1158 return -1; 1159 } 1160 1161 } 1162 break; 1163 default: 1164 fprintf(stderr, "unrecognized ELF class %d %s\n", 1165 ehdr->e32.e_ident[EI_CLASS], fname); 1166 return -1; 1167 } 1168 1169 return do_sort(ehdr, fname, custom_sort); 1170 } 1171 1172 int main(int argc, char *argv[]) 1173 { 1174 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 1175 size_t size = 0; 1176 void *addr = NULL; 1177 1178 if (argc < 2) { 1179 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1180 return 0; 1181 } 1182 1183 /* Process each file in turn, allowing deep failure. */ 1184 for (i = 1; i < argc; i++) { 1185 addr = mmap_file(argv[i], &size); 1186 if (!addr) { 1187 ++n_error; 1188 continue; 1189 } 1190 1191 if (do_file(argv[i], addr)) 1192 ++n_error; 1193 1194 munmap(addr, size); 1195 } 1196 1197 return !!n_error; 1198 } 1199