1#!/usr/bin/env python3
2#===- lib/hwasan/scripts/hwasan_symbolize ----------------------------------===#
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https:#llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7#
8#===------------------------------------------------------------------------===#
9#
10# HWAddressSanitizer offline symbolization script.
11#
12#===------------------------------------------------------------------------===#
13
14from __future__ import print_function
15from __future__ import unicode_literals
16
17import argparse
18import glob
19import html
20import json
21import mmap
22import os
23import re
24import struct
25import subprocess
26import sys
27
28if sys.version_info.major < 3:
29  # Simulate Python 3.x behaviour of defaulting to UTF-8 for print. This is
30  # important in case any symbols are non-ASCII.
31  import codecs
32  sys.stdout = codecs.getwriter("utf-8")(sys.stdout)
33
34last_access_address = None
35last_access_tag = None
36
37# Below, a parser for a subset of ELF. It only supports 64 bit, little-endian,
38# and only parses what is necessary to find the build ids. It uses a memoryview
39# into an mmap to avoid copying.
40Ehdr_size = 64
41e_shnum_offset = 60
42e_shoff_offset = 40
43
44Shdr_size = 64
45sh_type_offset = 4
46sh_offset_offset = 24
47sh_size_offset = 32
48SHT_NOTE = 7
49
50Nhdr_size = 12
51NT_GNU_BUILD_ID = 3
52
53def align_up(size, alignment):
54  return (size + alignment - 1) & ~(alignment - 1)
55
56def handle_Nhdr(mv, sh_size):
57  offset = 0
58  while offset < sh_size:
59    n_namesz, n_descsz, n_type = struct.unpack_from('<III', buffer=mv,
60                                                    offset=offset)
61    if (n_type == NT_GNU_BUILD_ID and n_namesz == 4 and
62        mv[offset + Nhdr_size: offset + Nhdr_size + 4] == b"GNU\x00"):
63      value = mv[offset + Nhdr_size + 4: offset + Nhdr_size + 4 + n_descsz]
64      return value.hex()
65    offset += Nhdr_size + align_up(n_namesz, 4) + align_up(n_descsz, 4)
66  return None
67
68def handle_Shdr(mv):
69  sh_type, = struct.unpack_from('<I', buffer=mv, offset=sh_type_offset)
70  if sh_type != SHT_NOTE:
71    return None, None
72  sh_offset, = struct.unpack_from('<Q', buffer=mv, offset=sh_offset_offset)
73  sh_size, = struct.unpack_from('<Q', buffer=mv, offset=sh_size_offset)
74  return sh_offset, sh_size
75
76def handle_elf(mv):
77  # \x02 is ELFCLASS64, \x01 is ELFDATA2LSB. HWASan currently only works on
78  # 64-bit little endian platforms (x86_64 and ARM64). If this changes, we will
79  # have to extend the parsing code.
80  if mv[:6] != b'\x7fELF\x02\x01':
81    return None
82  e_shnum, = struct.unpack_from('<H', buffer=mv, offset=e_shnum_offset)
83  e_shoff, = struct.unpack_from('<Q', buffer=mv, offset=e_shoff_offset)
84  for i in range(0, e_shnum):
85    start = e_shoff + i * Shdr_size
86    sh_offset, sh_size = handle_Shdr(mv[start: start + Shdr_size])
87    if sh_offset is None:
88      continue
89    note_hdr = mv[sh_offset: sh_offset + sh_size]
90    result = handle_Nhdr(note_hdr, sh_size)
91    if result is not None:
92      return result
93
94def get_buildid(filename):
95  with open(filename, "r") as fd:
96    if os.fstat(fd.fileno()).st_size < Ehdr_size:
97      return None
98    with mmap.mmap(fd.fileno(), 0, access=mmap.ACCESS_READ) as m:
99      with memoryview(m) as mv:
100        return handle_elf(mv)
101
102class Symbolizer:
103  def __init__(self, path, binary_prefixes, paths_to_cut):
104    self.__pipe = None
105    self.__path = path
106    self.__binary_prefixes = binary_prefixes
107    self.__paths_to_cut = paths_to_cut
108    self.__log = False
109    self.__warnings = set()
110    self.__index = {}
111    self.__link_prefixes = []
112    self.__html = False
113
114  def enable_html(self, enable):
115    self.__html = enable
116
117  def enable_logging(self, enable):
118    self.__log = enable
119
120  def maybe_escape(self, text):
121    if self.__html:
122      # We need to manually use &nbsp; for leading spaces, html.escape does
123      # not do that, and HTML ignores them.
124      spaces = 0
125      for i, c in enumerate(text):
126        spaces = i
127        if c != ' ':
128          break
129      text = text[spaces:]
130      return spaces * '&nbsp;' + html.escape(text)
131    return text
132
133  def print(self, line, escape=True):
134    if escape:
135      line = self.maybe_escape(line)
136    if self.__html:
137      line += '<br/>'
138    print(line)
139
140  def read_linkify(self, filename):
141    with open(filename, 'r') as fd:
142      data = json.load(fd)
143    self.__link_prefixes = [(e["prefix"], e["link"]) for e in data]
144
145  def __open_pipe(self):
146    if not self.__pipe:
147      opt = {}
148      if sys.version_info.major > 2:
149        opt['encoding'] = 'utf-8'
150      self.__pipe = subprocess.Popen([self.__path, "--inlining", "--functions"],
151                                     stdin=subprocess.PIPE, stdout=subprocess.PIPE,
152                                     **opt)
153
154  class __EOF(Exception):
155    pass
156
157  def __write(self, s):
158    print(s, file=self.__pipe.stdin)
159    self.__pipe.stdin.flush()
160    if self.__log:
161      print("#>>  |%s|" % (s,), file=sys.stderr)
162
163  def __read(self):
164    s = self.__pipe.stdout.readline().rstrip()
165    if self.__log:
166      print("# << |%s|" % (s,), file=sys.stderr)
167    if s == '':
168      raise Symbolizer.__EOF
169    return s
170
171  def __process_source_path(self, file_name):
172    for path_to_cut in self.__paths_to_cut:
173      file_name = re.sub(".*" + path_to_cut, "", file_name)
174    file_name = re.sub(".*hwasan_[a-z_]*.(cc|h):[0-9]*", "[hwasan_rtl]", file_name)
175    file_name = re.sub(".*asan_[a-z_]*.(cc|h):[0-9]*", "[asan_rtl]", file_name)
176    file_name = re.sub(".*crtstuff.c:0", "???:0", file_name)
177    return file_name
178
179  def __process_binary_name(self, name, buildid):
180    if name.startswith('/'):
181      name = name[1:]
182    if buildid is not None and buildid in self.__index:
183      return self.__index[buildid]
184
185    for p in self.__binary_prefixes:
186      full_path = os.path.join(p, name)
187      if os.path.exists(full_path):
188        return full_path
189      apex_prefix = "apex/com.android."
190      if name.startswith(apex_prefix):
191        full_path = os.path.join(p, "apex/com.google.android." + name[len(apex_prefix):])
192        if os.path.exists(full_path):
193          return full_path
194    # Try stripping extra path components as the last resort.
195    for p in self.__binary_prefixes:
196      full_path = os.path.join(p, os.path.basename(name))
197      if os.path.exists(full_path):
198        return full_path
199    if name not in self.__warnings:
200      print("Could not find symbols for", name, file=sys.stderr)
201      self.__warnings.add(name)
202    return None
203
204  def iter_locals(self, binary, addr, buildid):
205    self.__open_pipe()
206    p = self.__pipe
207    binary = self.__process_binary_name(binary, buildid)
208    if not binary:
209      return
210    self.__write("FRAME %s %s" % (binary, addr))
211    try:
212      while True:
213        function_name = self.__read()
214        local_name = self.__read()
215        file_line = self.__read()
216        extra = self.__read().split()
217
218        file_line = self.__process_source_path(file_line)
219        offset = None if extra[0] == '??' else int(extra[0])
220        size = None if extra[1] == '??' else int(extra[1])
221        tag_offset = None if extra[2] == '??' else int(extra[2])
222        yield (function_name, file_line, local_name, offset, size, tag_offset)
223    except Symbolizer.__EOF:
224      pass
225
226  def iter_call_stack(self, binary, buildid, addr):
227    self.__open_pipe()
228    p = self.__pipe
229    binary = self.__process_binary_name(binary, buildid)
230    if not binary:
231      return
232    self.__write("CODE %s %s" % (binary, addr))
233    try:
234      while True:
235        function_name = self.__read()
236        file_line = self.__read()
237        file_line = self.__process_source_path(file_line)
238        yield (function_name, file_line)
239    except Symbolizer.__EOF:
240      pass
241
242  def maybe_linkify(self, file_line):
243    if not self.__html or not self.__link_prefixes:
244      return file_line
245    filename, line_col = file_line.split(':', 1)
246    if not line_col:
247      line = '0' # simplify the link generation
248    else:
249      line = line_col.split(':')[0]
250    longest_prefix = max((
251      (prefix, link) for prefix, link in self.__link_prefixes
252      if filename.startswith(prefix)),
253      key=lambda x: len(x[0]), default=None)
254    if longest_prefix is None:
255      return file_line
256    else:
257      prefix, link = longest_prefix
258      return '<a href="{}">{}</a>'.format(
259        html.escape(link.format(file=filename[len(prefix):], line=line,
260                                file_line=file_line, prefix=prefix)), file_line)
261
262  def build_index(self):
263    for p in self.__binary_prefixes:
264      for dname, _, fnames in os.walk(p):
265        for fn in fnames:
266          filename = os.path.join(dname, fn)
267          bid = get_buildid(filename)
268          if bid is not None:
269            self.__index[bid] = filename
270
271def symbolize_line(line, symbolizer_path):
272  #0 0x7f6e35cf2e45  (/blah/foo.so+0x11fe45) (BuildId: 4abce4cd41ea5c2f34753297b7e774d9)
273  match = re.match(r'^(.*?)#([0-9]+)( *)(0x[0-9a-f]*) *\((.*)\+(0x[0-9a-f]+)\)'
274                   r'(?:\s*\(BuildId: ([0-9a-f]+)\))?', line, re.UNICODE)
275  if match:
276    frameno = match.group(2)
277    binary = match.group(5)
278    addr = int(match.group(6), 16)
279    buildid = match.group(7)
280
281    frames = list(symbolizer.iter_call_stack(binary, buildid, addr))
282
283    if len(frames) > 0:
284      symbolizer.print(
285        symbolizer.maybe_escape(
286          "%s#%s%s%s in " % (match.group(1), match.group(2), match.group(3),
287                            frames[0][0])
288        ) + symbolizer.maybe_linkify(frames[0][1]),
289        escape=False)
290      for i in range(1, len(frames)):
291        space1 = ' ' * match.end(1)
292        space2 = ' ' * (match.start(4) - match.end(1) - 2)
293        symbolizer.print(
294          symbolizer.maybe_escape("%s->%s%s in " % (space1, space2, frames[i][0]))
295            + symbolizer.maybe_linkify(frames[i][1]), escape=False)
296    else:
297      symbolizer.print(line.rstrip())
298  else:
299    symbolizer.print(line.rstrip())
300
301def save_access_address(line):
302  global last_access_address, last_access_tag
303  match = re.match(r'^(.*?)HWAddressSanitizer: tag-mismatch on address (0x[0-9a-f]+) ', line, re.UNICODE)
304  if match:
305    last_access_address = int(match.group(2), 16)
306  match = re.match(r'^(.*?) of size [0-9]+ at 0x[0-9a-f]* tags: ([0-9a-f]+)/[0-9a-f]+ \(ptr/mem\)', line, re.UNICODE)
307  if match:
308    last_access_tag = int(match.group(2), 16)
309
310def process_stack_history(line, symbolizer, ignore_tags=False):
311  if last_access_address is None or last_access_tag is None:
312    return
313  if re.match(r'Previously allocated frames:', line, re.UNICODE):
314    return True
315  pc_mask = (1 << 48) - 1
316  fp_mask = (1 << 20) - 1
317  # record_addr:0x1234ABCD record:0x1234ABCD (/path/to/binary+0x1234ABCD) (BuildId: 4abce4cd41ea5c2f34753297b7e774d9)
318  match = re.match(r'^(.*?)record_addr:(0x[0-9a-f]+) +record:(0x[0-9a-f]+) +\((.*)\+(0x[0-9a-f]+)\)'
319                   r'(?:\s*\(BuildId: ([0-9a-f]+)\))?', line, re.UNICODE)
320  if match:
321    record_addr = int(match.group(2), 16)
322    record = int(match.group(3), 16)
323    binary = match.group(4)
324    addr = int(match.group(5), 16)
325    buildid = match.group(6)
326    base_tag = (record_addr >> 3) & 0xFF
327    fp = (record >> 48) << 4
328    pc = record & pc_mask
329
330    for local in symbolizer.iter_locals(binary, addr, buildid):
331      frame_offset = local[3]
332      size = local[4]
333      if frame_offset is None or size is None:
334        continue
335      obj_offset = (last_access_address - fp - frame_offset) & fp_mask
336      if obj_offset >= size:
337        continue
338      tag_offset = local[5]
339      if not ignore_tags and (tag_offset is None or base_tag ^ tag_offset != last_access_tag):
340        continue
341      symbolizer.print('')
342      symbolizer.print('Potentially referenced stack object:')
343      symbolizer.print('  %d bytes inside variable "%s" in stack frame of function "%s"' % (obj_offset, local[2], local[0]))
344      symbolizer.print('  at %s' % (local[1],))
345    return True
346  return False
347
348parser = argparse.ArgumentParser()
349parser.add_argument('-d', action='store_true')
350parser.add_argument('-v', action='store_true')
351parser.add_argument('--ignore-tags', action='store_true')
352parser.add_argument('--symbols', action='append')
353parser.add_argument('--source', action='append')
354parser.add_argument('--index', action='store_true')
355parser.add_argument('--symbolizer')
356parser.add_argument('--linkify', type=str)
357parser.add_argument('--html', action='store_true')
358parser.add_argument('args', nargs=argparse.REMAINDER)
359args = parser.parse_args()
360
361# Unstripped binaries location.
362binary_prefixes = args.symbols or []
363if not binary_prefixes:
364  if 'ANDROID_PRODUCT_OUT' in os.environ:
365    product_out = os.path.join(os.environ['ANDROID_PRODUCT_OUT'], 'symbols')
366    binary_prefixes.append(product_out)
367  binary_prefixes.append('/')
368
369for p in binary_prefixes:
370  if not os.path.isdir(p):
371    print("Symbols path does not exist or is not a directory:", p, file=sys.stderr)
372    sys.exit(1)
373
374# Source location.
375paths_to_cut = args.source or []
376if not paths_to_cut:
377  paths_to_cut.append(os.getcwd() + '/')
378  if 'ANDROID_BUILD_TOP' in os.environ:
379    paths_to_cut.append(os.environ['ANDROID_BUILD_TOP'] + '/')
380
381# llvm-symbolizer binary.
382# 1. --symbolizer flag
383# 2. environment variable
384# 3. unsuffixed binary in the current directory
385# 4. if inside Android platform, prebuilt binary at a known path
386# 5. first "llvm-symbolizer", then "llvm-symbolizer-$VER" with the
387#    highest available version in $PATH
388symbolizer_path = args.symbolizer
389if not symbolizer_path:
390  if 'LLVM_SYMBOLIZER_PATH' in os.environ:
391    symbolizer_path = os.environ['LLVM_SYMBOLIZER_PATH']
392  elif 'HWASAN_SYMBOLIZER_PATH' in os.environ:
393    symbolizer_path = os.environ['HWASAN_SYMBOLIZER_PATH']
394
395if not symbolizer_path:
396  s = os.path.join(os.path.dirname(sys.argv[0]), 'llvm-symbolizer')
397  if os.path.exists(s):
398    symbolizer_path = s
399
400if not symbolizer_path:
401  if 'ANDROID_BUILD_TOP' in os.environ:
402    s = os.path.join(os.environ['ANDROID_BUILD_TOP'], 'prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer')
403    if os.path.exists(s):
404      symbolizer_path = s
405
406if not symbolizer_path:
407  for path in os.environ["PATH"].split(os.pathsep):
408    p = os.path.join(path, 'llvm-symbolizer')
409    if os.path.exists(p):
410      symbolizer_path = p
411      break
412
413def extract_version(s):
414  idx = s.rfind('-')
415  if idx == -1:
416    return 0
417  x = float(s[idx + 1:])
418  return x
419
420if not symbolizer_path:
421  for path in os.environ["PATH"].split(os.pathsep):
422    candidates = glob.glob(os.path.join(path, 'llvm-symbolizer-*'))
423    if len(candidates) > 0:
424      candidates.sort(key = extract_version, reverse = True)
425      symbolizer_path = candidates[0]
426      break
427
428if not os.path.exists(symbolizer_path):
429  print("Symbolizer path does not exist:", symbolizer_path, file=sys.stderr)
430  sys.exit(1)
431
432if args.v:
433  print("Looking for symbols in:")
434  for s in binary_prefixes:
435    print("  %s" % (s,))
436  print("Stripping source path prefixes:")
437  for s in paths_to_cut:
438    print("  %s" % (s,))
439  print("Using llvm-symbolizer binary in:\n  %s" % (symbolizer_path,))
440  print()
441
442symbolizer = Symbolizer(symbolizer_path, binary_prefixes, paths_to_cut)
443symbolizer.enable_html(args.html)
444symbolizer.enable_logging(args.d)
445if args.index:
446  symbolizer.build_index()
447
448if args.linkify:
449  if not args.html:
450    print('Need --html to --linkify', file=sys.stderr)
451    sys.exit(1)
452  symbolizer.read_linkify(args.linkify)
453
454for line in sys.stdin:
455  if sys.version_info.major < 3:
456    line = line.decode('utf-8')
457  save_access_address(line)
458  if process_stack_history(line, symbolizer, ignore_tags=args.ignore_tags):
459    continue
460  symbolize_line(line, symbolizer_path)
461