1#!/usr/bin/env python
2
3"""A script to generate FileCheck statements for 'opt' regression tests.
4
5This script is a utility to update LLVM opt test cases with new
6FileCheck patterns. It can either update all of the tests in the file or
7a single test function.
8
9Example usage:
10$ update_test_checks.py --opt=../bin/opt test/foo.ll
11
12Workflow:
131. Make a compiler patch that requires updating some number of FileCheck lines
14   in regression test files.
152. Save the patch and revert it from your local work area.
163. Update the RUN-lines in the affected regression tests to look canonical.
17   Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
184. Refresh the FileCheck lines for either the entire file or select functions by
19   running this script.
205. Commit the fresh baseline of checks.
216. Apply your patch from step 1 and rebuild your local binaries.
227. Re-run this script on affected regression tests.
238. Check the diffs to ensure the script has done something reasonable.
249. Submit a patch including the regression test diffs for review.
25
26A common pattern is to have the script insert complete checking of every
27instruction. Then, edit it down to only check the relevant instructions.
28The script is designed to make adding checks to a test case fast, it is *not*
29designed to be authoratitive about what constitutes a good test!
30"""
31
32from __future__ import print_function
33
34import argparse
35import glob
36import itertools
37import os         # Used to advertise this file's name ("autogenerated_note").
38import string
39import subprocess
40import sys
41import tempfile
42import re
43
44from UpdateTestChecks import common
45
46ADVERT = '; NOTE: Assertions have been autogenerated by '
47
48# RegEx: this is where the magic happens.
49
50IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
51
52
53
54
55
56def main():
57  from argparse import RawTextHelpFormatter
58  parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
59  parser.add_argument('-v', '--verbose', action='store_true',
60                      help='Show verbose output')
61  parser.add_argument('--opt-binary', default='opt',
62                      help='The opt binary used to generate the test case')
63  parser.add_argument(
64      '--function', help='The function in the test file to update')
65  parser.add_argument('-u', '--update-only', action='store_true',
66                      help='Only update test if it was already autogened')
67  parser.add_argument('-p', '--preserve-names', action='store_true',
68                      help='Do not scrub IR names')
69  parser.add_argument('--function-signature', action='store_true',
70                      help='Keep function signature information around for the check line')
71  parser.add_argument('tests', nargs='+')
72  args = parser.parse_args()
73
74  script_name = os.path.basename(__file__)
75  autogenerated_note = (ADVERT + 'utils/' + script_name)
76
77  opt_basename = os.path.basename(args.opt_binary)
78  if not re.match(r'^opt(-\d+)?$', opt_basename):
79    common.error('Unexpected opt name: ' + opt_basename)
80    sys.exit(1)
81  opt_basename = 'opt'
82
83  for test in args.tests:
84    if not glob.glob(test):
85      common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,))
86      continue
87
88  # On Windows we must expand the patterns ourselves.
89  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
90  for test in test_paths:
91    if args.verbose:
92      print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
93    with open(test) as f:
94      input_lines = [l.rstrip() for l in f]
95
96    first_line = input_lines[0] if input_lines else ""
97    if 'autogenerated' in first_line and script_name not in first_line:
98      common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
99      continue
100
101    if args.update_only:
102      if not first_line or 'autogenerated' not in first_line:
103        common.warn("Skipping test which isn't autogenerated: " + test)
104        continue
105
106    raw_lines = [m.group(1)
107                 for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
108    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
109    for l in raw_lines[1:]:
110      if run_lines[-1].endswith('\\'):
111        run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
112      else:
113        run_lines.append(l)
114
115    if args.verbose:
116      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
117      for l in run_lines:
118        print('  RUN: ' + l, file=sys.stderr)
119
120    prefix_list = []
121    for l in run_lines:
122      if '|' not in l:
123        common.warn('Skipping unparseable RUN line: ' + l)
124        continue
125
126      (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
127      common.verify_filecheck_prefixes(filecheck_cmd)
128      if not tool_cmd.startswith(opt_basename + ' '):
129        common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
130        continue
131
132      if not filecheck_cmd.startswith('FileCheck '):
133        common.warn('Skipping non-FileChecked RUN line: ' + l)
134        continue
135
136      tool_cmd_args = tool_cmd[len(opt_basename):].strip()
137      tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
138
139      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
140                               for item in m.group(1).split(',')]
141      if not check_prefixes:
142        check_prefixes = ['CHECK']
143
144      # FIXME: We should use multiple check prefixes to common check lines. For
145      # now, we just ignore all but the last.
146      prefix_list.append((check_prefixes, tool_cmd_args))
147
148    func_dict = {}
149    for prefixes, _ in prefix_list:
150      for prefix in prefixes:
151        func_dict.update({prefix: dict()})
152    for prefixes, opt_args in prefix_list:
153      if args.verbose:
154        print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
155        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
156
157      raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
158      common.build_function_body_dictionary(
159              common.OPT_FUNCTION_RE, common.scrub_body, [],
160              raw_tool_output, prefixes, func_dict, args.verbose,
161              args.function_signature)
162
163    is_in_function = False
164    is_in_function_start = False
165    prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
166    if args.verbose:
167      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
168    output_lines = []
169    output_lines.append(autogenerated_note)
170
171    for input_line in input_lines:
172      if is_in_function_start:
173        if input_line == '':
174          continue
175        if input_line.lstrip().startswith(';'):
176          m = common.CHECK_RE.match(input_line)
177          if not m or m.group(1) not in prefix_set:
178            output_lines.append(input_line)
179            continue
180
181        # Print out the various check lines here.
182        common.add_ir_checks(output_lines, ';', prefix_list, func_dict,
183                             func_name, args.preserve_names, args.function_signature)
184        is_in_function_start = False
185
186      if is_in_function:
187        if common.should_add_line_to_output(input_line, prefix_set):
188          # This input line of the function body will go as-is into the output.
189          # Except make leading whitespace uniform: 2 spaces.
190          input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
191          output_lines.append(input_line)
192        else:
193          continue
194        if input_line.strip() == '}':
195          is_in_function = False
196        continue
197
198      # Discard any previous script advertising.
199      if input_line.startswith(ADVERT):
200        continue
201
202      # If it's outside a function, it just gets copied to the output.
203      output_lines.append(input_line)
204
205      m = IR_FUNCTION_RE.match(input_line)
206      if not m:
207        continue
208      func_name = m.group(1)
209      if args.function is not None and func_name != args.function:
210        # When filtering on a specific function, skip all others.
211        continue
212      is_in_function = is_in_function_start = True
213
214    if args.verbose:
215      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
216
217    with open(test, 'wb') as f:
218      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
219
220
221if __name__ == '__main__':
222  main()
223