[UpdateTestChecks] Share the code to parse RUN: lines between all scripts
Summary: This commit also introduces a common.debug() function to avoid many `if args.verbose:` statements. Depends on D70428. Reviewers: xbolva00, MaskRay, jdoerfert Reviewed By: MaskRay Subscribers: llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D70432
This commit is contained in:
Alex Richardson
parent
160a5045c6
commit
d9542db49e
|
|
@ -14,12 +14,17 @@ else:
|
|||
##### Common utilities for update_*test_checks.py
|
||||
|
||||
|
||||
_verbose = False
|
||||
|
||||
def parse_commandline_args(parser):
|
||||
parser.add_argument('-v', '--verbose', action='store_true',
|
||||
help='Show verbose output')
|
||||
parser.add_argument('-u', '--update-only', action='store_true',
|
||||
help='Only update test if it was already autogened')
|
||||
return parser.parse_args()
|
||||
args = parser.parse_args()
|
||||
global _verbose
|
||||
_verbose = args.verbose
|
||||
return args
|
||||
|
||||
def should_add_line_to_output(input_line, prefix_set):
|
||||
# Skip any blank comment lines in the IR.
|
||||
|
|
@ -53,7 +58,7 @@ def invoke_tool(exe, cmd_args, ir):
|
|||
|
||||
##### LLVM IR parser
|
||||
|
||||
RUN_LINE_RE = re.compile(r'^\s*[;#]\s*RUN:\s*(.*)$')
|
||||
RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$')
|
||||
CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')
|
||||
PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')
|
||||
CHECK_RE = re.compile(r'^\s*[;#]\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME)?:')
|
||||
|
|
@ -91,6 +96,28 @@ def warn(msg, test_file=None):
|
|||
msg = '{}: {}'.format(msg, test_file)
|
||||
print('WARNING: {}'.format(msg), file=sys.stderr)
|
||||
|
||||
def debug(*args, **kwargs):
|
||||
# Python2 does not allow def debug(*args, file=sys.stderr, **kwargs):
|
||||
if 'file' not in kwargs:
|
||||
kwargs['file'] = sys.stderr
|
||||
if _verbose:
|
||||
print(*args, **kwargs)
|
||||
|
||||
def find_run_lines(test, lines):
|
||||
debug('Scanning for RUN lines in test file:', test)
|
||||
raw_lines = [m.group(1)
|
||||
for m in [RUN_LINE_RE.match(l) for l in lines] if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith('\\'):
|
||||
run_lines[-1] = run_lines[-1].rstrip('\\' + ' ' + l)
|
||||
else:
|
||||
run_lines.append(l)
|
||||
debug('Found {} RUN lines in {}:'.format(len(run_lines), test))
|
||||
for l in run_lines:
|
||||
debug(' RUN: {}'.format(l))
|
||||
return run_lines
|
||||
|
||||
def scrub_body(body):
|
||||
# Scrub runs of whitespace out of the assembly, but leave the leading
|
||||
# whitespace in place.
|
||||
|
|
|
|||
|
|
@ -69,8 +69,6 @@ def main():
|
|||
|
||||
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
|
||||
for test in test_paths:
|
||||
if args.verbose:
|
||||
print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
|
||||
with open(test) as f:
|
||||
input_lines = [l.rstrip() for l in f]
|
||||
|
||||
|
|
@ -84,20 +82,7 @@ def main():
|
|||
common.warn("Skipping test which isn't autogenerated: " + test)
|
||||
continue
|
||||
|
||||
raw_lines = [m.group(1)
|
||||
for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith("\\"):
|
||||
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
|
||||
else:
|
||||
run_lines.append(l)
|
||||
|
||||
if args.verbose:
|
||||
print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
|
||||
for l in run_lines:
|
||||
print(' RUN: ' + l, file=sys.stderr)
|
||||
|
||||
run_lines = common.find_run_lines(test, input_lines)
|
||||
prefix_list = []
|
||||
for l in run_lines:
|
||||
if '|' not in l:
|
||||
|
|
@ -132,9 +117,8 @@ def main():
|
|||
for prefix in prefixes:
|
||||
func_dict.update({prefix: dict()})
|
||||
for prefixes, opt_args in prefix_list:
|
||||
if args.verbose:
|
||||
print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
|
||||
print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
|
||||
common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
|
||||
common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
|
||||
|
||||
raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
|
||||
|
||||
|
|
@ -147,8 +131,7 @@ def main():
|
|||
is_in_function = False
|
||||
is_in_function_start = False
|
||||
prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
|
||||
if args.verbose:
|
||||
print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
|
||||
common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
|
||||
output_lines = []
|
||||
output_lines.append(autogenerated_note)
|
||||
|
||||
|
|
@ -194,8 +177,7 @@ def main():
|
|||
continue
|
||||
is_in_function = is_in_function_start = True
|
||||
|
||||
if args.verbose:
|
||||
print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
|
||||
common.debug('Writing %d lines to %s...' % (len(output_lines), test))
|
||||
|
||||
with open(test, 'wb') as f:
|
||||
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ from UpdateTestChecks import asm, common
|
|||
ADVERT = '// NOTE: Assertions have been autogenerated by '
|
||||
|
||||
CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
|
||||
RUN_LINE_RE = re.compile(r'^//\s*RUN:\s*(.*)$')
|
||||
|
||||
SUBST = {
|
||||
'%clang': [],
|
||||
|
|
@ -38,9 +37,6 @@ SUBST = {
|
|||
}
|
||||
|
||||
def get_line2spell_and_mangled(args, clang_args):
|
||||
def debug_mangled(*print_args, **kwargs):
|
||||
if args.verbose:
|
||||
print(*print_args, file=sys.stderr, **kwargs)
|
||||
ret = {}
|
||||
# Use clang's JSON AST dump to get the mangled name
|
||||
json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-']
|
||||
|
|
@ -49,7 +45,7 @@ def get_line2spell_and_mangled(args, clang_args):
|
|||
# -Xclang -ast-dump=json instead:
|
||||
json_dump_args.append('-Xclang')
|
||||
json_dump_args.append('-ast-dump=json')
|
||||
debug_mangled('Running', ' '.join(json_dump_args))
|
||||
common.debug('Running', ' '.join(json_dump_args))
|
||||
status = subprocess.run(json_dump_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if status.returncode != 0:
|
||||
sys.stderr.write('Failed to run ' + ' '.join(json_dump_args) + '\n')
|
||||
|
|
@ -67,20 +63,19 @@ def get_line2spell_and_mangled(args, clang_args):
|
|||
if node['kind'] != 'FunctionDecl':
|
||||
continue
|
||||
if node.get('isImplicit') is True and node.get('storageClass') == 'extern':
|
||||
debug_mangled('Skipping builtin function:', node['name'], '@', node['loc'])
|
||||
common.debug('Skipping builtin function:', node['name'], '@', node['loc'])
|
||||
continue
|
||||
debug_mangled('Found function:', node['kind'], node['name'], '@', node['loc'])
|
||||
common.debug('Found function:', node['kind'], node['name'], '@', node['loc'])
|
||||
line = node['loc'].get('line')
|
||||
# If there is no line it is probably a builtin function -> skip
|
||||
if line is None:
|
||||
debug_mangled('Skipping function without line number:', node['name'], '@', node['loc'])
|
||||
common.debug('Skipping function without line number:', node['name'], '@', node['loc'])
|
||||
continue
|
||||
spell = node['name']
|
||||
mangled = node.get('mangledName', spell)
|
||||
ret[int(line)-1] = (spell, mangled)
|
||||
if args.verbose:
|
||||
for line, func_name in sorted(ret.items()):
|
||||
print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
|
||||
for line, func_name in sorted(ret.items()):
|
||||
common.debug('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
|
||||
if not ret:
|
||||
common.warn('Did not find any functions using', ' '.join(json_dump_args))
|
||||
return ret
|
||||
|
|
@ -191,19 +186,7 @@ def main():
|
|||
continue
|
||||
|
||||
# Extract RUN lines.
|
||||
raw_lines = [m.group(1)
|
||||
for m in [RUN_LINE_RE.match(l) for l in input_lines] if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith("\\"):
|
||||
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
|
||||
else:
|
||||
run_lines.append(l)
|
||||
|
||||
if args.verbose:
|
||||
print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr)
|
||||
for l in run_lines:
|
||||
print(' RUN: ' + l, file=sys.stderr)
|
||||
run_lines = common.find_run_lines(filename, input_lines)
|
||||
|
||||
# Build a list of clang command lines and check prefixes from RUN lines.
|
||||
run_list = []
|
||||
|
|
@ -260,9 +243,8 @@ def main():
|
|||
for prefix in prefixes:
|
||||
func_dict.update({prefix: dict()})
|
||||
for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
|
||||
if args.verbose:
|
||||
print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)
|
||||
print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)
|
||||
common.debug('Extracted clang cmd: clang {}'.format(clang_args))
|
||||
common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
|
||||
|
||||
get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -43,8 +43,6 @@ def main():
|
|||
|
||||
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
|
||||
for test in test_paths:
|
||||
if args.verbose:
|
||||
print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
|
||||
with open(test) as f:
|
||||
input_lines = [l.rstrip() for l in f]
|
||||
|
||||
|
|
@ -65,20 +63,7 @@ def main():
|
|||
triple_in_ir = m.groups()[0]
|
||||
break
|
||||
|
||||
raw_lines = [m.group(1)
|
||||
for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith("\\"):
|
||||
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
|
||||
else:
|
||||
run_lines.append(l)
|
||||
|
||||
if args.verbose:
|
||||
print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
|
||||
for l in run_lines:
|
||||
print(' RUN: ' + l, file=sys.stderr)
|
||||
|
||||
run_lines = common.find_run_lines(test, input_lines)
|
||||
run_list = []
|
||||
for l in run_lines:
|
||||
if '|' not in l:
|
||||
|
|
@ -115,12 +100,6 @@ def main():
|
|||
llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()
|
||||
if test.endswith('.mir'):
|
||||
llc_cmd_args += ' -x mir'
|
||||
comment_sym = '#'
|
||||
check_indent = ' '
|
||||
else:
|
||||
comment_sym = ';'
|
||||
check_indent = ''
|
||||
|
||||
check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
|
||||
for item in m.group(1).split(',')]
|
||||
if not check_prefixes:
|
||||
|
|
@ -130,6 +109,12 @@ def main():
|
|||
# now, we just ignore all but the last.
|
||||
run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
|
||||
|
||||
if test.endswith('.mir'):
|
||||
comment_sym = '#'
|
||||
check_indent = ' '
|
||||
else:
|
||||
comment_sym = ';'
|
||||
check_indent = ''
|
||||
autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name)
|
||||
|
||||
func_dict = {}
|
||||
|
|
@ -138,9 +123,8 @@ def main():
|
|||
for prefix in prefixes:
|
||||
func_dict.update({prefix: dict()})
|
||||
for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:
|
||||
if args.verbose:
|
||||
print('Extracted LLC cmd: ' + llc_tool + ' ' + llc_args, file=sys.stderr)
|
||||
print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
|
||||
common.debug('Extracted LLC cmd:', llc_tool, llc_args)
|
||||
common.debug('Extracted FileCheck prefixes:', str(prefixes))
|
||||
|
||||
raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test)
|
||||
triple = triple_in_cmd or triple_in_ir
|
||||
|
|
@ -154,8 +138,7 @@ def main():
|
|||
is_in_function_start = False
|
||||
func_name = None
|
||||
prefix_set = set([prefix for p in run_list for prefix in p[0]])
|
||||
if args.verbose:
|
||||
print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
|
||||
common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
|
||||
output_lines = []
|
||||
output_lines.append(autogenerated_note)
|
||||
|
||||
|
|
@ -199,8 +182,7 @@ def main():
|
|||
continue
|
||||
is_in_function = is_in_function_start = True
|
||||
|
||||
if args.verbose:
|
||||
print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
|
||||
common.debug('Writing %d lines to %s...' % (len(output_lines), test))
|
||||
|
||||
with open(test, 'wb') as f:
|
||||
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
|
||||
|
|
|
|||
|
|
@ -83,26 +83,6 @@ def _parse_args():
|
|||
return args
|
||||
|
||||
|
||||
def _find_run_lines(input_lines, args):
|
||||
raw_lines = [m.group(1)
|
||||
for m in [common.RUN_LINE_RE.match(l) for l in input_lines]
|
||||
if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith(r'\\'):
|
||||
run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
|
||||
else:
|
||||
run_lines.append(l)
|
||||
|
||||
if args.verbose:
|
||||
sys.stderr.write('Found {} RUN line{}:\n'.format(
|
||||
len(run_lines), '' if len(run_lines) == 1 else 's'))
|
||||
for line in run_lines:
|
||||
sys.stderr.write(' RUN: {}\n'.format(line))
|
||||
|
||||
return run_lines
|
||||
|
||||
|
||||
def _get_run_infos(run_lines, args):
|
||||
run_infos = []
|
||||
for run_line in run_lines:
|
||||
|
|
@ -544,9 +524,7 @@ def _write_output(test_path, input_lines, prefix_list, block_infos, # noqa
|
|||
return
|
||||
sys.stderr.write(' [{} lines total]\n'.format(len(output_lines)))
|
||||
|
||||
if args.verbose:
|
||||
sys.stderr.write(
|
||||
'Writing {} lines to {}...\n\n'.format(len(output_lines), test_path))
|
||||
common.debug('Writing', len(output_lines), 'lines to', test_path, '..\n\n')
|
||||
|
||||
with open(test_path, 'wb') as f:
|
||||
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
|
||||
|
|
@ -562,17 +540,13 @@ def main():
|
|||
# will be written once per source location per test.
|
||||
_configure_warnings(args)
|
||||
|
||||
if args.verbose:
|
||||
sys.stderr.write(
|
||||
'Scanning for RUN lines in test file: {}\n'.format(test_path))
|
||||
|
||||
if not os.path.isfile(test_path):
|
||||
raise Error('could not find test file: {}'.format(test_path))
|
||||
|
||||
with open(test_path) as f:
|
||||
input_lines = [l.rstrip() for l in f]
|
||||
|
||||
run_lines = _find_run_lines(input_lines, args)
|
||||
run_lines = common.find_run_lines(test_path, input_lines)
|
||||
run_infos = _get_run_infos(run_lines, args)
|
||||
common_prefix, prefix_pad = _get_useful_prefix_info(run_infos)
|
||||
block_infos = _get_block_infos(run_infos, test_path, args, common_prefix)
|
||||
|
|
|
|||
|
|
@ -96,22 +96,6 @@ def find_triple_in_ir(lines, verbose=False):
|
|||
return None
|
||||
|
||||
|
||||
def find_run_lines(test, lines, verbose=False):
|
||||
raw_lines = [m.group(1)
|
||||
for m in [common.RUN_LINE_RE.match(l) for l in lines] if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith("\\"):
|
||||
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
|
||||
else:
|
||||
run_lines.append(l)
|
||||
if verbose:
|
||||
log('Found {} RUN lines:'.format(len(run_lines)))
|
||||
for l in run_lines:
|
||||
log(' RUN: {}'.format(l))
|
||||
return run_lines
|
||||
|
||||
|
||||
def build_run_list(test, run_lines, verbose=False):
|
||||
run_list = []
|
||||
all_prefixes = []
|
||||
|
|
@ -296,7 +280,6 @@ def should_add_line_to_output(input_line, prefix_set):
|
|||
|
||||
|
||||
def update_test_file(args, test):
|
||||
log('Scanning for RUN lines in test file: {}'.format(test), args.verbose)
|
||||
with open(test) as fd:
|
||||
input_lines = [l.rstrip() for l in fd]
|
||||
|
||||
|
|
@ -313,7 +296,7 @@ def update_test_file(args, test):
|
|||
return
|
||||
|
||||
triple_in_ir = find_triple_in_ir(input_lines, args.verbose)
|
||||
run_lines = find_run_lines(test, input_lines, args.verbose)
|
||||
run_lines = common.find_run_lines(test, input_lines)
|
||||
run_list, common_prefixes = build_run_list(test, run_lines, args.verbose)
|
||||
|
||||
simple_functions = find_functions_with_one_bb(input_lines, args.verbose)
|
||||
|
|
|
|||
|
|
@ -84,8 +84,6 @@ def main():
|
|||
# On Windows we must expand the patterns ourselves.
|
||||
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
|
||||
for test in test_paths:
|
||||
if args.verbose:
|
||||
print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
|
||||
with open(test) as f:
|
||||
input_lines = [l.rstrip() for l in f]
|
||||
|
||||
|
|
@ -99,20 +97,7 @@ def main():
|
|||
common.warn("Skipping test which isn't autogenerated: " + test)
|
||||
continue
|
||||
|
||||
raw_lines = [m.group(1)
|
||||
for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
|
||||
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
|
||||
for l in raw_lines[1:]:
|
||||
if run_lines[-1].endswith('\\'):
|
||||
run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
|
||||
else:
|
||||
run_lines.append(l)
|
||||
|
||||
if args.verbose:
|
||||
print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
|
||||
for l in run_lines:
|
||||
print(' RUN: ' + l, file=sys.stderr)
|
||||
|
||||
run_lines = common.find_run_lines(test, input_lines)
|
||||
prefix_list = []
|
||||
for l in run_lines:
|
||||
if '|' not in l:
|
||||
|
|
@ -146,9 +131,8 @@ def main():
|
|||
for prefix in prefixes:
|
||||
func_dict.update({prefix: dict()})
|
||||
for prefixes, opt_args in prefix_list:
|
||||
if args.verbose:
|
||||
print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
|
||||
print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
|
||||
common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
|
||||
common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
|
||||
|
||||
raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
|
||||
common.build_function_body_dictionary(
|
||||
|
|
@ -159,8 +143,7 @@ def main():
|
|||
is_in_function = False
|
||||
is_in_function_start = False
|
||||
prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
|
||||
if args.verbose:
|
||||
print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
|
||||
common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
|
||||
output_lines = []
|
||||
output_lines.append(autogenerated_note)
|
||||
|
||||
|
|
@ -207,8 +190,7 @@ def main():
|
|||
continue
|
||||
is_in_function = is_in_function_start = True
|
||||
|
||||
if args.verbose:
|
||||
print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
|
||||
common.debug('Writing %d lines to %s...' % (len(output_lines), test))
|
||||
|
||||
with open(test, 'wb') as f:
|
||||
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
|
||||
|
|
|
|||
Loading…
Reference in New Issue