# -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80: # Configuration file for the 'lit' test runner. import os import sys import platform import tempfile import signal import subprocess import errno import time import shlex # FIXME: For now, this is cribbed from lit.TestRunner, to avoid introducing a # dependency there. What we more ideally would like to do is lift the "xfail" # and "requires" handling to be a core lit framework feature. def isExpectedFail(test, xfails): # Check if any of the xfails match an available feature or the target. for item in xfails: # If this is the wildcard, it always fails. if item == '*': return True # If this is a part of any of the features, it fails. for feature in test.config.available_features: if item in feature: return True # If this is a part of the target triple, it fails. if item in test.suite.config.target_triple: return True return False class LibcxxTestFormat(lit.formats.FileBasedTest): """ Custom test format handler for use with the test format use by libc++. Tests fall into two categories: FOO.pass.cpp - Executable test which should compile, run, and exit with code 0. FOO.fail.cpp - Negative test case which is expected to fail compilation. """ def __init__(self, cxx_under_test, cpp_flags, ld_flags, exec_env): self.cxx_under_test = cxx_under_test self.cpp_flags = list(cpp_flags) self.ld_flags = list(ld_flags) self.exec_env = dict(exec_env) def execute_command(self, command, in_dir=None): kwargs = { 'stdin' :subprocess.PIPE, 'stdout':subprocess.PIPE, 'stderr':subprocess.PIPE, } if in_dir: kwargs['cwd'] = in_dir p = subprocess.Popen(command, **kwargs) out,err = p.communicate() exitCode = p.wait() # Detect Ctrl-C in subprocess. if exitCode == -signal.SIGINT: raise KeyboardInterrupt return out, err, exitCode def execute(self, test, lit_config): while True: try: return self._execute(test, lit_config) except OSError, oe: if oe.errno != errno.ETXTBSY: raise time.sleep(0.1) def _execute(self, test, lit_config): # Extract test metadata from the test file. xfails = [] requires = [] with open(test.getSourcePath()) as f: for ln in f: if 'XFAIL:' in ln: items = ln[ln.index('XFAIL:') + 6:].split(',') xfails.extend([s.strip() for s in items]) elif 'REQUIRES:' in ln: items = ln[ln.index('REQUIRES:') + 9:].split(',') requires.extend([s.strip() for s in items]) elif not ln.startswith("//") and ln.strip(): # Stop at the first non-empty line that is not a C++ # comment. break # Check that we have the required features. # # FIXME: For now, this is cribbed from lit.TestRunner, to avoid # introducing a dependency there. What we more ideally would like to do # is lift the "xfail" and "requires" handling to be a core lit # framework feature. missing_required_features = [f for f in requires if f not in test.config.available_features] if missing_required_features: return (lit.Test.UNSUPPORTED, "Test requires the following features: %s" % ( ', '.join(missing_required_features),)) # Determine if this test is an expected failure. isXFail = isExpectedFail(test, xfails) # Evaluate the test. result, report = self._evaluate_test(test, lit_config) # Convert the test result based on whether this is an expected failure. if isXFail: if result != lit.Test.FAIL: report += "\n\nTest was expected to FAIL, but did not.\n" result = lit.Test.XPASS else: result = lit.Test.XFAIL return result, report def _evaluate_test(self, test, lit_config): name = test.path_in_suite[-1] source_path = test.getSourcePath() source_dir = os.path.dirname(source_path) # Check what kind of test this is. assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp') expected_compile_fail = name.endswith('.fail.cpp') # If this is a compile (failure) test, build it and check for failure. if expected_compile_fail: cmd = [self.cxx_under_test, '-c', '-o', '/dev/null', source_path] + self.cpp_flags out, err, exitCode = self.execute_command(cmd) if exitCode == 1: return lit.Test.PASS, "" else: report = """Command: %s\n""" % ' '.join(["'%s'" % a for a in cmd]) report += """Exit Code: %d\n""" % exitCode if out: report += """Standard Output:\n--\n%s--""" % out if err: report += """Standard Error:\n--\n%s--""" % err report += "\n\nExpected compilation to fail!" return lit.Test.FAIL, report else: exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False) exec_path = exec_file.name exec_file.close() try: compile_cmd = [self.cxx_under_test, '-o', exec_path, source_path] + self.cpp_flags + self.ld_flags cmd = compile_cmd out, err, exitCode = self.execute_command(cmd) if exitCode != 0: report = """Command: %s\n""" % ' '.join(["'%s'" % a for a in cmd]) report += """Exit Code: %d\n""" % exitCode if out: report += """Standard Output:\n--\n%s--""" % out if err: report += """Standard Error:\n--\n%s--""" % err report += "\n\nCompilation failed unexpectedly!" return lit.Test.FAIL, report cmd = [] if self.exec_env: cmd.append('env') cmd.extend('%s=%s' % (name, value) for name,value in self.exec_env.items()) cmd.append(exec_path) if lit_config.useValgrind: cmd = lit_config.valgrindArgs + cmd out, err, exitCode = self.execute_command(cmd, source_dir) if exitCode != 0: report = """Compiled With: %s\n""" % \ ' '.join(["'%s'" % a for a in compile_cmd]) report += """Command: %s\n""" % \ ' '.join(["'%s'" % a for a in cmd]) report += """Exit Code: %d\n""" % exitCode if out: report += """Standard Output:\n--\n%s--""" % out if err: report += """Standard Error:\n--\n%s--""" % err report += "\n\nCompiled test failed unexpectedly!" return lit.Test.FAIL, report finally: try: os.remove(exec_path) except: pass return lit.Test.PASS, "" # name: The name of this test suite. config.name = 'libc++' # suffixes: A list of file extensions to treat as test files. config.suffixes = ['.cpp'] # test_source_root: The root path where tests are located. config.test_source_root = os.path.dirname(__file__) # Gather various compiler parameters. cxx_under_test = lit.params.get('cxx_under_test', None) if cxx_under_test is None: cxx_under_test = getattr(config, 'cxx_under_test', None) # If no specific cxx_under_test was given, attempt to infer it as clang++. clangxx = lit.util.which('clang++', config.environment['PATH']) if clangxx is not None: cxx_under_test = clangxx lit.note("inferred cxx_under_test as: %r" % (cxx_under_test,)) if cxx_under_test is None: lit.fatal('must specify user parameter cxx_under_test ' '(e.g., --param=cxx_under_test=clang++)') libcxx_src_root = lit.params.get('libcxx_src_root', None) if libcxx_src_root is None: libcxx_src_root = getattr(config, 'libcxx_src_root', None) if libcxx_src_root is None: libcxx_src_root = os.path.dirname(config.test_source_root) libcxx_obj_root = lit.params.get('libcxx_obj_root', None) if libcxx_obj_root is None: libcxx_obj_root = getattr(config, 'libcxx_obj_root', None) if libcxx_obj_root is None: libcxx_obj_root = libcxx_src_root cxx_has_stdcxx0x_flag_str = lit.params.get('cxx_has_stdcxx0x_flag', None) if cxx_has_stdcxx0x_flag_str is not None: if cxx_has_stdcxx0x_flag_str.lower() in ('1', 'true'): cxx_has_stdcxx0x_flag = True elif cxx_has_stdcxx0x_flag_str.lower() in ('', '0', 'false'): cxx_has_stdcxx0x_flag = False else: lit.fatal('user parameter cxx_has_stdcxx0x_flag_str should be 0 or 1') else: cxx_has_stdcxx0x_flag = getattr(config, 'cxx_has_stdcxx0x_flag', True) # This test suite supports testing against either the system library or the # locally built one; the former mode is useful for testing ABI compatibility # between the current headers and a shipping dynamic library. use_system_lib_str = lit.params.get('use_system_lib', None) if use_system_lib_str is not None: if use_system_lib_str.lower() in ('1', 'true'): use_system_lib = True elif use_system_lib_str.lower() in ('', '0', 'false'): use_system_lib = False else: lit.fatal('user parameter use_system_lib should be 0 or 1') else: # Default to testing against the locally built libc++ library. use_system_lib = False lit.note("inferred use_system_lib as: %r" % (use_system_lib,)) link_flags = [] link_flags_str = lit.params.get('link_flags', None) if link_flags_str is None: link_flags_str = getattr(config, 'link_flags', None) if link_flags_str is None: if sys.platform == 'darwin': link_flags += ['-lSystem'] elif sys.platform == 'linux2': link_flags += ['-lsupc++', '-lgcc_eh', '-lc', '-lm', '-lpthread', '-lrt', '-lgcc_s'] else: lit.fatal("unrecognized system") lit.note("inferred link_flags as: %r" % (link_flags,)) if not link_flags_str is None: link_flags += shlex.split(link_flags_str) # Configure extra compiler flags. include_paths = ['-I' + libcxx_src_root + '/include', '-I' + libcxx_src_root + '/test/support'] library_paths = ['-L' + libcxx_obj_root + '/lib'] compile_flags = [] if cxx_has_stdcxx0x_flag: compile_flags += ['-std=c++0x'] # Configure extra linker parameters. exec_env = {} if sys.platform == 'darwin': if not use_system_lib: exec_env['DYLD_LIBRARY_PATH'] = os.path.join(libcxx_obj_root, 'lib') elif sys.platform == 'linux2': if not use_system_lib: link_flags += ['-Wl,-R', libcxx_obj_root + '/lib'] compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS'] else: lit.fatal("unrecognized system") config.test_format = LibcxxTestFormat( cxx_under_test, cpp_flags = ['-nostdinc++'] + compile_flags + include_paths, ld_flags = ['-nodefaultlibs'] + library_paths + ['-lc++'] + link_flags, exec_env = exec_env) # Get or infer the target triple. config.target_triple = lit.params.get('target_triple', None) # If no target triple was given, try to infer it from the compiler under test. if config.target_triple is None: config.target_triple = lit.util.capture( [cxx_under_test, '-dumpmachine']).strip() lit.note("inferred target_triple as: %r" % (config.target_triple,)) # Write an "available feature" that combines the triple when use_system_lib is # enabled. This is so that we can easily write XFAIL markers for tests that are # known to fail with versions of libc++ as were shipped with a particular # triple. if use_system_lib: config.available_features.add('with_system_lib=%s' % ( config.target_triple,))