[lit] Add an option to print all features used in tests
Lit test suites can tend to accumulate annotations that are not necessarily relevant as time goes by, for example XFAILS on old compilers or platforms. To help spot old annotations that can be cleaned up, it can be useful to look at all features used inside a test suite. This commit adds a new Lit option '--show-used-features' that prints all the features used in XFAIL, REQUIRES and UNSUPPORTED of all tests that are discovered. Differential Revision: https://reviews.llvm.org/D78589
This commit is contained in:
parent
01f999ae88
commit
dac21fd29c
|
@ -79,9 +79,10 @@ class BooleanExpression:
|
|||
raise ValueError("expected: %s\nhave: %s" %
|
||||
(self.quote(t), self.quote(self.token)))
|
||||
|
||||
def isIdentifier(self, t):
|
||||
if (t is BooleanExpression.END or t == '&&' or t == '||' or
|
||||
t == '!' or t == '(' or t == ')'):
|
||||
@staticmethod
|
||||
def isIdentifier(token):
|
||||
if (token is BooleanExpression.END or token == '&&' or token == '||' or
|
||||
token == '!' or token == '(' or token == ')'):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
@ -92,7 +93,7 @@ class BooleanExpression:
|
|||
elif self.accept('('):
|
||||
self.parseOR()
|
||||
self.expect(')')
|
||||
elif not self.isIdentifier(self.token):
|
||||
elif not BooleanExpression.isIdentifier(self.token):
|
||||
raise ValueError("expected: '!' or '(' or identifier\nhave: %s" %
|
||||
self.quote(self.token))
|
||||
else:
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import itertools
|
||||
import os
|
||||
from xml.sax.saxutils import quoteattr
|
||||
from json import JSONEncoder
|
||||
|
@ -359,6 +360,26 @@ class Test:
|
|||
except ValueError as e:
|
||||
raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e))
|
||||
|
||||
def getUsedFeatures(self):
|
||||
"""
|
||||
getUsedFeatures() -> list of strings
|
||||
|
||||
Returns a list of all features appearing in XFAIL, UNSUPPORTED and
|
||||
REQUIRES annotations for this test.
|
||||
"""
|
||||
import lit.TestRunner
|
||||
parsed = lit.TestRunner._parseKeywords(self.getSourcePath(), require_script=False)
|
||||
feature_keywords = ('UNSUPPORTED:', 'REQUIRES:', 'XFAIL:')
|
||||
boolean_expressions = itertools.chain.from_iterable(
|
||||
parsed[k] or [] for k in feature_keywords
|
||||
)
|
||||
tokens = itertools.chain.from_iterable(
|
||||
BooleanExpression.tokenize(expr) for expr in
|
||||
boolean_expressions if expr != '*'
|
||||
)
|
||||
identifiers = set(filter(BooleanExpression.isIdentifier, tokens))
|
||||
return identifiers
|
||||
|
||||
def isEarlyTest(self):
|
||||
"""
|
||||
isEarlyTest() -> bool
|
||||
|
|
|
@ -1373,31 +1373,26 @@ class IntegratedTestKeywordParser(object):
|
|||
BooleanExpression.evaluate(s, [])
|
||||
return output
|
||||
|
||||
def parseIntegratedTestScript(test, additional_parsers=[],
|
||||
require_script=True):
|
||||
"""parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
|
||||
script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
|
||||
'UNSUPPORTED' and 'ALLOW_RETRIES' information.
|
||||
|
||||
If additional parsers are specified then the test is also scanned for the
|
||||
keywords they specify and all matches are passed to the custom parser.
|
||||
def _parseKeywords(sourcepath, additional_parsers=[],
|
||||
require_script=True):
|
||||
"""_parseKeywords
|
||||
|
||||
If 'require_script' is False an empty script
|
||||
may be returned. This can be used for test formats where the actual script
|
||||
is optional or ignored.
|
||||
Scan an LLVM/Clang style integrated test script and extract all the lines
|
||||
pertaining to a special parser. This includes 'RUN', 'XFAIL', 'REQUIRES',
|
||||
'UNSUPPORTED' and 'ALLOW_RETRIES', as well as other specified custom
|
||||
parsers.
|
||||
|
||||
Returns a dictionary mapping each custom parser to its value after
|
||||
parsing the test.
|
||||
"""
|
||||
|
||||
# Install the built-in keyword parsers.
|
||||
script = []
|
||||
builtin_parsers = [
|
||||
IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND,
|
||||
initial_value=script),
|
||||
IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR,
|
||||
initial_value=test.xfails),
|
||||
IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
|
||||
initial_value=test.requires),
|
||||
IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
|
||||
initial_value=test.unsupported),
|
||||
IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, initial_value=script),
|
||||
IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR),
|
||||
IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR),
|
||||
IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR),
|
||||
IntegratedTestKeywordParser('ALLOW_RETRIES:', ParserKind.INTEGER),
|
||||
IntegratedTestKeywordParser('END.', ParserKind.TAG)
|
||||
]
|
||||
|
@ -1414,7 +1409,6 @@ def parseIntegratedTestScript(test, additional_parsers=[],
|
|||
keyword_parsers[parser.keyword] = parser
|
||||
|
||||
# Collect the test lines from the script.
|
||||
sourcepath = test.getSourcePath()
|
||||
for line_number, command_type, ln in \
|
||||
parseIntegratedTestScriptCommands(sourcepath,
|
||||
keyword_parsers.keys()):
|
||||
|
@ -1441,6 +1435,37 @@ def parseIntegratedTestScript(test, additional_parsers=[],
|
|||
if value and value[-1][-1] == '\\':
|
||||
raise ValueError("Test has unterminated %s lines (with '\\')" % key)
|
||||
|
||||
# Make sure there's at most one ALLOW_RETRIES: line
|
||||
allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue()
|
||||
if allowed_retries and len(allowed_retries) > 1:
|
||||
return lit.Test.Result(Test.UNRESOLVED,
|
||||
"Test has more than one ALLOW_RETRIES lines")
|
||||
|
||||
return {p.keyword: p.getValue() for p in keyword_parsers.values()}
|
||||
|
||||
|
||||
def parseIntegratedTestScript(test, additional_parsers=[],
|
||||
require_script=True):
|
||||
"""parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
|
||||
script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
|
||||
'UNSUPPORTED' and 'ALLOW_RETRIES' information into the given test.
|
||||
|
||||
If additional parsers are specified then the test is also scanned for the
|
||||
keywords they specify and all matches are passed to the custom parser.
|
||||
|
||||
If 'require_script' is False an empty script
|
||||
may be returned. This can be used for test formats where the actual script
|
||||
is optional or ignored.
|
||||
"""
|
||||
# Parse the test sources and extract test properties
|
||||
parsed = _parseKeywords(test.getSourcePath(), additional_parsers, require_script)
|
||||
script = parsed['RUN:'] or []
|
||||
test.xfails = parsed['XFAIL:'] or []
|
||||
test.requires = parsed['REQUIRES:'] or []
|
||||
test.unsupported = parsed['UNSUPPORTED:'] or []
|
||||
if parsed['ALLOW_RETRIES:']:
|
||||
test.allowed_retries = parsed['ALLOW_RETRIES:'][0]
|
||||
|
||||
# Enforce REQUIRES:
|
||||
missing_required_features = test.getMissingRequiredFeatures()
|
||||
if missing_required_features:
|
||||
|
@ -1458,14 +1483,6 @@ def parseIntegratedTestScript(test, additional_parsers=[],
|
|||
"Test does not support the following features "
|
||||
"and/or targets: %s" % msg)
|
||||
|
||||
# Handle ALLOW_RETRIES:
|
||||
allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue()
|
||||
if allowed_retries:
|
||||
if len(allowed_retries) > 1:
|
||||
return lit.Test.Result(Test.UNRESOLVED,
|
||||
"Test has more than one ALLOW_RETRIES lines")
|
||||
test.allowed_retries = allowed_retries[0]
|
||||
|
||||
# Enforce limit_to_features.
|
||||
if not test.isWithinFeatureLimits():
|
||||
msg = ', '.join(test.config.limit_to_features)
|
||||
|
|
|
@ -157,6 +157,9 @@ def parse_args():
|
|||
debug_group.add_argument("--show-tests",
|
||||
help="Show all discovered tests and exit",
|
||||
action="store_true")
|
||||
debug_group.add_argument("--show-used-features",
|
||||
help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
|
||||
action="store_true")
|
||||
|
||||
# LIT is special: environment variables override command line arguments.
|
||||
env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
|
||||
|
|
|
@ -4,6 +4,7 @@ lit - LLVM Integrated Tester.
|
|||
See lit.pod for more information.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
|
@ -47,6 +48,11 @@ def main(builtin_params={}):
|
|||
print_discovered(discovered_tests, opts.show_suites, opts.show_tests)
|
||||
sys.exit(0)
|
||||
|
||||
if opts.show_used_features:
|
||||
features = set(itertools.chain.from_iterable(t.getUsedFeatures() for t in discovered_tests))
|
||||
print(' '.join(sorted(features)))
|
||||
sys.exit(0)
|
||||
|
||||
# Command line overrides configuration for maxIndividualTestTime.
|
||||
if opts.maxIndividualTestTime is not None: # `not None` is important (default: 0)
|
||||
if opts.maxIndividualTestTime != lit_config.maxIndividualTestTime:
|
||||
|
@ -127,7 +133,6 @@ def print_discovered(tests, show_suites, show_tests):
|
|||
tests.sort(key=lit.reports.by_suite_and_test_path)
|
||||
|
||||
if show_suites:
|
||||
import itertools
|
||||
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
|
||||
print('-- Test Suites --')
|
||||
for suite, test_iter in tests_by_suite:
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
import lit.formats
|
||||
config.name = 'show-used-features'
|
||||
config.suffixes = ['.txt']
|
||||
config.test_format = lit.formats.ShTest()
|
||||
config.test_source_root = None
|
||||
config.test_exec_root = None
|
|
@ -0,0 +1,4 @@
|
|||
|
||||
// REQUIRES: my-require-feature-2 || my-require-feature-3
|
||||
// UNSUPPORTED: my-unsupported-feature-2, my-unsupported-feature-3
|
||||
// XFAIL: my-xfail-feature-2, my-xfail-feature-3
|
|
@ -0,0 +1,2 @@
|
|||
|
||||
// REQUIRES: my-require-feature-1
|
|
@ -0,0 +1,2 @@
|
|||
|
||||
// UNSUPPORTED: my-unsupported-feature-1
|
|
@ -0,0 +1,2 @@
|
|||
|
||||
// XFAIL: my-xfail-feature-1
|
|
@ -0,0 +1,6 @@
|
|||
# Check that --show-used-features works correctly.
|
||||
#
|
||||
# RUN: %{lit} %{inputs}/show-used-features --show-used-features | FileCheck %s
|
||||
# CHECK: my-require-feature-1 my-require-feature-2 my-require-feature-3
|
||||
# CHECK: my-unsupported-feature-1 my-unsupported-feature-2 my-unsupported-feature-3
|
||||
# CHECK: my-xfail-feature-1 my-xfail-feature-2 my-xfail-feature-3
|
Loading…
Reference in New Issue