From dac21fd29cd2ae2b979a276747ad5ad82fca09bf Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Fri, 10 Apr 2020 17:41:45 -0400 Subject: [PATCH] [lit] Add an option to print all features used in tests Lit test suites can tend to accumulate annotations that are not necessarily relevant as time goes by, for example XFAILS on old compilers or platforms. To help spot old annotations that can be cleaned up, it can be useful to look at all features used inside a test suite. This commit adds a new Lit option '--show-used-features' that prints all the features used in XFAIL, REQUIRES and UNSUPPORTED of all tests that are discovered. Differential Revision: https://reviews.llvm.org/D78589 --- llvm/utils/lit/lit/BooleanExpression.py | 11 +-- llvm/utils/lit/lit/Test.py | 23 +++++- llvm/utils/lit/lit/TestRunner.py | 73 ++++++++++++------- llvm/utils/lit/lit/cl_arguments.py | 3 + llvm/utils/lit/lit/main.py | 7 +- .../tests/Inputs/show-used-features/lit.cfg | 6 ++ .../tests/Inputs/show-used-features/mixed.txt | 4 + .../Inputs/show-used-features/requires.txt | 2 + .../Inputs/show-used-features/unsupported.txt | 2 + .../tests/Inputs/show-used-features/xfail.txt | 2 + llvm/utils/lit/tests/show-used-features.py | 6 ++ 11 files changed, 104 insertions(+), 35 deletions(-) create mode 100644 llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg create mode 100644 llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt create mode 100644 llvm/utils/lit/tests/Inputs/show-used-features/requires.txt create mode 100644 llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt create mode 100644 llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt create mode 100644 llvm/utils/lit/tests/show-used-features.py diff --git a/llvm/utils/lit/lit/BooleanExpression.py b/llvm/utils/lit/lit/BooleanExpression.py index 3eb5060de3e3..34e07fc1b8e5 100644 --- a/llvm/utils/lit/lit/BooleanExpression.py +++ b/llvm/utils/lit/lit/BooleanExpression.py @@ -79,9 +79,10 @@ class BooleanExpression: raise ValueError("expected: %s\nhave: %s" % (self.quote(t), self.quote(self.token))) - def isIdentifier(self, t): - if (t is BooleanExpression.END or t == '&&' or t == '||' or - t == '!' or t == '(' or t == ')'): + @staticmethod + def isIdentifier(token): + if (token is BooleanExpression.END or token == '&&' or token == '||' or + token == '!' or token == '(' or token == ')'): return False return True @@ -92,7 +93,7 @@ class BooleanExpression: elif self.accept('('): self.parseOR() self.expect(')') - elif not self.isIdentifier(self.token): + elif not BooleanExpression.isIdentifier(self.token): raise ValueError("expected: '!' or '(' or identifier\nhave: %s" % self.quote(self.token)) else: @@ -191,7 +192,7 @@ class TestBooleanExpression(unittest.TestCase): "actual error was:\n%s\n" + "expected error was:\n%s\n") % (expr, e, error)) except BaseException as e: - self.fail(("expression %r caused the wrong exception; actual " + + self.fail(("expression %r caused the wrong exception; actual " + "exception was: \n%r") % (expr, e)) def test_errors(self): diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py index 627785829efc..4c94e9806a16 100644 --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -1,3 +1,4 @@ +import itertools import os from xml.sax.saxutils import quoteattr from json import JSONEncoder @@ -162,7 +163,7 @@ class Result(object): addMicroResult(microResult) Attach a micro-test result to the test result, with the given name and - result. It is an error to attempt to attach a micro-test with the + result. It is an error to attempt to attach a micro-test with the same name multiple times. Each micro-test result must be an instance of the Result class. @@ -359,6 +360,26 @@ class Test: except ValueError as e: raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e)) + def getUsedFeatures(self): + """ + getUsedFeatures() -> list of strings + + Returns a list of all features appearing in XFAIL, UNSUPPORTED and + REQUIRES annotations for this test. + """ + import lit.TestRunner + parsed = lit.TestRunner._parseKeywords(self.getSourcePath(), require_script=False) + feature_keywords = ('UNSUPPORTED:', 'REQUIRES:', 'XFAIL:') + boolean_expressions = itertools.chain.from_iterable( + parsed[k] or [] for k in feature_keywords + ) + tokens = itertools.chain.from_iterable( + BooleanExpression.tokenize(expr) for expr in + boolean_expressions if expr != '*' + ) + identifiers = set(filter(BooleanExpression.isIdentifier, tokens)) + return identifiers + def isEarlyTest(self): """ isEarlyTest() -> bool diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py index 6c92a2bc26c4..edcb696a8b16 100644 --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1373,31 +1373,26 @@ class IntegratedTestKeywordParser(object): BooleanExpression.evaluate(s, []) return output -def parseIntegratedTestScript(test, additional_parsers=[], - require_script=True): - """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test - script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES', - 'UNSUPPORTED' and 'ALLOW_RETRIES' information. - If additional parsers are specified then the test is also scanned for the - keywords they specify and all matches are passed to the custom parser. +def _parseKeywords(sourcepath, additional_parsers=[], + require_script=True): + """_parseKeywords - If 'require_script' is False an empty script - may be returned. This can be used for test formats where the actual script - is optional or ignored. + Scan an LLVM/Clang style integrated test script and extract all the lines + pertaining to a special parser. This includes 'RUN', 'XFAIL', 'REQUIRES', + 'UNSUPPORTED' and 'ALLOW_RETRIES', as well as other specified custom + parsers. + + Returns a dictionary mapping each custom parser to its value after + parsing the test. """ - # Install the built-in keyword parsers. script = [] builtin_parsers = [ - IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, - initial_value=script), - IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR, - initial_value=test.xfails), - IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR, - initial_value=test.requires), - IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR, - initial_value=test.unsupported), + IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, initial_value=script), + IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR), + IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR), + IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR), IntegratedTestKeywordParser('ALLOW_RETRIES:', ParserKind.INTEGER), IntegratedTestKeywordParser('END.', ParserKind.TAG) ] @@ -1414,7 +1409,6 @@ def parseIntegratedTestScript(test, additional_parsers=[], keyword_parsers[parser.keyword] = parser # Collect the test lines from the script. - sourcepath = test.getSourcePath() for line_number, command_type, ln in \ parseIntegratedTestScriptCommands(sourcepath, keyword_parsers.keys()): @@ -1441,6 +1435,37 @@ def parseIntegratedTestScript(test, additional_parsers=[], if value and value[-1][-1] == '\\': raise ValueError("Test has unterminated %s lines (with '\\')" % key) + # Make sure there's at most one ALLOW_RETRIES: line + allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue() + if allowed_retries and len(allowed_retries) > 1: + return lit.Test.Result(Test.UNRESOLVED, + "Test has more than one ALLOW_RETRIES lines") + + return {p.keyword: p.getValue() for p in keyword_parsers.values()} + + +def parseIntegratedTestScript(test, additional_parsers=[], + require_script=True): + """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test + script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES', + 'UNSUPPORTED' and 'ALLOW_RETRIES' information into the given test. + + If additional parsers are specified then the test is also scanned for the + keywords they specify and all matches are passed to the custom parser. + + If 'require_script' is False an empty script + may be returned. This can be used for test formats where the actual script + is optional or ignored. + """ + # Parse the test sources and extract test properties + parsed = _parseKeywords(test.getSourcePath(), additional_parsers, require_script) + script = parsed['RUN:'] or [] + test.xfails = parsed['XFAIL:'] or [] + test.requires = parsed['REQUIRES:'] or [] + test.unsupported = parsed['UNSUPPORTED:'] or [] + if parsed['ALLOW_RETRIES:']: + test.allowed_retries = parsed['ALLOW_RETRIES:'][0] + # Enforce REQUIRES: missing_required_features = test.getMissingRequiredFeatures() if missing_required_features: @@ -1458,14 +1483,6 @@ def parseIntegratedTestScript(test, additional_parsers=[], "Test does not support the following features " "and/or targets: %s" % msg) - # Handle ALLOW_RETRIES: - allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue() - if allowed_retries: - if len(allowed_retries) > 1: - return lit.Test.Result(Test.UNRESOLVED, - "Test has more than one ALLOW_RETRIES lines") - test.allowed_retries = allowed_retries[0] - # Enforce limit_to_features. if not test.isWithinFeatureLimits(): msg = ', '.join(test.config.limit_to_features) diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py index 06b1313cb8bb..6bbaa4203b47 100644 --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -157,6 +157,9 @@ def parse_args(): debug_group.add_argument("--show-tests", help="Show all discovered tests and exit", action="store_true") + debug_group.add_argument("--show-used-features", + help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit", + action="store_true") # LIT is special: environment variables override command line arguments. env_args = shlex.split(os.environ.get("LIT_OPTS", "")) diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py index d2958590aa8a..6c423167ff4c 100755 --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -4,6 +4,7 @@ lit - LLVM Integrated Tester. See lit.pod for more information. """ +import itertools import os import platform import sys @@ -47,6 +48,11 @@ def main(builtin_params={}): print_discovered(discovered_tests, opts.show_suites, opts.show_tests) sys.exit(0) + if opts.show_used_features: + features = set(itertools.chain.from_iterable(t.getUsedFeatures() for t in discovered_tests)) + print(' '.join(sorted(features))) + sys.exit(0) + # Command line overrides configuration for maxIndividualTestTime. if opts.maxIndividualTestTime is not None: # `not None` is important (default: 0) if opts.maxIndividualTestTime != lit_config.maxIndividualTestTime: @@ -127,7 +133,6 @@ def print_discovered(tests, show_suites, show_tests): tests.sort(key=lit.reports.by_suite_and_test_path) if show_suites: - import itertools tests_by_suite = itertools.groupby(tests, lambda t: t.suite) print('-- Test Suites --') for suite, test_iter in tests_by_suite: diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg b/llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg new file mode 100644 index 000000000000..7ee2154d2e19 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg @@ -0,0 +1,6 @@ +import lit.formats +config.name = 'show-used-features' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt b/llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt new file mode 100644 index 000000000000..1de0f7442a08 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt @@ -0,0 +1,4 @@ + +// REQUIRES: my-require-feature-2 || my-require-feature-3 +// UNSUPPORTED: my-unsupported-feature-2, my-unsupported-feature-3 +// XFAIL: my-xfail-feature-2, my-xfail-feature-3 diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/requires.txt b/llvm/utils/lit/tests/Inputs/show-used-features/requires.txt new file mode 100644 index 000000000000..3e550f4495fb --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-used-features/requires.txt @@ -0,0 +1,2 @@ + +// REQUIRES: my-require-feature-1 diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt b/llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt new file mode 100644 index 000000000000..b19582624abc --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt @@ -0,0 +1,2 @@ + +// UNSUPPORTED: my-unsupported-feature-1 diff --git a/llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt b/llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt new file mode 100644 index 000000000000..f8cde28ab4c7 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt @@ -0,0 +1,2 @@ + +// XFAIL: my-xfail-feature-1 diff --git a/llvm/utils/lit/tests/show-used-features.py b/llvm/utils/lit/tests/show-used-features.py new file mode 100644 index 000000000000..069ee0819670 --- /dev/null +++ b/llvm/utils/lit/tests/show-used-features.py @@ -0,0 +1,6 @@ +# Check that --show-used-features works correctly. +# +# RUN: %{lit} %{inputs}/show-used-features --show-used-features | FileCheck %s +# CHECK: my-require-feature-1 my-require-feature-2 my-require-feature-3 +# CHECK: my-unsupported-feature-1 my-unsupported-feature-2 my-unsupported-feature-3 +# CHECK: my-xfail-feature-1 my-xfail-feature-2 my-xfail-feature-3