[lit] Add an option to print all features used in tests

Lit test suites can tend to accumulate annotations that are not necessarily
relevant as time goes by, for example XFAILS on old compilers or platforms.
To help spot old annotations that can be cleaned up, it can be useful to
look at all features used inside a test suite.

This commit adds a new Lit option '--show-used-features' that prints all
the features used in XFAIL, REQUIRES and UNSUPPORTED of all tests that
are discovered.

Differential Revision: https://reviews.llvm.org/D78589

llvm/utils/lit/lit/BooleanExpression.py

@@ -79,9 +79,10 @@ class BooleanExpression:
             raise ValueError("expected: %s\nhave: %s" %
                              (self.quote(t), self.quote(self.token)))
 
-    def isIdentifier(self, t):
+    @staticmethod
-        if (t is BooleanExpression.END or t == '&&' or t == '||' or
+    def isIdentifier(token):
-            t == '!' or t == '(' or t == ')'):
+        if (token is BooleanExpression.END or token == '&&' or token == '||' or
+            token == '!' or token == '(' or token == ')'):
             return False
         return True
 
@@ -92,7 +93,7 @@ class BooleanExpression:
         elif self.accept('('):
             self.parseOR()
             self.expect(')')
-        elif not self.isIdentifier(self.token):
+        elif not BooleanExpression.isIdentifier(self.token):
             raise ValueError("expected: '!' or '(' or identifier\nhave: %s" %
                              self.quote(self.token))
         else:
@@ -191,7 +192,7 @@ class TestBooleanExpression(unittest.TestCase):
                            "actual error was:\n%s\n" +
                            "expected error was:\n%s\n") % (expr, e, error))
         except BaseException as e:
-            self.fail(("expression %r caused the wrong exception; actual " + 
+            self.fail(("expression %r caused the wrong exception; actual " +
                       "exception was: \n%r") % (expr, e))
 
     def test_errors(self):

llvm/utils/lit/lit/Test.py

@@ -1,3 +1,4 @@
+import itertools
 import os
 from xml.sax.saxutils import quoteattr
 from json import JSONEncoder
@@ -162,7 +163,7 @@ class Result(object):
         addMicroResult(microResult)
 
         Attach a micro-test result to the test result, with the given name and
-        result.  It is an error to attempt to attach a micro-test with the 
+        result.  It is an error to attempt to attach a micro-test with the
         same name multiple times.
 
         Each micro-test result must be an instance of the Result class.
@@ -359,6 +360,26 @@ class Test:
         except ValueError as e:
             raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e))
 
+    def getUsedFeatures(self):
+        """
+        getUsedFeatures() -> list of strings
+
+        Returns a list of all features appearing in XFAIL, UNSUPPORTED and
+        REQUIRES annotations for this test.
+        """
+        import lit.TestRunner
+        parsed = lit.TestRunner._parseKeywords(self.getSourcePath(), require_script=False)
+        feature_keywords = ('UNSUPPORTED:', 'REQUIRES:', 'XFAIL:')
+        boolean_expressions = itertools.chain.from_iterable(
+            parsed[k] or [] for k in feature_keywords
+        )
+        tokens = itertools.chain.from_iterable(
+            BooleanExpression.tokenize(expr) for expr in
+                boolean_expressions if expr != '*'
+        )
+        identifiers = set(filter(BooleanExpression.isIdentifier, tokens))
+        return identifiers
+
     def isEarlyTest(self):
         """
         isEarlyTest() -> bool

llvm/utils/lit/lit/TestRunner.py

@@ -1373,31 +1373,26 @@ class IntegratedTestKeywordParser(object):
                 BooleanExpression.evaluate(s, [])
         return output
 
-def parseIntegratedTestScript(test, additional_parsers=[],
-                              require_script=True):
-    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
-    script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
-    'UNSUPPORTED' and 'ALLOW_RETRIES' information.
 
-    If additional parsers are specified then the test is also scanned for the
+def _parseKeywords(sourcepath, additional_parsers=[],
-    keywords they specify and all matches are passed to the custom parser.
+                   require_script=True):
+    """_parseKeywords
 
-    If 'require_script' is False an empty script
+    Scan an LLVM/Clang style integrated test script and extract all the lines
-    may be returned. This can be used for test formats where the actual script
+    pertaining to a special parser. This includes 'RUN', 'XFAIL', 'REQUIRES',
-    is optional or ignored.
+    'UNSUPPORTED' and 'ALLOW_RETRIES', as well as other specified custom
+    parsers.
+
+    Returns a dictionary mapping each custom parser to its value after
+    parsing the test.
     """
-
     # Install the built-in keyword parsers.
     script = []
     builtin_parsers = [
-        IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND,
+        IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, initial_value=script),
-                                    initial_value=script),
+        IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR),
-        IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR,
+        IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR),
-                                    initial_value=test.xfails),
+        IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR),
-        IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
-                                    initial_value=test.requires),
-        IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
-                                    initial_value=test.unsupported),
         IntegratedTestKeywordParser('ALLOW_RETRIES:', ParserKind.INTEGER),
         IntegratedTestKeywordParser('END.', ParserKind.TAG)
     ]
@@ -1414,7 +1409,6 @@ def parseIntegratedTestScript(test, additional_parsers=[],
         keyword_parsers[parser.keyword] = parser
 
     # Collect the test lines from the script.
-    sourcepath = test.getSourcePath()
     for line_number, command_type, ln in \
             parseIntegratedTestScriptCommands(sourcepath,
                                               keyword_parsers.keys()):
@@ -1441,6 +1435,37 @@ def parseIntegratedTestScript(test, additional_parsers=[],
         if value and value[-1][-1] == '\\':
             raise ValueError("Test has unterminated %s lines (with '\\')" % key)
 
+    # Make sure there's at most one ALLOW_RETRIES: line
+    allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue()
+    if allowed_retries and len(allowed_retries) > 1:
+        return lit.Test.Result(Test.UNRESOLVED,
+                               "Test has more than one ALLOW_RETRIES lines")
+
+    return {p.keyword: p.getValue() for p in keyword_parsers.values()}
+
+
+def parseIntegratedTestScript(test, additional_parsers=[],
+                              require_script=True):
+    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
+    script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
+    'UNSUPPORTED' and 'ALLOW_RETRIES' information into the given test.
+
+    If additional parsers are specified then the test is also scanned for the
+    keywords they specify and all matches are passed to the custom parser.
+
+    If 'require_script' is False an empty script
+    may be returned. This can be used for test formats where the actual script
+    is optional or ignored.
+    """
+    # Parse the test sources and extract test properties
+    parsed = _parseKeywords(test.getSourcePath(), additional_parsers, require_script)
+    script = parsed['RUN:'] or []
+    test.xfails = parsed['XFAIL:'] or []
+    test.requires = parsed['REQUIRES:'] or []
+    test.unsupported = parsed['UNSUPPORTED:'] or []
+    if parsed['ALLOW_RETRIES:']:
+        test.allowed_retries = parsed['ALLOW_RETRIES:'][0]
+
     # Enforce REQUIRES:
     missing_required_features = test.getMissingRequiredFeatures()
     if missing_required_features:
@@ -1458,14 +1483,6 @@ def parseIntegratedTestScript(test, additional_parsers=[],
             "Test does not support the following features "
             "and/or targets: %s" % msg)
 
-    # Handle ALLOW_RETRIES:
-    allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue()
-    if allowed_retries:
-        if len(allowed_retries) > 1:
-            return lit.Test.Result(Test.UNRESOLVED,
-                                   "Test has more than one ALLOW_RETRIES lines")
-        test.allowed_retries = allowed_retries[0]
-
     # Enforce limit_to_features.
     if not test.isWithinFeatureLimits():
         msg = ', '.join(test.config.limit_to_features)

llvm/utils/lit/lit/cl_arguments.py

@@ -157,6 +157,9 @@ def parse_args():
     debug_group.add_argument("--show-tests",
             help="Show all discovered tests and exit",
             action="store_true")
+    debug_group.add_argument("--show-used-features",
+            help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
+            action="store_true")
 
     # LIT is special: environment variables override command line arguments.
     env_args = shlex.split(os.environ.get("LIT_OPTS", ""))

llvm/utils/lit/lit/main.py

@@ -4,6 +4,7 @@ lit - LLVM Integrated Tester.
 See lit.pod for more information.
 """
 
+import itertools
 import os
 import platform
 import sys
@@ -47,6 +48,11 @@ def main(builtin_params={}):
         print_discovered(discovered_tests, opts.show_suites, opts.show_tests)
         sys.exit(0)
 
+    if opts.show_used_features:
+        features = set(itertools.chain.from_iterable(t.getUsedFeatures() for t in discovered_tests))
+        print(' '.join(sorted(features)))
+        sys.exit(0)
+
     # Command line overrides configuration for maxIndividualTestTime.
     if opts.maxIndividualTestTime is not None:  # `not None` is important (default: 0)
         if opts.maxIndividualTestTime != lit_config.maxIndividualTestTime:
@@ -127,7 +133,6 @@ def print_discovered(tests, show_suites, show_tests):
     tests.sort(key=lit.reports.by_suite_and_test_path)
 
     if show_suites:
-        import itertools
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
         print('-- Test Suites --')
         for suite, test_iter in tests_by_suite:

llvm/utils/lit/tests/Inputs/show-used-features/lit.cfg

@@ -0,0 +1,6 @@
+import lit.formats
+config.name = 'show-used-features'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None

llvm/utils/lit/tests/Inputs/show-used-features/mixed.txt

@@ -0,0 +1,4 @@
+
+// REQUIRES: my-require-feature-2 || my-require-feature-3
+// UNSUPPORTED: my-unsupported-feature-2, my-unsupported-feature-3
+// XFAIL: my-xfail-feature-2, my-xfail-feature-3

llvm/utils/lit/tests/Inputs/show-used-features/requires.txt

@@ -0,0 +1,2 @@
+
+// REQUIRES: my-require-feature-1

llvm/utils/lit/tests/Inputs/show-used-features/unsupported.txt

@@ -0,0 +1,2 @@
+
+// UNSUPPORTED: my-unsupported-feature-1

llvm/utils/lit/tests/Inputs/show-used-features/xfail.txt

@@ -0,0 +1,2 @@
+
+// XFAIL: my-xfail-feature-1

llvm/utils/lit/tests/show-used-features.py

@@ -0,0 +1,6 @@
+# Check that --show-used-features works correctly.
+#
+# RUN: %{lit} %{inputs}/show-used-features --show-used-features | FileCheck %s
+# CHECK: my-require-feature-1 my-require-feature-2 my-require-feature-3
+# CHECK: my-unsupported-feature-1 my-unsupported-feature-2 my-unsupported-feature-3
+# CHECK: my-xfail-feature-1 my-xfail-feature-2 my-xfail-feature-3