From 8245a11be40b73b693ca22f0d7e1340800fa6199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20M=2E=20Rodr=C3=ADguez?= Date: Fri, 28 Dec 2018 15:09:14 +0100 Subject: [PATCH] Move QiskitTestCase to qiskit.test (#1616) * Move JobTestCase to test.python.ibmq * Move common testing functionality to qiskit.test Temporary commit for moving the files to qiskit.test. * Split qiskit.test.common into separate modules * Style and docstring adjustments * Add new Path.QASMS, revise existing ones * Update CHANGELOG --- .pylintrc | 3 +- CHANGELOG.rst | 2 + qiskit/test/__init__.py | 12 + qiskit/test/base.py | 123 ++++++ qiskit/test/decorators.py | 164 ++++++++ {test/python => qiskit/test}/http_recorder.py | 38 +- .../test/testing_options.py | 10 +- qiskit/test/utils.py | 82 ++++ .../circuit/test_circuit_load_from_qasm.py | 2 +- test/python/common.py | 386 +----------------- test/python/converters/test_ast_to_dag.py | 5 +- test/python/ibmq/jobtestcase.py | 33 ++ test/python/ibmq/test_ibmq_qobj.py | 3 +- test/python/ibmq/test_ibmqjob.py | 3 +- test/python/ibmq/test_ibmqjob_states.py | 2 +- test/python/simulators/test_qasm_simulator.py | 3 +- .../simulators/test_qasm_simulator_py.py | 4 +- .../simulators/test_unitary_simulator_py.py | 4 +- test/python/test_mapper.py | 6 +- test/python/test_qasm_parser.py | 8 +- test/python/test_schemas.py | 7 +- test/python/test_unroller.py | 4 +- 22 files changed, 475 insertions(+), 429 deletions(-) create mode 100644 qiskit/test/__init__.py create mode 100644 qiskit/test/base.py create mode 100644 qiskit/test/decorators.py rename {test/python => qiskit/test}/http_recorder.py (94%) rename test/python/_test_options.py => qiskit/test/testing_options.py (92%) create mode 100644 qiskit/test/utils.py create mode 100644 test/python/ibmq/jobtestcase.py diff --git a/.pylintrc b/.pylintrc index e4c82c9f05..732d17c296 100644 --- a/.pylintrc +++ b/.pylintrc @@ -115,7 +115,8 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme # pi = the PI constant # op = operation iterator # b = basis iterator -good-names=i,j,k,n,m,ex,v,w,x,y,z,Run,_,logger,q,r,qr,cr,qc,pi,op,b,ar,br +good-names=i,j,k,n,m,ex,v,w,x,y,z,Run,_,logger,q,r,qr,cr,qc,pi,op,b,ar,br, + __unittest # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,toto,tutu,tata diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc4223d7a1..e9e216f788 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,8 @@ Changed - The ``Exception`` subclasses have been moved to an ``.exceptions`` module within each package (for example, ``qiskit.exceptions.QiskitError``). (#1600). +- The ``QiskitTestCase`` and testing utilities are now included as part of + ``qiskit.test`` and thus available for third-party implementations. (#1616). Removed ------- diff --git a/qiskit/test/__init__.py b/qiskit/test/__init__.py new file mode 100644 index 0000000000..72ef4b6790 --- /dev/null +++ b/qiskit/test/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018, IBM. +# +# This source code is licensed under the Apache License, Version 2.0 found in +# the LICENSE.txt file in the root directory of this source tree. + +"""Functionality and helpers for testing Qiskit.""" + +from .base import QiskitTestCase +from .decorators import requires_cpp_simulator, requires_qe_access, slow_test +from .utils import Path diff --git a/qiskit/test/base.py b/qiskit/test/base.py new file mode 100644 index 0000000000..4a51a9f156 --- /dev/null +++ b/qiskit/test/base.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018, IBM. +# +# This source code is licensed under the Apache License, Version 2.0 found in +# the LICENSE.txt file in the root directory of this source tree. + +"""Base TestCases for the unit tests. + +Implementors of unit tests for Terra are encouraged to subclass +``QiskitTestCase`` in order to take advantage of utility functions (for example, +the environment variables for customizing different options), and the +decorators in the ``decorators`` package. +""" + +import inspect +import logging +import os +import unittest +from unittest.util import safe_repr + +from .utils import Path, _AssertNoLogsContext, setup_test_logging + + +__unittest = True # Allows shorter stack trace for .assertDictAlmostEqual + + +class QiskitTestCase(unittest.TestCase): + """Helper class that contains common functionality.""" + + @classmethod + def setUpClass(cls): + # Determines if the TestCase is using IBMQ credentials. + cls.using_ibmq_credentials = False + + # Set logging to file and stdout if the LOG_LEVEL envar is set. + cls.log = logging.getLogger(cls.__name__) + if os.getenv('LOG_LEVEL'): + filename = '%s.log' % os.path.splitext(inspect.getfile(cls))[0] + setup_test_logging(cls.log, os.getenv('LOG_LEVEL'), filename) + + def tearDown(self): + # Reset the default providers, as in practice they acts as a singleton + # due to importing the wrapper from qiskit. + from qiskit.providers.ibmq import IBMQ + from qiskit.providers.builtinsimulators import BasicAer + + IBMQ._accounts.clear() + BasicAer._backends = BasicAer._verify_backends() + + @staticmethod + def _get_resource_path(filename, path=Path.TEST): + """Get the absolute path to a resource. + + Args: + filename (string): filename or relative path to the resource. + path (Path): path used as relative to the filename. + Returns: + str: the absolute path to the resource. + """ + return os.path.normpath(os.path.join(path.value, filename)) + + def assertNoLogs(self, logger=None, level=None): + """Assert that no message is sent to the specified logger and level. + + Context manager to test that no message is sent to the specified + logger and level (the opposite of TestCase.assertLogs()). + """ + return _AssertNoLogsContext(self, logger, level) + + def assertDictAlmostEqual(self, dict1, dict2, delta=None, msg=None, + places=None, default_value=0): + """Assert two dictionaries with numeric values are almost equal. + + Fail if the two dictionaries are unequal as determined by + comparing that the difference between values with the same key are + not greater than delta (default 1e-8), or that difference rounded + to the given number of decimal places is not zero. If a key in one + dictionary is not in the other the default_value keyword argument + will be used for the missing value (default 0). If the two objects + compare equal then they will automatically compare almost equal. + + Args: + dict1 (dict): a dictionary. + dict2 (dict): a dictionary. + delta (number): threshold for comparison (defaults to 1e-8). + msg (str): return a custom message on failure. + places (int): number of decimal places for comparison. + default_value (number): default value for missing keys. + + Raises: + TypeError: raises TestCase failureException if the test fails. + """ + def valid_comparison(value): + if places is not None: + return round(value, places) == 0 + else: + return value < delta + + # Check arguments. + if dict1 == dict2: + return + if places is not None: + if delta is not None: + raise TypeError("specify delta or places not both") + msg_suffix = ' within %s places' % places + else: + delta = delta or 1e-8 + msg_suffix = ' within %s delta' % delta + + # Compare all keys in both dicts, populating error_msg. + error_msg = '' + for key in set(dict1.keys()) | set(dict2.keys()): + val1 = dict1.get(key, default_value) + val2 = dict2.get(key, default_value) + if not valid_comparison(abs(val1 - val2)): + error_msg += '(%s: %s != %s), ' % (safe_repr(key), + safe_repr(val1), + safe_repr(val2)) + + if error_msg: + msg = self._formatMessage(msg, error_msg[:-2] + msg_suffix) + raise self.failureException(msg) diff --git a/qiskit/test/decorators.py b/qiskit/test/decorators.py new file mode 100644 index 0000000000..e6427df388 --- /dev/null +++ b/qiskit/test/decorators.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018, IBM. +# +# This source code is licensed under the Apache License, Version 2.0 found in +# the LICENSE.txt file in the root directory of this source tree. + +"""Decorator for using with Qiskit unit tests.""" + +import functools +import os +import unittest + +from qiskit.providers.ibmq.credentials import Credentials, discover_credentials +from qiskit.providers.legacysimulators import QasmSimulator + +from .utils import Path +from .http_recorder import http_recorder +from .testing_options import get_test_options + + +def is_cpp_simulator_available(): + """Check if the C++ simulator can be instantiated. + + Returns: + bool: True if simulator executable is available + """ + try: + QasmSimulator() + except FileNotFoundError: + return False + return True + + +def requires_cpp_simulator(test_item): + """Decorator that skips test if C++ simulator is not available + + Args: + test_item (callable): function or class to be decorated. + + Returns: + callable: the decorated function. + """ + reason = 'C++ simulator not found, skipping test' + return unittest.skipIf(not is_cpp_simulator_available(), reason)(test_item) + + +def slow_test(func): + """Decorator that signals that the test takes minutes to run. + + Args: + func (callable): test function to be decorated. + + Returns: + callable: the decorated function. + """ + + @functools.wraps(func) + def _wrapper(*args, **kwargs): + skip_slow = not TEST_OPTIONS['run_slow'] + if skip_slow: + raise unittest.SkipTest('Skipping slow tests') + + return func(*args, **kwargs) + + return _wrapper + + +def _get_credentials(test_object, test_options): + """Finds the credentials for a specific test and options. + + Args: + test_object (QiskitTestCase): The test object asking for credentials + test_options (dict): Options after QISKIT_TESTS was parsed by get_test_options. + + Returns: + Credentials: set of credentials + + Raises: + Exception: When the credential could not be set and they are needed for that set of options + """ + + dummy_credentials = Credentials('dummyapiusersloginWithTokenid01', + 'https://quantumexperience.ng.bluemix.net/api') + + if test_options['mock_online']: + return dummy_credentials + + if os.getenv('USE_ALTERNATE_ENV_CREDENTIALS', ''): + # Special case: instead of using the standard credentials mechanism, + # load them from different environment variables. This assumes they + # will always be in place, as is used by the Travis setup. + return Credentials(os.getenv('IBMQ_TOKEN'), os.getenv('IBMQ_URL')) + else: + # Attempt to read the standard credentials. + discovered_credentials = discover_credentials() + + if discovered_credentials: + # Decide which credentials to use for testing. + if len(discovered_credentials) > 1: + try: + # Attempt to use QE credentials. + return discovered_credentials[dummy_credentials.unique_id()] + except KeyError: + pass + + # Use the first available credentials. + return list(discovered_credentials.values())[0] + + # No user credentials were found. + if test_options['rec']: + raise Exception('Could not locate valid credentials. You need them for recording ' + 'tests against the remote API.') + + test_object.log.warning("No user credentials were detected. Running with mocked data.") + test_options['mock_online'] = True + return dummy_credentials + + +def requires_qe_access(func): + """Decorator that signals that the test uses the online API: + + It involves: + * determines if the test should be skipped by checking environment + variables. + * if the `USE_ALTERNATE_ENV_CREDENTIALS` environment variable is + set, it reads the credentials from an alternative set of environment + variables. + * if the test is not skipped, it reads `qe_token` and `qe_url` from + `Qconfig.py`, environment variables or qiskitrc. + * if the test is not skipped, it appends `qe_token` and `qe_url` as + arguments to the test function. + + Args: + func (callable): test function to be decorated. + + Returns: + callable: the decorated function. + """ + + @functools.wraps(func) + def _wrapper(self, *args, **kwargs): + if TEST_OPTIONS['skip_online']: + raise unittest.SkipTest('Skipping online tests') + + credentials = _get_credentials(self, TEST_OPTIONS) + self.using_ibmq_credentials = credentials.is_ibmq() + kwargs.update({'qe_token': credentials.token, + 'qe_url': credentials.url}) + + decorated_func = func + if TEST_OPTIONS['rec'] or TEST_OPTIONS['mock_online']: + # For recording or for replaying existing cassettes, the test + # should be decorated with @use_cassette. + vcr_mode = 'new_episodes' if TEST_OPTIONS['rec'] else 'none' + decorated_func = http_recorder( + vcr_mode, Path.CASSETTES.value).use_cassette()(decorated_func) + + return decorated_func(self, *args, **kwargs) + + return _wrapper + + +TEST_OPTIONS = get_test_options() diff --git a/test/python/http_recorder.py b/qiskit/test/http_recorder.py similarity index 94% rename from test/python/http_recorder.py rename to qiskit/test/http_recorder.py index 1b453ac343..2bece15494 100644 --- a/test/python/http_recorder.py +++ b/qiskit/test/http_recorder.py @@ -14,7 +14,8 @@ from vcr import VCR class IdRemoverPersister(FilesystemPersister): - """ + """VCR Persister for Qiskit. + IdRemoverPersister is a VCR persister. This is, it implements a way to save and load cassettes. This persister in particular inherits load_cassette from FilesystemPersister (basically, it loads a standard cassette in the standard way from the file system). On the saving side, it @@ -23,8 +24,7 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def get_responses_with(string_to_find, cassette_dict): - """ - Filters the requests from cassette_dict + """Filters the requests from cassette_dict Args: string_to_find (str): request path @@ -39,8 +39,7 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def get_new_id(field, path, id_tracker, type_=str): - """ - Creates a new dummy id (or value) for replacing an existing id (or value). + """Creates a new dummy id (or value) for replacing an existing id (or value). Args: field (str): field name is used, in same cases, to create a dummy value. @@ -62,8 +61,8 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def get_matching_dicts(data_dict, map_list): - """ - Find subdicts that are described in map_list. + """Find subdicts that are described in map_list. + Args: data_dict (dict): in which the map_list is going to be searched. map_list (list): the list of nested keys to find in the data_dict @@ -88,7 +87,8 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def remove_id_in_a_json(jsonobj, field, path, id_tracker): - """ + """Replaces ids with dummy values in a json. + Replaces in jsonobj (in-place) the field with dummy value (which is constructed with id_tracker, if it was already replaced, or path, if it needs to be created). @@ -110,7 +110,8 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def remove_ids_in_a_response(response, fields, path, id_tracker): - """ + """Replaces ids with dummy values in a response. + Replaces in response (in-place) the fields with dummy values (which is constructed with id_tracker, if it was already replaced, or path, if it needs to be created). @@ -127,7 +128,8 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def remove_ids(ids2remove, cassette_dict): - """ + """Replaces ids with dummy values in a cassette. + Replaces in cassette_dict (in-place) the fields defined by ids2remove with dummy values. Internally, it used a map (id_tracker) between real values and dummy values to keep consistency during the renaming. @@ -149,7 +151,8 @@ class IdRemoverPersister(FilesystemPersister): @staticmethod def save_cassette(cassette_path, cassette_dict, serializer): - """ + """Extends FilesystemPersister.save_cassette + Extends FilesystemPersister.save_cassette. Replaces particular values (defined by ids2remove) which are replaced by a dummy value. The full manipulation is in cassette_dict, before saving it using FilesystemPersister.save_cassette @@ -181,8 +184,7 @@ class IdRemoverPersister(FilesystemPersister): def http_recorder(vcr_mode, cassette_dir): - """ - Creates a VCR object in vcr_mode mode. + """Creates a VCR object in vcr_mode mode. Args: vcr_mode (string): the parameter for record_mode. @@ -213,8 +215,7 @@ def http_recorder(vcr_mode, cassette_dir): def _purge_headers_cb(headers): - """ - Remove headers from the response. + """Remove headers from the response. Args: headers (list): headers to remove from the response @@ -222,7 +223,6 @@ def _purge_headers_cb(headers): Returns: callable: for been used in before_record_response VCR constructor. """ - header_list = [] for item in headers: if not isinstance(item, tuple): @@ -230,8 +230,7 @@ def _purge_headers_cb(headers): header_list.append(item[0:2]) # ensure the tuple is a pair def before_record_response_cb(response): - """ - Purge headers from response. + """Purge headers from response. Args: response (dict): a VCR response @@ -251,7 +250,8 @@ def _purge_headers_cb(headers): def _unordered_query_matcher(request1, request2): - """ + """A VCR matcher that ignores the order of values in the query string. + A VCR matcher (a la VCR.matcher) that ignores the order of the values in the query string. Useful for filter params, for example. diff --git a/test/python/_test_options.py b/qiskit/test/testing_options.py similarity index 92% rename from test/python/_test_options.py rename to qiskit/test/testing_options.py index 424ad90873..750c0ac7cb 100644 --- a/test/python/_test_options.py +++ b/qiskit/test/testing_options.py @@ -30,8 +30,8 @@ def get_test_options(option_var='QISKIT_TESTS'): } def turn_false(option): - """ - Turn an option to False + """Turn an option to False. + Args: option (str): Turns defaults[option] to False @@ -49,8 +49,7 @@ def get_test_options(option_var='QISKIT_TESTS'): } def set_flag(flag_): - """ - Set the flag to True and flip all the flags that need to be rewritten. + """Set the flag to True and flip all the flags that need to be rewritten. Args: flag_ (str): Option to be True @@ -74,7 +73,8 @@ def get_test_options(option_var='QISKIT_TESTS'): def _is_ci_fork_pull_request(): - """ + """Check if the tests are being run in a CI environment from a PR. + Check if the tests are being run in a CI environment and if it is a pull request. diff --git a/qiskit/test/utils.py b/qiskit/test/utils.py new file mode 100644 index 0000000000..db2557c724 --- /dev/null +++ b/qiskit/test/utils.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018, IBM. +# +# This source code is licensed under the Apache License, Version 2.0 found in +# the LICENSE.txt file in the root directory of this source tree. + +"""Utils for using with Qiskit unit tests.""" + +import logging +import os +import unittest +from enum import Enum + +from qiskit import __path__ as qiskit_path + + +class Path(Enum): + """Helper with paths commonly used during the tests.""" + + # Main SDK path: qiskit/ + SDK = qiskit_path[0] + # test.python path: qiskit/test/python/ + TEST = os.path.normpath(os.path.join(SDK, '..', 'test', 'python')) + # Examples path: examples/ + EXAMPLES = os.path.normpath(os.path.join(SDK, '..', 'examples')) + # Schemas path: qiskit/schemas + SCHEMAS = os.path.normpath(os.path.join(SDK, 'schemas')) + # VCR cassettes path: qiskit/test/cassettes/ + CASSETTES = os.path.normpath(os.path.join(TEST, '..', 'cassettes')) + # Sample QASMs path: qiskit/test/python/qasm + QASMS = os.path.normpath(os.path.join(TEST, 'qasm')) + + +def setup_test_logging(logger, log_level, filename): + """Set logging to file and stdout for a logger. + + Args: + logger (Logger): logger object to be updated. + log_level (str): logging level. + filename (str): name of the output file. + """ + # Set up formatter. + log_fmt = ('{}.%(funcName)s:%(levelname)s:%(asctime)s:' + ' %(message)s'.format(logger.name)) + formatter = logging.Formatter(log_fmt) + + # Set up the file handler. + file_handler = logging.FileHandler(filename) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Set the logging level from the environment variable, defaulting + # to INFO if it is not a valid level. + level = logging._nameToLevel.get(log_level, logging.INFO) + logger.setLevel(level) + + +class _AssertNoLogsContext(unittest.case._AssertLogsContext): + """A context manager used to implement TestCase.assertNoLogs().""" + + # pylint: disable=inconsistent-return-statements + def __exit__(self, exc_type, exc_value, tb): + """ + This is a modified version of TestCase._AssertLogsContext.__exit__(...) + """ + self.logger.handlers = self.old_handlers + self.logger.propagate = self.old_propagate + self.logger.setLevel(self.old_level) + if exc_type is not None: + # let unexpected exceptions pass through + return False + + if self.watcher.records: + msg = 'logs of level {} or higher triggered on {}:\n'.format( + logging.getLevelName(self.level), self.logger.name) + for record in self.watcher.records: + msg += 'logger %s %s:%i: %s\n' % (record.name, record.pathname, + record.lineno, + record.getMessage()) + + self._raiseFailure(msg) diff --git a/test/python/circuit/test_circuit_load_from_qasm.py b/test/python/circuit/test_circuit_load_from_qasm.py index eec6e1ea98..6f447f3ceb 100644 --- a/test/python/circuit/test_circuit_load_from_qasm.py +++ b/test/python/circuit/test_circuit_load_from_qasm.py @@ -120,7 +120,7 @@ class LoadFromQasmTest(QiskitTestCase): def test_qasm_example_file(self): """Loads qasm/example.qasm. """ - qasm_filename = self._get_resource_path('qasm/example.qasm') + qasm_filename = self._get_resource_path('example.qasm', Path.QASMS) expected_circuit = QuantumCircuit.from_qasm_str('\n'.join(["OPENQASM 2.0;", "include \"qelib1.inc\";", "qreg q[3];", diff --git a/test/python/common.py b/test/python/common.py index 8f84b74148..d587d3e5c4 100644 --- a/test/python/common.py +++ b/test/python/common.py @@ -7,384 +7,12 @@ """Shared functionality and helpers for the unit tests.""" -from enum import Enum -import functools -import inspect -import logging -import os -import time -import unittest -from unittest.util import safe_repr -from qiskit import __path__ as qiskit_path -from qiskit.providers import JobStatus -from qiskit.providers.legacysimulators import QasmSimulator -from qiskit.providers.ibmq.credentials import discover_credentials, Credentials +# pylint: disable=unused-import -from .http_recorder import http_recorder -from ._test_options import get_test_options +# TODO: once all the tests in test/python import from qiskit.test, this file +# can be safely removed. - -# Allows shorter stack trace for .assertDictAlmostEqual -__unittest = True # pylint: disable=invalid-name - - -class Path(Enum): - """Helper with paths commonly used during the tests.""" - # Main SDK path: qiskit/ - SDK = qiskit_path[0] - # test.python path: qiskit/test/python/ - TEST = os.path.dirname(__file__) - # Examples path: examples/ - EXAMPLES = os.path.join(SDK, '..', 'examples') - # Schemas path: qiskit/schemas - SCHEMAS = os.path.join(SDK, 'schemas') - # VCR cassettes path: qiskit/test/cassettes/ - CASSETTES = os.path.join(TEST, '..', 'cassettes') - - -class QiskitTestCase(unittest.TestCase): - """Helper class that contains common functionality.""" - - @classmethod - def setUpClass(cls): - cls.moduleName = os.path.splitext(inspect.getfile(cls))[0] - cls.log = logging.getLogger(cls.__name__) - # Determines if the TestCase is using IBMQ credentials. - cls.using_ibmq_credentials = False - - # Set logging to file and stdout if the LOG_LEVEL environment variable - # is set. - if os.getenv('LOG_LEVEL'): - # Set up formatter. - log_fmt = ('{}.%(funcName)s:%(levelname)s:%(asctime)s:' - ' %(message)s'.format(cls.__name__)) - formatter = logging.Formatter(log_fmt) - - # Set up the file handler. - log_file_name = '%s.log' % cls.moduleName - file_handler = logging.FileHandler(log_file_name) - file_handler.setFormatter(formatter) - cls.log.addHandler(file_handler) - - # Set the logging level from the environment variable, defaulting - # to INFO if it is not a valid level. - level = logging._nameToLevel.get(os.getenv('LOG_LEVEL'), - logging.INFO) - cls.log.setLevel(level) - cls.log.debug("QISKIT_TESTS: %s", str(TEST_OPTIONS)) - - def tearDown(self): - # Reset the default providers, as in practice they acts as a singleton - # due to importing the wrapper from qiskit. - from qiskit.providers.ibmq import IBMQ - from qiskit.providers.builtinsimulators import BasicAer - - IBMQ._accounts.clear() - BasicAer._backends = BasicAer._verify_backends() - - @staticmethod - def _get_resource_path(filename, path=Path.TEST): - """ Get the absolute path to a resource. - - Args: - filename (string): filename or relative path to the resource. - path (Path): path used as relative to the filename. - Returns: - str: the absolute path to the resource. - """ - return os.path.normpath(os.path.join(path.value, filename)) - - def assertNoLogs(self, logger=None, level=None): - """ - Context manager to test that no message is sent to the specified - logger and level (the opposite of TestCase.assertLogs()). - """ - return _AssertNoLogsContext(self, logger, level) - - def assertDictAlmostEqual(self, dict1, dict2, delta=None, msg=None, - places=None, default_value=0): - """ - Assert two dictionaries with numeric values are almost equal. - - Fail if the two dictionaries are unequal as determined by - comparing that the difference between values with the same key are - not greater than delta (default 1e-8), or that difference rounded - to the given number of decimal places is not zero. If a key in one - dictionary is not in the other the default_value keyword argument - will be used for the missing value (default 0). If the two objects - compare equal then they will automatically compare almost equal. - - Args: - dict1 (dict): a dictionary. - dict2 (dict): a dictionary. - delta (number): threshold for comparison (defaults to 1e-8). - msg (str): return a custom message on failure. - places (int): number of decimal places for comparison. - default_value (number): default value for missing keys. - - Raises: - TypeError: raises TestCase failureException if the test fails. - """ - if dict1 == dict2: - # Shortcut - return - if delta is not None and places is not None: - raise TypeError("specify delta or places not both") - - if places is not None: - success = True - standard_msg = '' - # check value for keys in target - keys1 = set(dict1.keys()) - for key in keys1: - val1 = dict1.get(key, default_value) - val2 = dict2.get(key, default_value) - if round(abs(val1 - val2), places) != 0: - success = False - standard_msg += '(%s: %s != %s), ' % (safe_repr(key), - safe_repr(val1), - safe_repr(val2)) - # check values for keys in counts, not in target - keys2 = set(dict2.keys()) - keys1 - for key in keys2: - val1 = dict1.get(key, default_value) - val2 = dict2.get(key, default_value) - if round(abs(val1 - val2), places) != 0: - success = False - standard_msg += '(%s: %s != %s), ' % (safe_repr(key), - safe_repr(val1), - safe_repr(val2)) - if success is True: - return - standard_msg = standard_msg[:-2] + ' within %s places' % places - - else: - if delta is None: - delta = 1e-8 # default delta value - success = True - standard_msg = '' - # check value for keys in target - keys1 = set(dict1.keys()) - for key in keys1: - val1 = dict1.get(key, default_value) - val2 = dict2.get(key, default_value) - if abs(val1 - val2) > delta: - success = False - standard_msg += '(%s: %s != %s), ' % (safe_repr(key), - safe_repr(val1), - safe_repr(val2)) - # check values for keys in counts, not in target - keys2 = set(dict2.keys()) - keys1 - for key in keys2: - val1 = dict1.get(key, default_value) - val2 = dict2.get(key, default_value) - if abs(val1 - val2) > delta: - success = False - standard_msg += '(%s: %s != %s), ' % (safe_repr(key), - safe_repr(val1), - safe_repr(val2)) - if success is True: - return - standard_msg = standard_msg[:-2] + ' within %s delta' % delta - - msg = self._formatMessage(msg, standard_msg) - raise self.failureException(msg) - - -class JobTestCase(QiskitTestCase): - """Include common functionality when testing jobs.""" - - def wait_for_initialization(self, job, timeout=1): - """Waits until the job progress from `INITIALIZING` to a different - status. - """ - waited = 0 - wait = 0.1 - while job.status() is JobStatus.INITIALIZING: - time.sleep(wait) - waited += wait - if waited > timeout: - self.fail( - msg="The JOB is still initializing after timeout ({}s)" - .format(timeout) - ) - - -class _AssertNoLogsContext(unittest.case._AssertLogsContext): - """A context manager used to implement TestCase.assertNoLogs().""" - - # pylint: disable=inconsistent-return-statements - def __exit__(self, exc_type, exc_value, tb): - """ - This is a modified version of TestCase._AssertLogsContext.__exit__(...) - """ - self.logger.handlers = self.old_handlers - self.logger.propagate = self.old_propagate - self.logger.setLevel(self.old_level) - if exc_type is not None: - # let unexpected exceptions pass through - return False - - if self.watcher.records: - msg = 'logs of level {} or higher triggered on {}:\n'.format( - logging.getLevelName(self.level), self.logger.name) - for record in self.watcher.records: - msg += 'logger %s %s:%i: %s\n' % (record.name, record.pathname, - record.lineno, - record.getMessage()) - - self._raiseFailure(msg) - - -def slow_test(func): - """ - Decorator that signals that the test takes minutes to run. - - Args: - func (callable): test function to be decorated. - - Returns: - callable: the decorated function. - """ - - @functools.wraps(func) - def _wrapper(*args, **kwargs): - skip_slow = not TEST_OPTIONS['run_slow'] - if skip_slow: - raise unittest.SkipTest('Skipping slow tests') - - return func(*args, **kwargs) - - return _wrapper - - -def _get_credentials(test_object, test_options): - """ - Finds the credentials for a specific test and options. - - Args: - test_object (QiskitTestCase): The test object asking for credentials - test_options (dict): Options after QISKIT_TESTS was parsed by get_test_options. - - Returns: - Credentials: set of credentials - - Raises: - Exception: When the credential could not be set and they are needed for that set of options - """ - - dummy_credentials = Credentials('dummyapiusersloginWithTokenid01', - 'https://quantumexperience.ng.bluemix.net/api') - - if test_options['mock_online']: - return dummy_credentials - - if os.getenv('USE_ALTERNATE_ENV_CREDENTIALS', ''): - # Special case: instead of using the standard credentials mechanism, - # load them from different environment variables. This assumes they - # will always be in place, as is used by the Travis setup. - return Credentials(os.getenv('IBMQ_TOKEN'), os.getenv('IBMQ_URL')) - else: - # Attempt to read the standard credentials. - discovered_credentials = discover_credentials() - - if discovered_credentials: - # Decide which credentials to use for testing. - if len(discovered_credentials) > 1: - try: - # Attempt to use QE credentials. - return discovered_credentials[dummy_credentials.unique_id()] - except KeyError: - pass - - # Use the first available credentials. - return list(discovered_credentials.values())[0] - - # No user credentials were found. - if test_options['rec']: - raise Exception('Could not locate valid credentials. You need them for recording ' - 'tests against the remote API.') - - test_object.log.warning("No user credentials were detected. Running with mocked data.") - test_options['mock_online'] = True - return dummy_credentials - - -def is_cpp_simulator_available(): - """ - Check if executable for C++ simulator is available in the expected - location. - - Returns: - bool: True if simulator executable is available - """ - try: - QasmSimulator() - except FileNotFoundError: - return False - return True - - -def requires_cpp_simulator(test_item): - """ - Decorator that skips test if C++ simulator is not available - - Args: - test_item (callable): function or class to be decorated. - - Returns: - callable: the decorated function. - """ - reason = 'C++ simulator not found, skipping test' - return unittest.skipIf(not is_cpp_simulator_available(), reason)(test_item) - - -def requires_qe_access(func): - """ - Decorator that signals that the test uses the online API: - * determines if the test should be skipped by checking environment - variables. - * if the `USE_ALTERNATE_ENV_CREDENTIALS` environment variable is - set, it reads the credentials from an alternative set of environment - variables. - * if the test is not skipped, it reads `qe_token` and `qe_url` from - `Qconfig.py`, environment variables or qiskitrc. - * if the test is not skipped, it appends `qe_token` and `qe_url` as - arguments to the test function. - Args: - func (callable): test function to be decorated. - - Returns: - callable: the decorated function. - """ - - @functools.wraps(func) - def _wrapper(self, *args, **kwargs): - if TEST_OPTIONS['skip_online']: - raise unittest.SkipTest('Skipping online tests') - - credentials = _get_credentials(self, TEST_OPTIONS) - self.using_ibmq_credentials = credentials.is_ibmq() - kwargs.update({'qe_token': credentials.token, - 'qe_url': credentials.url}) - - decorated_func = func - if TEST_OPTIONS['rec'] or TEST_OPTIONS['mock_online']: - # For recording or for replaying existing cassettes, the test should be decorated with - # use_cassette. - decorated_func = VCR.use_cassette()(decorated_func) - - return decorated_func(self, *args, **kwargs) - - return _wrapper - - -def _get_http_recorder(test_options): - vcr_mode = 'none' - if test_options['rec']: - vcr_mode = 'new_episodes' - return http_recorder(vcr_mode, Path.CASSETTES.value) - - -TEST_OPTIONS = get_test_options() -VCR = _get_http_recorder(TEST_OPTIONS) +from qiskit.test.base import QiskitTestCase +from qiskit.test.decorators import (requires_cpp_simulator, requires_qe_access, + slow_test, is_cpp_simulator_available) +from qiskit.test.utils import Path diff --git a/test/python/converters/test_ast_to_dag.py b/test/python/converters/test_ast_to_dag.py index 459a69ac5f..02f09fbc83 100644 --- a/test/python/converters/test_ast_to_dag.py +++ b/test/python/converters/test_ast_to_dag.py @@ -15,7 +15,7 @@ from qiskit.converters import ast_to_dag, circuit_to_dag from qiskit import QuantumRegister, ClassicalRegister, QuantumCircuit from qiskit import qasm -from ..common import QiskitTestCase +from ..common import QiskitTestCase, Path class TestAstToDag(QiskitTestCase): @@ -31,7 +31,8 @@ class TestAstToDag(QiskitTestCase): def test_from_ast_to_dag(self): """Test Unroller.execute()""" - ast = qasm.Qasm(filename=self._get_resource_path('qasm/example.qasm')).parse() + ast = qasm.Qasm(filename=self._get_resource_path('example.qasm', + Path.QASMS)).parse() dag_circuit = ast_to_dag(ast) expected_result = """\ OPENQASM 2.0; diff --git a/test/python/ibmq/jobtestcase.py b/test/python/ibmq/jobtestcase.py new file mode 100644 index 0000000000..9e393f4b91 --- /dev/null +++ b/test/python/ibmq/jobtestcase.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018, IBM. +# +# This source code is licensed under the Apache License, Version 2.0 found in +# the LICENSE.txt file in the root directory of this source tree. + +"""Custom TestCase for Jobs.""" + + +import time + +from qiskit.providers import JobStatus +from ..common import QiskitTestCase + + +class JobTestCase(QiskitTestCase): + """Include common functionality when testing jobs.""" + + def wait_for_initialization(self, job, timeout=1): + """Waits until the job progress from `INITIALIZING` to a different + status. + """ + waited = 0 + wait = 0.1 + while job.status() is JobStatus.INITIALIZING: + time.sleep(wait) + waited += wait + if waited > timeout: + self.fail( + msg="The JOB is still initializing after timeout ({}s)" + .format(timeout) + ) diff --git a/test/python/ibmq/test_ibmq_qobj.py b/test/python/ibmq/test_ibmq_qobj.py index 50fda4a75b..7155b59ae7 100644 --- a/test/python/ibmq/test_ibmq_qobj.py +++ b/test/python/ibmq/test_ibmq_qobj.py @@ -17,7 +17,8 @@ from qiskit import (ClassicalRegister, QuantumCircuit, QuantumRegister, compile) from qiskit import IBMQ, BasicAer from qiskit.qasm import pi -from ..common import requires_qe_access, JobTestCase, slow_test +from .jobtestcase import JobTestCase +from ..common import requires_qe_access, slow_test class TestIBMQQobj(JobTestCase): diff --git a/test/python/ibmq/test_ibmqjob.py b/test/python/ibmq/test_ibmqjob.py index 387261954f..927a3692f6 100644 --- a/test/python/ibmq/test_ibmqjob.py +++ b/test/python/ibmq/test_ibmqjob.py @@ -26,7 +26,8 @@ from qiskit.providers import JobStatus, JobError from qiskit.providers.ibmq import least_busy from qiskit.providers.ibmq.exceptions import IBMQBackendError from qiskit.providers.ibmq.ibmqjob import IBMQJob -from ..common import requires_qe_access, JobTestCase, slow_test +from .jobtestcase import JobTestCase +from ..common import requires_qe_access, slow_test class TestIBMQJob(JobTestCase): diff --git a/test/python/ibmq/test_ibmqjob_states.py b/test/python/ibmq/test_ibmqjob_states.py index 4a024b554d..2cd1ca8c4d 100644 --- a/test/python/ibmq/test_ibmqjob_states.py +++ b/test/python/ibmq/test_ibmqjob_states.py @@ -17,7 +17,7 @@ from qiskit.providers.jobstatus import JobStatus from qiskit.providers.ibmq.ibmqjob import IBMQJobPreQobj, IBMQJob, API_FINAL_STATES from qiskit.providers.ibmq.api import ApiError from qiskit.providers import JobError, JobTimeoutError -from ..common import JobTestCase +from .jobtestcase import JobTestCase from .._mockutils import new_fake_qobj, FakeBackend diff --git a/test/python/simulators/test_qasm_simulator.py b/test/python/simulators/test_qasm_simulator.py index 09a13bc1b6..da3fbbc757 100644 --- a/test/python/simulators/test_qasm_simulator.py +++ b/test/python/simulators/test_qasm_simulator.py @@ -35,8 +35,7 @@ class TestLegacyQasmSimulator(QiskitTestCase): self.backend = QasmSimulator() qasm_file_name = 'example.qasm' - qasm_file_path = self._get_resource_path( - 'qasm/' + qasm_file_name, Path.TEST) + qasm_file_path = self._get_resource_path(qasm_file_name, Path.QASMS) self.qc1 = QuantumCircuit.from_qasm_file(qasm_file_path) qr = QuantumRegister(2, 'q') diff --git a/test/python/simulators/test_qasm_simulator_py.py b/test/python/simulators/test_qasm_simulator_py.py index 0f97c24eea..c2591af75c 100644 --- a/test/python/simulators/test_qasm_simulator_py.py +++ b/test/python/simulators/test_qasm_simulator_py.py @@ -14,7 +14,7 @@ from qiskit import ClassicalRegister, QuantumRegister, QuantumCircuit from qiskit import compile from qiskit.providers.builtinsimulators.qasm_simulator import QasmSimulatorPy -from ..common import QiskitTestCase +from ..common import QiskitTestCase, Path class TestBuiltinQasmSimulatorPy(QiskitTestCase): @@ -23,7 +23,7 @@ class TestBuiltinQasmSimulatorPy(QiskitTestCase): def setUp(self): self.seed = 88 self.backend = QasmSimulatorPy() - qasm_filename = self._get_resource_path('qasm/example.qasm') + qasm_filename = self._get_resource_path('example.qasm', Path.QASMS) compiled_circuit = QuantumCircuit.from_qasm_file(qasm_filename) compiled_circuit.name = 'test' self.qobj = compile(compiled_circuit, backend=self.backend) diff --git a/test/python/simulators/test_unitary_simulator_py.py b/test/python/simulators/test_unitary_simulator_py.py index 61b375f066..608dae3ada 100644 --- a/test/python/simulators/test_unitary_simulator_py.py +++ b/test/python/simulators/test_unitary_simulator_py.py @@ -14,7 +14,7 @@ import numpy as np from qiskit import ClassicalRegister, QuantumRegister, QuantumCircuit from qiskit import compile from qiskit.providers.builtinsimulators.unitary_simulator import UnitarySimulatorPy -from ..common import QiskitTestCase +from ..common import QiskitTestCase, Path class BuiltinUnitarySimulatorPyTest(QiskitTestCase): @@ -22,7 +22,7 @@ class BuiltinUnitarySimulatorPyTest(QiskitTestCase): def setUp(self): self.seed = 88 - self.qasm_filename = self._get_resource_path('qasm/example.qasm') + self.qasm_filename = self._get_resource_path('example.qasm', Path.QASMS) self.backend = UnitarySimulatorPy() def test_unitary_simulator_py(self): diff --git a/test/python/test_mapper.py b/test/python/test_mapper.py index 8d6d762606..ac95ae9364 100644 --- a/test/python/test_mapper.py +++ b/test/python/test_mapper.py @@ -22,7 +22,7 @@ from qiskit.mapper._compiling import two_qubit_kak from qiskit.tools.qi.qi import random_unitary_matrix from qiskit.mapper._mapping import MapperError from qiskit.converters import circuit_to_dag -from .common import QiskitTestCase +from .common import QiskitTestCase, Path class FakeQX4BackEnd: @@ -159,7 +159,7 @@ class TestMapper(QiskitTestCase): def test_random_parameter_circuit(self): """Run a circuit with randomly generated parameters.""" circ = QuantumCircuit.from_qasm_file( - self._get_resource_path('qasm/random_n5_d5.qasm')) + self._get_resource_path('random_n5_d5.qasm', Path.QASMS)) coupling_map = [[0, 1], [1, 2], [2, 3], [3, 4]] shots = 1024 qobj = execute(circ, backend=self.backend, @@ -258,7 +258,7 @@ class TestMapper(QiskitTestCase): backend = FakeQX5BackEnd() cmap = backend.configuration().coupling_map circ = QuantumCircuit.from_qasm_file( - self._get_resource_path('qasm/move_measurements.qasm')) + self._get_resource_path('move_measurements.qasm', Path.QASMS)) dag_circuit = circuit_to_dag(circ) lay = {('qa', 0): ('q', 0), ('qa', 1): ('q', 1), ('qb', 0): ('q', 15), diff --git a/test/python/test_qasm_parser.py b/test/python/test_qasm_parser.py index 9200f1589a..b8af03c4ca 100644 --- a/test/python/test_qasm_parser.py +++ b/test/python/test_qasm_parser.py @@ -15,7 +15,7 @@ import ply from qiskit.qasm import Qasm, QasmError from qiskit.qasm._node._node import Node -from .common import QiskitTestCase +from .common import QiskitTestCase, Path def parse(file_path, prec=15): @@ -31,11 +31,11 @@ def parse(file_path, prec=15): class TestParser(QiskitTestCase): """QasmParser""" def setUp(self): - self.qasm_file_path = self._get_resource_path('qasm/example.qasm') + self.qasm_file_path = self._get_resource_path('example.qasm', Path.QASMS) self.qasm_file_path_fail = self._get_resource_path( - 'qasm/example_fail.qasm') + 'example_fail.qasm', Path.QASMS) self.qasm_file_path_if = self._get_resource_path( - 'qasm/example_if.qasm') + 'example_if.qasm', Path.QASMS) def test_parser(self): """should return a correct response for a valid circuit.""" diff --git a/test/python/test_schemas.py b/test/python/test_schemas.py index 6400ff02e2..911bdbf292 100644 --- a/test/python/test_schemas.py +++ b/test/python/test_schemas.py @@ -16,11 +16,10 @@ from marshmallow import ValidationError from qiskit.qobj._schema_validation import (validate_json_against_schema, _get_validator) -from qiskit import __path__ as qiskit_path from qiskit.providers.models import (BackendConfiguration, BackendProperties, BackendStatus, JobStatus) from qiskit.result import Result -from .common import QiskitTestCase +from .common import QiskitTestCase, Path logger = logging.getLogger(__name__) @@ -55,8 +54,8 @@ class TestSchemaExamples(QiskitTestCase): } def setUp(self): - self.examples_base_path = os.path.join(qiskit_path[0], 'schemas', - 'examples') + self.examples_base_path = self._get_resource_path('examples', + Path.SCHEMAS) def test_examples_are_valid(self): """Validate example json files against respective schemas""" diff --git a/test/python/test_unroller.py b/test/python/test_unroller.py index c7d8dc8fd6..0b9a31bc0d 100644 --- a/test/python/test_unroller.py +++ b/test/python/test_unroller.py @@ -12,7 +12,7 @@ import unittest from qiskit import qasm from qiskit.unroll import DagUnroller, JsonBackend from qiskit.converters import ast_to_dag -from .common import QiskitTestCase +from .common import QiskitTestCase, Path class UnrollerTest(QiskitTestCase): @@ -24,7 +24,7 @@ class UnrollerTest(QiskitTestCase): @unittest.skip("Temporary skipping") def test_dag_to_json(self): """Test DagUnroller with JSON backend.""" - ast = qasm.Qasm(filename=self._get_resource_path('qasm/example.qasm')).parse() + ast = qasm.Qasm(filename=self._get_resource_path('example.qasm', Path.QASMS)).parse() dag_circuit = ast_to_dag(ast) dag_unroller = DagUnroller(dag_circuit, JsonBackend()) json_circuit = dag_unroller.execute()