test_framework
/
pytest-bdd
mirror of https://github.com/pytest-dev/pytest-bdd.git
2
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #71 from bubenkoff/fix-json-timings 

Correct timing values for cucumber json reporting
This commit is contained in:
Oleg Pidsadnyi 2014-09-20 22:09:52 +02:00
parent 94067709cc 5433bfeb61
commit 471ae1d28b
13 changed files with 110 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.rst
View File

@ -6,6 +6,7 @@ Unreleased
- Better reporting of a not found scenario (bubenkoff) - Better reporting of a not found scenario (bubenkoff)
- Simple test code generation implemented (bubenkoff) - Simple test code generation implemented (bubenkoff)
- Correct timing values for cucumber json reporting (bubenkoff)
2.4.0 2.4.0

4
README.rst
View File

@ -693,8 +693,8 @@ Hooks
pytest-bdd exposes several pytest `hooks <http://pytest.org/latest/plugins.html#well-specified-hooks>`_ pytest-bdd exposes several pytest `hooks <http://pytest.org/latest/plugins.html#well-specified-hooks>`_
which might be helpful building useful reporting, visualization, etc on top of it: which might be helpful building useful reporting, visualization, etc on top of it:
* pytest_bdd_before_step(request, feature, scenario, step, step_func, step_func_args) - Called before step function * pytest_bdd_before_step(request, feature, scenario, step, step_func) - Called before step function
is executed is executed and it's arguments evaluated
* pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func_args) - Called after step function * pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func_args) - Called after step function
is successfully executed is successfully executed

6
pytest_bdd/__init__.py
View File

@ -3,10 +3,10 @@
__version__ = '2.4.0' __version__ = '2.4.0'
try: try:
from pytest_bdd.steps import given, when, then # pragma: no cover from pytest_bdd.steps import given, when, then
from pytest_bdd.scenario import scenario # pragma: no cover from pytest_bdd.scenario import scenario
__all__ = [given.__name__, when.__name__, then.__name__, scenario.__name__] # pragma: no cover __all__ = [given.__name__, when.__name__, then.__name__, scenario.__name__]
except ImportError: except ImportError:
# avoid import errors when only __version__ is needed (for setup.py) # avoid import errors when only __version__ is needed (for setup.py)
pass pass

19
pytest_bdd/cucumber_json.py
View File

@ -1,12 +1,16 @@
"""Cucumber json output formatter.""" """Cucumber json output formatter."""
import json import json
import math
import os import os
import time import time
import py import py
from .feature import force_unicode from .feature import force_unicode
from .steps import PY3
if PY3:
long = int
def pytest_addoption(parser): def pytest_addoption(parser):
@ -44,7 +48,7 @@ class LogBDDCucumberJSON(object):
def append(self, obj): def append(self, obj):
self.features[-1].append(obj) self.features[-1].append(obj)
def _get_result(self, step, report): def _get_result(self, step, report, error_message=False):
"""Get scenario test run result. """Get scenario test run result.
:param step: `Step` step we get result for :param step: `Step` step we get result for
@ -57,11 +61,11 @@ class LogBDDCucumberJSON(object):
elif report.failed and step['failed']: elif report.failed and step['failed']:
result = { result = {
'status': 'failed', 'status': 'failed',
'error_message': force_unicode(report.longrepr), 'error_message': force_unicode(report.longrepr) if error_message else '',
} }
elif report.skipped: elif report.skipped:
result = {'status': 'skipped'} result = {'status': 'skipped'}
result['duration'] = step['duration'] result['duration'] = long(math.floor((10 ** 9) * step['duration'])) # nanosec
return result return result
def _serialize_tags(self, item): def _serialize_tags(self, item):
@ -95,6 +99,11 @@ class LogBDDCucumberJSON(object):
return return
def stepmap(step): def stepmap(step):
error_message = False
if step['failed'] and not scenario.setdefault('failed', False):
scenario['failed'] = True
error_message = True
return { return {
"keyword": step['keyword'], "keyword": step['keyword'],
"name": step['name'], "name": step['name'],
@ -102,7 +111,7 @@ class LogBDDCucumberJSON(object):
"match": { "match": {
"location": "" "location": ""
}, },
"result": self._get_result(step, report), "result": self._get_result(step, report, error_message),
} }
if scenario['feature']['filename'] not in self.features: if scenario['feature']['filename'] not in self.features:

10
pytest_bdd/exceptions.py
View File

@ -9,21 +9,21 @@ class ScenarioValidationError(Exception):
"""Base class for scenario validation.""" """Base class for scenario validation."""
class ScenarioNotFound(ScenarioValidationError): # pragma: no cover class ScenarioNotFound(ScenarioValidationError):
"""Scenario Not Found""" """Scenario Not Found"""
class ScenarioExamplesNotValidError(ScenarioValidationError): # pragma: no cover class ScenarioExamplesNotValidError(ScenarioValidationError):
"""Scenario steps argumets do not match declared scenario examples.""" """Scenario steps argumets do not match declared scenario examples."""
class StepTypeError(ScenarioValidationError): # pragma: no cover class StepTypeError(ScenarioValidationError):
"""Step definition is not of the type expected in the scenario.""" """Step definition is not of the type expected in the scenario."""
class GivenAlreadyUsed(ScenarioValidationError): # pragma: no cover class GivenAlreadyUsed(ScenarioValidationError):
"""Fixture that implements the Given has been already used.""" """Fixture that implements the Given has been already used."""
class StepDefinitionNotFoundError(Exception): # pragma: no cover class StepDefinitionNotFoundError(Exception):
"""Step definition not found.""" """Step definition not found."""

21
pytest_bdd/feature.py
View File

@ -23,17 +23,17 @@ Syntax example:
one line. one line.
""" """
from os import path as op # pragma: no cover from os import path as op
import re # pragma: no cover import re
import sys # pragma: no cover import sys
import textwrap import textwrap
from pytest_bdd import types # pragma: no cover from . import types
from pytest_bdd import exceptions # pragma: no cover from . import exceptions
class FeatureError(Exception): # pragma: no cover class FeatureError(Exception):
"""Feature parse error.""" """Feature parse error."""
@ -47,10 +47,10 @@ class FeatureError(Exception): # pragma: no cover
# Global features dictionary # Global features dictionary
features = {} # pragma: no cover features = {}
STEP_PREFIXES = [ # pragma: no cover STEP_PREFIXES = [
('Feature: ', types.FEATURE), ('Feature: ', types.FEATURE),
('Scenario Outline: ', types.SCENARIO_OUTLINE), ('Scenario Outline: ', types.SCENARIO_OUTLINE),
('Examples: Vertical', types.EXAMPLES_VERTICAL), ('Examples: Vertical', types.EXAMPLES_VERTICAL),
@ -64,9 +64,9 @@ STEP_PREFIXES = [ # pragma: no cover
('And ', None), # Unknown step type, ('And ', None), # Unknown step type,
] ]
COMMENT_SYMBOLS = '#' # pragma: no cover COMMENT_SYMBOLS = '#'
STEP_PARAM_RE = re.compile('\<(.+?)\>') # pragma: no cover STEP_PARAM_RE = re.compile('\<(.+?)\>')
def get_step_type(line): def get_step_type(line):
@ -289,6 +289,7 @@ class Scenario(object):
self.line_number = line_number self.line_number = line_number
self.example_converters = example_converters self.example_converters = example_converters
self.tags = tags or set() self.tags = tags or set()
self.failed = False
def add_step(self, step_name, step_type, indent, line_number, keyword): def add_step(self, step_name, step_type, indent, line_number, keyword):
"""Add step to the scenario. """Add step to the scenario.

2
pytest_bdd/hooks.py
View File

@ -1,7 +1,7 @@
"""Pytest-bdd pytest hooks.""" """Pytest-bdd pytest hooks."""
def pytest_bdd_before_step(request, feature, scenario, step, step_func, step_func_args): def pytest_bdd_before_step(request, feature, scenario, step, step_func):
"""Called before step function is executed.""" """Called before step function is executed."""

20
pytest_bdd/plugin.py
View File

@ -1,18 +1,17 @@
"""Pytest plugin entry point. Used for any fixtures needed.""" """Pytest plugin entry point. Used for any fixtures needed."""
import os.path
import os.path # pragma: no cover
import time import time
import pytest # pragma: no cover import pytest
from pytest_bdd import ( from . import (
given, given,
when, when,
then, then,
) )
@pytest.fixture # pragma: no cover @pytest.fixture
def pytestbdd_feature_base_dir(request): def pytestbdd_feature_base_dir(request):
"""Base feature directory.""" """Base feature directory."""
return os.path.dirname(request.module.__file__) return os.path.dirname(request.module.__file__)
@ -30,20 +29,24 @@ def pytest_bdd_step_error(request, feature, scenario, step, step_func, step_func
Also store step start time. Also store step start time.
""" """
step.failed = True step.failed = True
scenario.failed = True
if step.start:
step.stop = time.time() step.stop = time.time()
for step in scenario.steps[scenario.steps.index(step):]:
step.failed = True
def pytest_bdd_before_step(request, feature, scenario, step, step_func, step_func_args): def pytest_bdd_before_step(request, feature, scenario, step, step_func):
"""Store step start time.""" """Store step start time."""
step.start = time.time() step.start = time.time()
def pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func_args): def pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func_args):
"""Store step duration.""" """Store step duration."""
if step.start and not step.stop:
step.stop = time.time() step.stop = time.time()
@pytest.mark.tryfirst
def pytest_runtest_makereport(item, call, __multicall__): def pytest_runtest_makereport(item, call, __multicall__):
"""Store item in the report object.""" """Store item in the report object."""
rep = __multicall__.execute() rep = __multicall__.execute()
@ -59,7 +62,7 @@ def pytest_runtest_makereport(item, call, __multicall__):
'keyword': step.keyword, 'keyword': step.keyword,
'line_number': step.line_number, 'line_number': step.line_number,
'failed': step.failed, 'failed': step.failed,
'duration': round(step.stop - step.start, 5) 'duration': step.stop - step.start
} for step in scenario.steps], } for step in scenario.steps],
'name': scenario.name, 'name': scenario.name,
'line_number': scenario.line_number, 'line_number': scenario.line_number,
@ -76,7 +79,6 @@ def pytest_runtest_makereport(item, call, __multicall__):
rep.item = { rep.item = {
'name': item.name 'name': item.name
} }
return rep return rep

35
pytest_bdd/scenario.py
View File

@ -10,24 +10,28 @@ test_publish_article = scenario(
scenario_name='Publishing the article', scenario_name='Publishing the article',
) )
""" """
import collections import collections
import inspect
import os import os
import sys
import inspect # pragma: no cover
import pytest import pytest
from _pytest import python from _pytest import python
from pytest_bdd.feature import Feature, force_encode # pragma: no cover from . import exceptions
from pytest_bdd.steps import execute, recreate_function, get_caller_module, get_caller_function from . import plugin
from pytest_bdd.types import GIVEN from .feature import (
from pytest_bdd import exceptions Feature,
from pytest_bdd import plugin force_encode,
)
from .steps import (
execute,
get_caller_function,
get_caller_module,
PY3,
recreate_function,
)
from .types import GIVEN
PY3 = sys.version_info[0] >= 3 # pragma: no cover
if PY3: if PY3:
import runpy import runpy
@ -108,6 +112,8 @@ def _find_step_function(request, step, encoding):
def _execute_step_function(request, feature, step, step_func, example=None): def _execute_step_function(request, feature, step, step_func, example=None):
"""Execute step function.""" """Execute step function."""
request.config.hook.pytest_bdd_before_step(
request=request, feature=feature, scenario=step.scenario, step=step, step_func=step_func)
kwargs = {} kwargs = {}
if example: if example:
for key in step.params: for key in step.params:
@ -118,17 +124,14 @@ def _execute_step_function(request, feature, step, step_func, example=None):
try: try:
# Get the step argument values # Get the step argument values
kwargs = dict((arg, request.getfuncargvalue(arg)) for arg in inspect.getargspec(step_func).args) kwargs = dict((arg, request.getfuncargvalue(arg)) for arg in inspect.getargspec(step_func).args)
request.config.hook.pytest_bdd_before_step(
request=request, feature=feature, scenario=scenario, step=step, step_func=step_func,
step_func_args=kwargs)
# Execute the step # Execute the step
step_func(**kwargs) step_func(**kwargs)
request.config.hook.pytest_bdd_after_step( request.config.hook.pytest_bdd_after_step(
request=request, feature=feature, scenario=scenario, step=step, step_func=step_func, request=request, feature=feature, scenario=step.scenario, step=step, step_func=step_func,
step_func_args=kwargs) step_func_args=kwargs)
except Exception as exception: except Exception as exception:
request.config.hook.pytest_bdd_step_error( request.config.hook.pytest_bdd_step_error(
request=request, feature=feature, scenario=scenario, step=step, step_func=step_func, request=request, feature=feature, scenario=step.scenario, step=step, step_func=step_func,
step_func_args=kwargs, exception=exception) step_func_args=kwargs, exception=exception)
raise raise

20
pytest_bdd/steps.py
View File

@ -31,25 +31,25 @@ Reusing existing fixtures for a different step name:
given('I have a beautiful article', fixture='article') given('I have a beautiful article', fixture='article')
""" """
from __future__ import absolute_import # pragma: no cover from __future__ import absolute_import
import re import re
from types import CodeType # pragma: no cover from types import CodeType
import inspect # pragma: no cover # pragma: no cover import inspect
import sys # pragma: no cover import sys
import pytest # pragma: no cover import pytest
from pytest_bdd.feature import parse_line # pragma: no cover from .feature import parse_line
from pytest_bdd.types import GIVEN, WHEN, THEN # pragma: no cover from .types import GIVEN, WHEN, THEN
PY3 = sys.version_info[0] >= 3 # pragma: no cover PY3 = sys.version_info[0] >= 3
class StepError(Exception): # pragma: no cover class StepError(Exception):
"""Step declaration error.""" """Step declaration error."""
RE_TYPE = type(re.compile('')) # pragma: no cover RE_TYPE = type(re.compile(''))
def given(name, fixture=None, converters=None): def given(name, fixture=None, converters=None):

26
pytest_bdd/types.py
View File

@ -1,15 +1,15 @@
"""Common type definitions.""" """Common type definitions."""
FEATURE = 'feature' # pragma: no cover FEATURE = 'feature'
SCENARIO_OUTLINE = 'scenario outline' # pragma: no cover SCENARIO_OUTLINE = 'scenario outline'
EXAMPLES = 'examples' # pragma: no cover EXAMPLES = 'examples'
EXAMPLES_VERTICAL = 'examples vertical' # pragma: no cover EXAMPLES_VERTICAL = 'examples vertical'
EXAMPLES_HEADERS = 'example headers' # pragma: no cover EXAMPLES_HEADERS = 'example headers'
EXAMPLE_LINE = 'example line' # pragma: no cover EXAMPLE_LINE = 'example line'
EXAMPLE_LINE_VERTICAL = 'example line vertical' # pragma: no cover EXAMPLE_LINE_VERTICAL = 'example line vertical'
SCENARIO = 'scenario' # pragma: no cover SCENARIO = 'scenario'
BACKGROUND = 'background' # pragma: no cover BACKGROUND = 'background'
GIVEN = 'given' # pragma: no cover GIVEN = 'given'
WHEN = 'when' # pragma: no cover WHEN = 'when'
THEN = 'then' # pragma: no cover THEN = 'then'
TAG = 'tag' # pragma: no cover TAG = 'tag'

31
tests/feature/test_cucumber_json.py
View File

@ -3,8 +3,6 @@ import json
import os.path import os.path
import textwrap import textwrap
import pytest
def runandparse(testdir, *args): def runandparse(testdir, *args):
"""Run tests in testdir and parse json output.""" """Run tests in testdir and parse json output."""
@ -14,21 +12,24 @@ def runandparse(testdir, *args):
return result, jsonobject return result, jsonobject
@pytest.fixture(scope='session')
def equals_any():
"""Helper object comparison to which is always 'equal'."""
class equals_any(object): class equals_any(object):
"""Helper object comparison to which is always 'equal'."""
def __init__(self, type=None):
self.type = type
def __eq__(self, other): def __eq__(self, other):
return True return isinstance(other, self.type) if self.type else True
def __cmp__(self, other): def __cmp__(self, other):
return 0 return 0 if (isinstance(other, self.type) if self.type else False) else -1
return equals_any()
def test_step_trace(testdir, equals_any): string = type(u'')
def test_step_trace(testdir):
"""Test step trace.""" """Test step trace."""
testdir.makefile('.feature', test=textwrap.dedent(""" testdir.makefile('.feature', test=textwrap.dedent("""
@feature-tag @feature-tag
@ -90,7 +91,7 @@ def test_step_trace(testdir, equals_any):
"name": "a passing step", "name": "a passing step",
"result": { "result": {
"status": "passed", "status": "passed",
"duration": equals_any "duration": equals_any(int)
} }
}, },
{ {
@ -102,7 +103,7 @@ def test_step_trace(testdir, equals_any):
"name": "some other passing step", "name": "some other passing step",
"result": { "result": {
"status": "passed", "status": "passed",
"duration": equals_any "duration": equals_any(int)
} }
} }
@ -131,7 +132,7 @@ def test_step_trace(testdir, equals_any):
"name": "a passing step", "name": "a passing step",
"result": { "result": {
"status": "passed", "status": "passed",
"duration": equals_any "duration": equals_any(int)
} }
}, },
{ {
@ -142,9 +143,9 @@ def test_step_trace(testdir, equals_any):
}, },
"name": "a failing step", "name": "a failing step",
"result": { "result": {
"error_message": equals_any, "error_message": equals_any(string),
"status": "failed", "status": "failed",
"duration": equals_any "duration": equals_any(int)
} }
} }
], ],

2
tests/feature/test_steps.py
View File

@ -175,7 +175,7 @@ def test_step_hooks(testdir):
assert reprec.ret == 1 assert reprec.ret == 1
calls = reprec.getcalls("pytest_bdd_before_step") calls = reprec.getcalls("pytest_bdd_before_step")
assert len(calls) == 1 assert len(calls) == 2
calls = reprec.getcalls("pytest_bdd_step_error") calls = reprec.getcalls("pytest_bdd_step_error")
assert calls[0].request assert calls[0].request