Merge pull request #71 from bubenkoff/fix-json-timings

Correct timing values for cucumber json reporting

CHANGES.rst

@@ -6,6 +6,7 @@ Unreleased
 
 - Better reporting of a not found scenario (bubenkoff)
 - Simple test code generation implemented (bubenkoff)
+- Correct timing values for cucumber json reporting (bubenkoff)
 
 
 2.4.0

README.rst

@@ -693,8 +693,8 @@ Hooks
 pytest-bdd exposes several pytest `hooks <http://pytest.org/latest/plugins.html#well-specified-hooks>`_
 which might be helpful building useful reporting, visualization, etc on top of it:
 
-* pytest_bdd_before_step(request, feature, scenario, step, step_func, step_func_args) - Called before step function
-  is executed
+* pytest_bdd_before_step(request, feature, scenario, step, step_func) - Called before step function
+  is executed and it's arguments evaluated
 
 * pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func_args) - Called after step function
   is successfully executed

pytest_bdd/__init__.py

@@ -3,10 +3,10 @@
 __version__ = '2.4.0'
 
 try:
-    from pytest_bdd.steps import given, when, then  # pragma: no cover
-    from pytest_bdd.scenario import scenario  # pragma: no cover
+    from pytest_bdd.steps import given, when, then
+    from pytest_bdd.scenario import scenario
 
-    __all__ = [given.__name__, when.__name__, then.__name__, scenario.__name__]  # pragma: no cover
+    __all__ = [given.__name__, when.__name__, then.__name__, scenario.__name__]
 except ImportError:
     # avoid import errors when only __version__ is needed (for setup.py)
     pass

pytest_bdd/cucumber_json.py

@@ -1,12 +1,16 @@
 """Cucumber json output formatter."""
-
 import json
+import math
 import os
 import time
 
 import py
 
 from .feature import force_unicode
+from .steps import PY3
+
+if PY3:
+    long = int
 
 
 def pytest_addoption(parser):
@@ -44,7 +48,7 @@ class LogBDDCucumberJSON(object):
     def append(self, obj):
         self.features[-1].append(obj)
 
-    def _get_result(self, step, report):
+    def _get_result(self, step, report, error_message=False):
         """Get scenario test run result.
 
         :param step: `Step` step we get result for
@@ -57,11 +61,11 @@ class LogBDDCucumberJSON(object):
         elif report.failed and step['failed']:
             result = {
                 'status': 'failed',
-                'error_message': force_unicode(report.longrepr),
+                'error_message': force_unicode(report.longrepr) if error_message else '',
             }
         elif report.skipped:
             result = {'status': 'skipped'}
-        result['duration'] = step['duration']
+        result['duration'] = long(math.floor((10 ** 9) * step['duration']))  # nanosec
         return result
 
     def _serialize_tags(self, item):
@@ -95,6 +99,11 @@ class LogBDDCucumberJSON(object):
             return
 
         def stepmap(step):
+            error_message = False
+            if step['failed'] and not scenario.setdefault('failed', False):
+                scenario['failed'] = True
+                error_message = True
+
             return {
                 "keyword": step['keyword'],
                 "name": step['name'],
@@ -102,7 +111,7 @@ class LogBDDCucumberJSON(object):
                 "match": {
                     "location": ""
                 },
-                "result": self._get_result(step, report),
+                "result": self._get_result(step, report, error_message),
             }
 
         if scenario['feature']['filename'] not in self.features:

pytest_bdd/exceptions.py

@@ -9,21 +9,21 @@ class ScenarioValidationError(Exception):
     """Base class for scenario validation."""
 
 
-class ScenarioNotFound(ScenarioValidationError):  # pragma: no cover
+class ScenarioNotFound(ScenarioValidationError):
     """Scenario Not Found"""
 
 
-class ScenarioExamplesNotValidError(ScenarioValidationError):  # pragma: no cover
+class ScenarioExamplesNotValidError(ScenarioValidationError):
     """Scenario steps argumets do not match declared scenario examples."""
 
 
-class StepTypeError(ScenarioValidationError):  # pragma: no cover
+class StepTypeError(ScenarioValidationError):
     """Step definition is not of the type expected in the scenario."""
 
 
-class GivenAlreadyUsed(ScenarioValidationError):  # pragma: no cover
+class GivenAlreadyUsed(ScenarioValidationError):
     """Fixture that implements the Given has been already used."""
 
 
-class StepDefinitionNotFoundError(Exception):  # pragma: no cover
+class StepDefinitionNotFoundError(Exception):
     """Step definition not found."""

pytest_bdd/feature.py

@@ -23,17 +23,17 @@ Syntax example:
 one line.
 """
 
-from os import path as op  # pragma: no cover
+from os import path as op
 
-import re  # pragma: no cover
-import sys  # pragma: no cover
+import re
+import sys
 import textwrap
 
-from pytest_bdd import types  # pragma: no cover
-from pytest_bdd import exceptions  # pragma: no cover
+from . import types
+from . import exceptions
 
 
-class FeatureError(Exception):  # pragma: no cover
+class FeatureError(Exception):
 
     """Feature parse error."""
 
@@ -47,10 +47,10 @@ class FeatureError(Exception):  # pragma: no cover
 
 
 # Global features dictionary
-features = {}  # pragma: no cover
+features = {}
 
 
-STEP_PREFIXES = [  # pragma: no cover
+STEP_PREFIXES = [
     ('Feature: ', types.FEATURE),
     ('Scenario Outline: ', types.SCENARIO_OUTLINE),
     ('Examples: Vertical', types.EXAMPLES_VERTICAL),
@@ -64,9 +64,9 @@ STEP_PREFIXES = [  # pragma: no cover
     ('And ', None),  # Unknown step type,
 ]
 
-COMMENT_SYMBOLS = '#'  # pragma: no cover
+COMMENT_SYMBOLS = '#'
 
-STEP_PARAM_RE = re.compile('\<(.+?)\>')  # pragma: no cover
+STEP_PARAM_RE = re.compile('\<(.+?)\>')
 
 
 def get_step_type(line):
@@ -289,6 +289,7 @@ class Scenario(object):
         self.line_number = line_number
         self.example_converters = example_converters
         self.tags = tags or set()
+        self.failed = False
 
     def add_step(self, step_name, step_type, indent, line_number, keyword):
         """Add step to the scenario.

pytest_bdd/hooks.py

@@ -1,7 +1,7 @@
 """Pytest-bdd pytest hooks."""
 
 
-def pytest_bdd_before_step(request, feature, scenario, step, step_func, step_func_args):
+def pytest_bdd_before_step(request, feature, scenario, step, step_func):
     """Called before step function is executed."""
 
 

pytest_bdd/plugin.py

@@ -1,18 +1,17 @@
 """Pytest plugin entry point. Used for any fixtures needed."""
-
-import os.path  # pragma: no cover
+import os.path
 import time
 
-import pytest  # pragma: no cover
+import pytest
 
-from pytest_bdd import (
+from . import (
     given,
     when,
     then,
 )
 
 
-@pytest.fixture  # pragma: no cover
+@pytest.fixture
 def pytestbdd_feature_base_dir(request):
     """Base feature directory."""
     return os.path.dirname(request.module.__file__)
@@ -30,20 +29,24 @@ def pytest_bdd_step_error(request, feature, scenario, step, step_func, step_func
     Also store step start time.
     """
     step.failed = True
-    step.stop = time.time()
+    scenario.failed = True
+    if step.start:
+        step.stop = time.time()
+    for step in scenario.steps[scenario.steps.index(step):]:
+        step.failed = True
 
 
-def pytest_bdd_before_step(request, feature, scenario, step, step_func, step_func_args):
+def pytest_bdd_before_step(request, feature, scenario, step, step_func):
     """Store step start time."""
     step.start = time.time()
 
 
 def pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func_args):
     """Store step duration."""
-    step.stop = time.time()
+    if step.start and not step.stop:
+        step.stop = time.time()
 
 
-@pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call, __multicall__):
     """Store item in the report object."""
     rep = __multicall__.execute()
@@ -59,7 +62,7 @@ def pytest_runtest_makereport(item, call, __multicall__):
                 'keyword': step.keyword,
                 'line_number': step.line_number,
                 'failed': step.failed,
-                'duration': round(step.stop - step.start, 5)
+                'duration': step.stop - step.start
             } for step in scenario.steps],
             'name': scenario.name,
             'line_number': scenario.line_number,
@@ -76,7 +79,6 @@ def pytest_runtest_makereport(item, call, __multicall__):
         rep.item = {
             'name': item.name
         }
-
     return rep
 
 

pytest_bdd/scenario.py

@@ -10,24 +10,28 @@ test_publish_article = scenario(
     scenario_name='Publishing the article',
 )
 """
-
 import collections
+import inspect
 import os
-import sys
-
-import inspect  # pragma: no cover
 
 import pytest
-
 from _pytest import python
 
-from pytest_bdd.feature import Feature, force_encode  # pragma: no cover
-from pytest_bdd.steps import execute, recreate_function, get_caller_module, get_caller_function
-from pytest_bdd.types import GIVEN
-from pytest_bdd import exceptions
-from pytest_bdd import plugin
+from . import exceptions
+from . import plugin
+from .feature import (
+    Feature,
+    force_encode,
+)
+from .steps import (
+    execute,
+    get_caller_function,
+    get_caller_module,
+    PY3,
+    recreate_function,
+)
+from .types import GIVEN
 
-PY3 = sys.version_info[0] >= 3  # pragma: no cover
 
 if PY3:
     import runpy
@@ -108,6 +112,8 @@ def _find_step_function(request, step, encoding):
 
 def _execute_step_function(request, feature, step, step_func, example=None):
     """Execute step function."""
+    request.config.hook.pytest_bdd_before_step(
+        request=request, feature=feature, scenario=step.scenario, step=step, step_func=step_func)
     kwargs = {}
     if example:
         for key in step.params:
@@ -118,17 +124,14 @@ def _execute_step_function(request, feature, step, step_func, example=None):
     try:
         # Get the step argument values
         kwargs = dict((arg, request.getfuncargvalue(arg)) for arg in inspect.getargspec(step_func).args)
-        request.config.hook.pytest_bdd_before_step(
-            request=request, feature=feature, scenario=scenario, step=step, step_func=step_func,
-            step_func_args=kwargs)
         # Execute the step
         step_func(**kwargs)
         request.config.hook.pytest_bdd_after_step(
-            request=request, feature=feature, scenario=scenario, step=step, step_func=step_func,
+            request=request, feature=feature, scenario=step.scenario, step=step, step_func=step_func,
             step_func_args=kwargs)
     except Exception as exception:
         request.config.hook.pytest_bdd_step_error(
-            request=request, feature=feature, scenario=scenario, step=step, step_func=step_func,
+            request=request, feature=feature, scenario=step.scenario, step=step, step_func=step_func,
             step_func_args=kwargs, exception=exception)
         raise
 

pytest_bdd/steps.py

@@ -31,25 +31,25 @@ Reusing existing fixtures for a different step name:
 given('I have a beautiful article', fixture='article')
 """
 
-from __future__ import absolute_import  # pragma: no cover
+from __future__ import absolute_import
 import re
-from types import CodeType  # pragma: no cover
-import inspect  # pragma: no cover  # pragma: no cover
-import sys  # pragma: no cover
+from types import CodeType
+import inspect
+import sys
 
-import pytest  # pragma: no cover
+import pytest
 
-from pytest_bdd.feature import parse_line  # pragma: no cover
-from pytest_bdd.types import GIVEN, WHEN, THEN  # pragma: no cover
+from .feature import parse_line
+from .types import GIVEN, WHEN, THEN
 
-PY3 = sys.version_info[0] >= 3  # pragma: no cover
+PY3 = sys.version_info[0] >= 3
 
 
-class StepError(Exception):  # pragma: no cover
+class StepError(Exception):
 
     """Step declaration error."""
 
-RE_TYPE = type(re.compile(''))  # pragma: no cover
+RE_TYPE = type(re.compile(''))
 
 
 def given(name, fixture=None, converters=None):

pytest_bdd/types.py

@@ -1,15 +1,15 @@
 """Common type definitions."""
 
-FEATURE = 'feature'  # pragma: no cover
-SCENARIO_OUTLINE = 'scenario outline'  # pragma: no cover
-EXAMPLES = 'examples'  # pragma: no cover
-EXAMPLES_VERTICAL = 'examples vertical'  # pragma: no cover
-EXAMPLES_HEADERS = 'example headers'  # pragma: no cover
-EXAMPLE_LINE = 'example line'  # pragma: no cover
-EXAMPLE_LINE_VERTICAL = 'example line vertical'  # pragma: no cover
-SCENARIO = 'scenario'  # pragma: no cover
-BACKGROUND = 'background'  # pragma: no cover
-GIVEN = 'given'  # pragma: no cover
-WHEN = 'when'  # pragma: no cover
-THEN = 'then'  # pragma: no cover
-TAG = 'tag'  # pragma: no cover
+FEATURE = 'feature'
+SCENARIO_OUTLINE = 'scenario outline'
+EXAMPLES = 'examples'
+EXAMPLES_VERTICAL = 'examples vertical'
+EXAMPLES_HEADERS = 'example headers'
+EXAMPLE_LINE = 'example line'
+EXAMPLE_LINE_VERTICAL = 'example line vertical'
+SCENARIO = 'scenario'
+BACKGROUND = 'background'
+GIVEN = 'given'
+WHEN = 'when'
+THEN = 'then'
+TAG = 'tag'

tests/feature/test_cucumber_json.py

@@ -3,8 +3,6 @@ import json
 import os.path
 import textwrap
 
-import pytest
-
 
 def runandparse(testdir, *args):
     """Run tests in testdir and parse json output."""
@@ -14,21 +12,24 @@ def runandparse(testdir, *args):
     return result, jsonobject
 
 
-@pytest.fixture(scope='session')
-def equals_any():
+class equals_any(object):
+
     """Helper object comparison to which is always 'equal'."""
-    class equals_any(object):
 
-        def __eq__(self, other):
-            return True
+    def __init__(self, type=None):
+        self.type = type
 
-        def __cmp__(self, other):
-            return 0
+    def __eq__(self, other):
+        return isinstance(other, self.type) if self.type else True
 
-    return equals_any()
+    def __cmp__(self, other):
+        return 0 if (isinstance(other, self.type) if self.type else False) else -1
 
 
-def test_step_trace(testdir, equals_any):
+string = type(u'')
+
+
+def test_step_trace(testdir):
     """Test step trace."""
     testdir.makefile('.feature', test=textwrap.dedent("""
     @feature-tag
@@ -90,7 +91,7 @@ def test_step_trace(testdir, equals_any):
                             "name": "a passing step",
                             "result": {
                                 "status": "passed",
-                                "duration": equals_any
+                                "duration": equals_any(int)
                             }
                         },
                         {
@@ -102,7 +103,7 @@ def test_step_trace(testdir, equals_any):
                             "name": "some other passing step",
                             "result": {
                                 "status": "passed",
-                                "duration": equals_any
+                                "duration": equals_any(int)
                             }
                         }
 
@@ -131,7 +132,7 @@ def test_step_trace(testdir, equals_any):
                             "name": "a passing step",
                             "result": {
                                 "status": "passed",
-                                "duration": equals_any
+                                "duration": equals_any(int)
                             }
                         },
                         {
@@ -142,9 +143,9 @@ def test_step_trace(testdir, equals_any):
                             },
                             "name": "a failing step",
                             "result": {
-                                "error_message": equals_any,
+                                "error_message": equals_any(string),
                                 "status": "failed",
-                                "duration": equals_any
+                                "duration": equals_any(int)
                             }
                         }
                     ],

tests/feature/test_steps.py

@@ -175,7 +175,7 @@ def test_step_hooks(testdir):
     assert reprec.ret == 1
 
     calls = reprec.getcalls("pytest_bdd_before_step")
-    assert len(calls) == 1
+    assert len(calls) == 2
 
     calls = reprec.getcalls("pytest_bdd_step_error")
     assert calls[0].request