test_framework
/
pytest-bdd
mirror of https://github.com/pytest-dev/pytest-bdd.git
2
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Fill in step names in cucumber report. 

According to [1], when the outline scenarios are
reported in 'expand' mode, all the outlines will
be expanded to scenarios with their test results.
Also, the step name parameters are filled in.

This is a different mode than without 'expand'
mode[2], where the scenario outlines are reported
so the step names are not filled in and there are
no test results. The "examples" are reported in
this mode.

Since pytest-bdd does report the results of each
step of an outline scenario as separate scenarios,
it looks more like the 'expand' mode, so I've
added the filling in of the parameters in the
step names to make it more compliant.

[1] https://relishapp.com/cucumber/cucumber/docs/formatters/json-output-formatter#scenario-outline-expanded
[2] https://relishapp.com/cucumber/cucumber/docs/formatters/json-output-formatter#scenario-outline
This commit is contained in:
Michiel Holtkamp 2017-10-27 12:40:29 +02:00
parent 11c6097836
commit 2ab960a626
2 changed files with 171 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
pytest_bdd/cucumber_json.py
View File

@ -6,6 +6,7 @@ import os
import time
import py
import re
import six
from .feature import force_unicode
@ -27,12 +28,21 @@ def add_options(parser):
help="create cucumber json style report file at given path.",
)
group._addoption(
"--cucumberjson-expanded",
"--cucumber-json-expanded",
action="store_true",
dest="expand",
default=False,
help="expand scenario outlines into scenarios and fill in the step names",
)
def configure(config):
cucumber_json_path = config.option.cucumber_json_path
# prevent opening json log on slave nodes (xdist)
if cucumber_json_path and not hasattr(config, "slaveinput"):
config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path)
config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path, expand=config.option.expand)
config.pluginmanager.register(config._bddcucumberjson)
@ -47,10 +57,11 @@ class LogBDDCucumberJSON(object):
"""Logging plugin for cucumber like json output."""
def __init__(self, logfile):
def __init__(self, logfile, expand=False):
logfile = os.path.expanduser(os.path.expandvars(logfile))
self.logfile = os.path.normpath(os.path.abspath(logfile))
self.features = {}
self.expand = expand
def append(self, obj):
self.features[-1].append(obj)
@ -95,6 +106,23 @@ class LogBDDCucumberJSON(object):
for tag in item["tags"]
]
def _format_name(self, name, keys, values):
for param, value in zip(keys, values):
name = name.replace('<{}>'.format(param), value)
return name
def _format_step_name(self, report, step):
examples = report.scenario["examples"]
if len(examples) == 0:
return step["name"]
# we take the keys from the first "examples", but in each table, the keys should
# be the same anyway since all the variables need to be filled in.
keys, values = examples[0]["rows"]
row_index = examples[0]["row_index"]
return self._format_name(step["name"], keys, values[row_index])
def pytest_runtest_logreport(self, report):
try:
scenario = report.scenario
@ -112,9 +140,17 @@ class LogBDDCucumberJSON(object):
scenario['failed'] = True
error_message = True
if self.expand:
# XXX The format is already 'expanded' (scenario oultines -> scenarios),
# but the step names were not filled in with parameters. To be backwards
# compatible, do not fill in the step names unless explicitly asked for.
step_name = self._format_step_name(report, step)
else:
step_name = step["name"]
return {
"keyword": step['keyword'],
"name": step['name'],
"name": step_name,
"line": step['line_number'],
"match": {
"location": "",

132
tests/feature/test_cucumber_json.py
View File

@ -44,6 +44,16 @@ def test_step_trace(testdir):
Scenario: Failing
Given a passing step
And a failing step
@scenario-outline-passing-tag
Scenario: Passing outline
Given type <type> and value <value>
Examples: example1
| type | value |
| str | hello |
| int | 42 |
| float | 1.0 |
"""))
testdir.makepyfile(textwrap.dedent("""
import pytest
@ -61,6 +71,10 @@ def test_step_trace(testdir):
def a_failing_step():
raise Exception('Error')
@given('type <type> and value <value>')
def type_type_and_value_value():
return 'pass'
@scenario('test.feature', 'Passing')
def test_passing():
pass
@ -68,6 +82,10 @@ def test_step_trace(testdir):
@scenario('test.feature', 'Failing')
def test_failing():
pass
@scenario('test.feature', 'Passing outline')
def test_passing_outline():
pass
"""))
result, jsonobject = runandparse(testdir)
assert result.ret
@ -156,6 +174,84 @@ def test_step_trace(testdir):
}
],
"type": "scenario"
},
{
"description": "",
"keyword": "Scenario",
"tags": [
{
"line": 14,
"name": "scenario-outline-passing-tag"
}
],
"steps": [
{
"line": 16,
"match": {"location": ""},
"result": {
"status": "passed",
"duration": equals_any(int)
},
"keyword": "Given",
"name": "type <type> and value <value>"
}
],
"line": 15,
"type": "scenario",
"id": "test_passing_outline[str-hello]",
"name": "Passing outline"
},
{
"description": "",
"keyword": "Scenario",
"tags": [
{
"line": 14,
"name": "scenario-outline-passing-tag"
}
],
"steps": [
{
"line": 16,
"match": {"location": ""},
"result": {
"status": "passed",
"duration": equals_any(int)
},
"keyword": "Given",
"name": "type <type> and value <value>"
}
],
"line": 15,
"type": "scenario",
"id": "test_passing_outline[int-42]",
"name": "Passing outline"
},
{
"description": "",
"keyword": "Scenario",
"tags": [
{
"line": 14,
"name": "scenario-outline-passing-tag"
}
],
"steps": [
{
"line": 16,
"match": {"location": ""},
"result": {
"status": "passed",
"duration": equals_any(int)
},
"keyword": "Given",
"name": "type <type> and value <value>"
}
],
"line": 15,
"type": "scenario",
"id": "test_passing_outline[float-1.0]",
"name": "Passing outline"
}
],
"id": os.path.join("test_step_trace0", "test.feature"),
@ -173,3 +269,39 @@ def test_step_trace(testdir):
]
assert jsonobject == expected
def test_step_trace_with_expand_option(testdir):
"""Test step trace."""
testdir.makefile('.feature', test=textwrap.dedent("""
@feature-tag
Feature: One scenario outline, expanded to multiple scenarios
@scenario-outline-passing-tag
Scenario: Passing outline
Given type <type> and value <value>
Examples: example1
| type | value |
| str | hello |
| int | 42 |
| float | 1.0 |
"""))
testdir.makepyfile(textwrap.dedent("""
import pytest
from pytest_bdd import given, scenario
@given('type <type> and value <value>')
def type_type_and_value_value():
return 'pass'
@scenario('test.feature', 'Passing outline')
def test_passing_outline():
pass
"""))
result, jsonobject = runandparse(testdir, '--cucumber-json-expand')
assert result.ret == 0
assert jsonobject[0]["elements"][0]["steps"][0]["name"] == "type str and value hello"
assert jsonobject[0]["elements"][1]["steps"][0]["name"] == "type int and value 42"
assert jsonobject[0]["elements"][2]["steps"][0]["name"] == "type float and value 1.0"