mirror of https://github.com/rust-lang/rust.git
Auto merge of #84568 - andoriyu:libtest/junit_formatter, r=yaahc
feat(libtest): Add JUnit formatter tracking issue: https://github.com/rust-lang/rust/issues/85563 Add an alternative formatter to `libtest`. Formatter produces valid xml that later can be interpreted as JUnit report. Caveats: - `timestamp` is required by schema, but every viewer/parser ignores it. Attribute is not set to avoid depending on chrono; - Running all "suits" (unit tests, doc-tests and integration tests) will produce a mess; - I couldn't find a way to get integration test binary name, so it's just goes by "integration"; Sample output for unit tests (pretty printed by 3rd party tool): ``` <?xml version="1.0" encoding="UTF-8"?> <testsuites> <testsuite name="test" package="test" id="0" errors="0" failures="0" tests="13" skipped="1"> <testcase classname="results::tests" name="test_completed_bad" time="0"/> <testcase classname="results::tests" name="suite_started" time="0"/> <testcase classname="results::tests" name="suite_ended_ok" time="0"/> <testcase classname="results::tests" name="suite_ended_bad" time="0"/> <testcase classname="junit::tests" name="test_failed_output" time="0"/> <testcase classname="junit::tests" name="test_simple_output" time="0"/> <testcase classname="junit::tests" name="test_multiple_outputs" time="0"/> <testcase classname="results::tests" name="test_completed_ok" time="0"/> <testcase classname="results::tests" name="test_stared" time="0"/> <testcase classname="junit::tests" name="test_generate_xml_no_error_single_testsuite" time="0"/> <testcase classname="results::tests" name="test_simple_output" time="0"/> <testcase classname="test" name="should_panic" time="0"/> <system-out/> <system-err/> </testsuite> </testsuites> ``` Sample output for integration tests (pretty printed by 3rd party tool): ``` <?xml version="1.0" encoding="UTF-8"?> <testsuites> <testsuite name="test" package="test" id="0" errors="0" failures="0" tests="1" skipped="0"> <testcase classname="integration" name="test_add" time="0"/> <system-out/> <system-err/> </testsuite> </testsuites> ``` Sample output for Doc-tests (pretty printed by 3rd party tool): ``` <?xml version="1.0" encoding="UTF-8"?> <testsuites> <testsuite name="test" package="test" id="0" errors="0" failures="0" tests="1" skipped="0"> <testcase classname="src/lib.rs" name="(line 2)" time="0"/> <system-out/> <system-err/> </testsuite> </testsuites> ```
This commit is contained in:
commit
1c6868aa21
|
@ -95,8 +95,9 @@ fn optgroups() -> getopts::Options {
|
||||||
"Configure formatting of output:
|
"Configure formatting of output:
|
||||||
pretty = Print verbose output;
|
pretty = Print verbose output;
|
||||||
terse = Display one character per test;
|
terse = Display one character per test;
|
||||||
json = Output a json document",
|
json = Output a json document;
|
||||||
"pretty|terse|json",
|
junit = Output a JUnit document",
|
||||||
|
"pretty|terse|json|junit",
|
||||||
)
|
)
|
||||||
.optflag("", "show-output", "Show captured stdout of successful tests")
|
.optflag("", "show-output", "Show captured stdout of successful tests")
|
||||||
.optopt(
|
.optopt(
|
||||||
|
@ -336,10 +337,15 @@ fn get_format(
|
||||||
}
|
}
|
||||||
OutputFormat::Json
|
OutputFormat::Json
|
||||||
}
|
}
|
||||||
|
Some("junit") => {
|
||||||
|
if !allow_unstable {
|
||||||
|
return Err("The \"junit\" format is only accepted on the nightly compiler".into());
|
||||||
|
}
|
||||||
|
OutputFormat::Junit
|
||||||
|
}
|
||||||
Some(v) => {
|
Some(v) => {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"argument for --format must be pretty, terse, or json (was \
|
"argument for --format must be pretty, terse, json or junit (was \
|
||||||
{})",
|
{})",
|
||||||
v
|
v
|
||||||
));
|
));
|
||||||
|
|
|
@ -10,7 +10,7 @@ use super::{
|
||||||
cli::TestOpts,
|
cli::TestOpts,
|
||||||
event::{CompletedTest, TestEvent},
|
event::{CompletedTest, TestEvent},
|
||||||
filter_tests,
|
filter_tests,
|
||||||
formatters::{JsonFormatter, OutputFormatter, PrettyFormatter, TerseFormatter},
|
formatters::{JsonFormatter, JunitFormatter, OutputFormatter, PrettyFormatter, TerseFormatter},
|
||||||
helpers::{concurrency::get_concurrency, metrics::MetricMap},
|
helpers::{concurrency::get_concurrency, metrics::MetricMap},
|
||||||
options::{Options, OutputFormat},
|
options::{Options, OutputFormat},
|
||||||
run_tests,
|
run_tests,
|
||||||
|
@ -277,6 +277,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Resu
|
||||||
Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded))
|
Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded))
|
||||||
}
|
}
|
||||||
OutputFormat::Json => Box::new(JsonFormatter::new(output)),
|
OutputFormat::Json => Box::new(JsonFormatter::new(output)),
|
||||||
|
OutputFormat::Junit => Box::new(JunitFormatter::new(output)),
|
||||||
};
|
};
|
||||||
let mut st = ConsoleTestState::new(opts)?;
|
let mut st = ConsoleTestState::new(opts)?;
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,174 @@
|
||||||
|
use std::io::{self, prelude::Write};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use super::OutputFormatter;
|
||||||
|
use crate::{
|
||||||
|
console::{ConsoleTestState, OutputLocation},
|
||||||
|
test_result::TestResult,
|
||||||
|
time,
|
||||||
|
types::{TestDesc, TestType},
|
||||||
|
};
|
||||||
|
|
||||||
|
pub struct JunitFormatter<T> {
|
||||||
|
out: OutputLocation<T>,
|
||||||
|
results: Vec<(TestDesc, TestResult, Duration)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Write> JunitFormatter<T> {
|
||||||
|
pub fn new(out: OutputLocation<T>) -> Self {
|
||||||
|
Self { out, results: Vec::new() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_message(&mut self, s: &str) -> io::Result<()> {
|
||||||
|
assert!(!s.contains('\n'));
|
||||||
|
|
||||||
|
self.out.write_all(s.as_ref())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Write> OutputFormatter for JunitFormatter<T> {
|
||||||
|
fn write_run_start(&mut self, _test_count: usize) -> io::Result<()> {
|
||||||
|
// We write xml header on run start
|
||||||
|
self.write_message(&"<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> {
|
||||||
|
// We do not output anything on test start.
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_timeout(&mut self, _desc: &TestDesc) -> io::Result<()> {
|
||||||
|
// We do not output anything on test timeout.
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_result(
|
||||||
|
&mut self,
|
||||||
|
desc: &TestDesc,
|
||||||
|
result: &TestResult,
|
||||||
|
exec_time: Option<&time::TestExecTime>,
|
||||||
|
_stdout: &[u8],
|
||||||
|
_state: &ConsoleTestState,
|
||||||
|
) -> io::Result<()> {
|
||||||
|
// Because the testsuit node holds some of the information as attributes, we can't write it
|
||||||
|
// until all of the tests has ran. Instead of writting every result as they come in, we add
|
||||||
|
// them to a Vec and write them all at once when run is complete.
|
||||||
|
let duration = exec_time.map(|t| t.0.clone()).unwrap_or_default();
|
||||||
|
self.results.push((desc.clone(), result.clone(), duration));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
|
||||||
|
self.write_message("<testsuites>")?;
|
||||||
|
|
||||||
|
self.write_message(&*format!(
|
||||||
|
"<testsuite name=\"test\" package=\"test\" id=\"0\" \
|
||||||
|
errors=\"0\" \
|
||||||
|
failures=\"{}\" \
|
||||||
|
tests=\"{}\" \
|
||||||
|
skipped=\"{}\" \
|
||||||
|
>",
|
||||||
|
state.failed, state.total, state.ignored
|
||||||
|
))?;
|
||||||
|
for (desc, result, duration) in std::mem::replace(&mut self.results, Vec::new()) {
|
||||||
|
let (class_name, test_name) = parse_class_name(&desc);
|
||||||
|
match result {
|
||||||
|
TestResult::TrIgnored => { /* no-op */ }
|
||||||
|
TestResult::TrFailed => {
|
||||||
|
self.write_message(&*format!(
|
||||||
|
"<testcase classname=\"{}\" \
|
||||||
|
name=\"{}\" time=\"{}\">",
|
||||||
|
class_name,
|
||||||
|
test_name,
|
||||||
|
duration.as_secs()
|
||||||
|
))?;
|
||||||
|
self.write_message("<failure type=\"assert\"/>")?;
|
||||||
|
self.write_message("</testcase>")?;
|
||||||
|
}
|
||||||
|
|
||||||
|
TestResult::TrFailedMsg(ref m) => {
|
||||||
|
self.write_message(&*format!(
|
||||||
|
"<testcase classname=\"{}\" \
|
||||||
|
name=\"{}\" time=\"{}\">",
|
||||||
|
class_name,
|
||||||
|
test_name,
|
||||||
|
duration.as_secs()
|
||||||
|
))?;
|
||||||
|
self.write_message(&*format!("<failure message=\"{}\" type=\"assert\"/>", m))?;
|
||||||
|
self.write_message("</testcase>")?;
|
||||||
|
}
|
||||||
|
|
||||||
|
TestResult::TrTimedFail => {
|
||||||
|
self.write_message(&*format!(
|
||||||
|
"<testcase classname=\"{}\" \
|
||||||
|
name=\"{}\" time=\"{}\">",
|
||||||
|
class_name,
|
||||||
|
test_name,
|
||||||
|
duration.as_secs()
|
||||||
|
))?;
|
||||||
|
self.write_message("<failure type=\"timeout\"/>")?;
|
||||||
|
self.write_message("</testcase>")?;
|
||||||
|
}
|
||||||
|
|
||||||
|
TestResult::TrBench(ref b) => {
|
||||||
|
self.write_message(&*format!(
|
||||||
|
"<testcase classname=\"benchmark::{}\" \
|
||||||
|
name=\"{}\" time=\"{}\" />",
|
||||||
|
class_name, test_name, b.ns_iter_summ.sum
|
||||||
|
))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
TestResult::TrOk | TestResult::TrAllowedFail => {
|
||||||
|
self.write_message(&*format!(
|
||||||
|
"<testcase classname=\"{}\" \
|
||||||
|
name=\"{}\" time=\"{}\"/>",
|
||||||
|
class_name,
|
||||||
|
test_name,
|
||||||
|
duration.as_secs()
|
||||||
|
))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.write_message("<system-out/>")?;
|
||||||
|
self.write_message("<system-err/>")?;
|
||||||
|
self.write_message("</testsuite>")?;
|
||||||
|
self.write_message("</testsuites>")?;
|
||||||
|
|
||||||
|
Ok(state.failed == 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_class_name(desc: &TestDesc) -> (String, String) {
|
||||||
|
match desc.test_type {
|
||||||
|
TestType::UnitTest => parse_class_name_unit(desc),
|
||||||
|
TestType::DocTest => parse_class_name_doc(desc),
|
||||||
|
TestType::IntegrationTest => parse_class_name_integration(desc),
|
||||||
|
TestType::Unknown => (String::from("unknown"), String::from(desc.name.as_slice())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_class_name_unit(desc: &TestDesc) -> (String, String) {
|
||||||
|
// Module path => classname
|
||||||
|
// Function name => name
|
||||||
|
let module_segments: Vec<&str> = desc.name.as_slice().split("::").collect();
|
||||||
|
let (class_name, test_name) = match module_segments[..] {
|
||||||
|
[test] => (String::from("crate"), String::from(test)),
|
||||||
|
[ref path @ .., test] => (path.join("::"), String::from(test)),
|
||||||
|
[..] => unreachable!(),
|
||||||
|
};
|
||||||
|
(class_name, test_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_class_name_doc(desc: &TestDesc) -> (String, String) {
|
||||||
|
// File path => classname
|
||||||
|
// Line # => test name
|
||||||
|
let segments: Vec<&str> = desc.name.as_slice().split(" - ").collect();
|
||||||
|
let (class_name, test_name) = match segments[..] {
|
||||||
|
[file, line] => (String::from(file.trim()), String::from(line.trim())),
|
||||||
|
[..] => unreachable!(),
|
||||||
|
};
|
||||||
|
(class_name, test_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_class_name_integration(desc: &TestDesc) -> (String, String) {
|
||||||
|
(String::from("integration"), String::from(desc.name.as_slice()))
|
||||||
|
}
|
|
@ -8,10 +8,12 @@ use crate::{
|
||||||
};
|
};
|
||||||
|
|
||||||
mod json;
|
mod json;
|
||||||
|
mod junit;
|
||||||
mod pretty;
|
mod pretty;
|
||||||
mod terse;
|
mod terse;
|
||||||
|
|
||||||
pub(crate) use self::json::JsonFormatter;
|
pub(crate) use self::json::JsonFormatter;
|
||||||
|
pub(crate) use self::junit::JunitFormatter;
|
||||||
pub(crate) use self::pretty::PrettyFormatter;
|
pub(crate) use self::pretty::PrettyFormatter;
|
||||||
pub(crate) use self::terse::TerseFormatter;
|
pub(crate) use self::terse::TerseFormatter;
|
||||||
|
|
||||||
|
|
|
@ -39,6 +39,8 @@ pub enum OutputFormat {
|
||||||
Terse,
|
Terse,
|
||||||
/// JSON output
|
/// JSON output
|
||||||
Json,
|
Json,
|
||||||
|
/// JUnit output
|
||||||
|
Junit,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether ignored test should be run or not
|
/// Whether ignored test should be run or not
|
||||||
|
|
Loading…
Reference in New Issue