feat: support to output the HTML report (#88)

Co-authored-by: rick <linuxsuren@users.noreply.github.com>

.gitignore

@@ -5,3 +5,4 @@ collector-coverage.out
 dist/
 .vscode/launch.json
 sample.yaml
+.DS_Store

README.md

@@ -8,6 +8,7 @@ This is a API testing tool.
 
 ## Features
 
+*   Multiple test report formats: Markdown, HTML, Stdout
 *   Response Body fields equation check
 *   Response Body [eval](https://expr.medv.io/)
 *   Verify the Kubernetes resources

cmd/run.go

@@ -46,7 +46,7 @@ func newDefaultRunOption() *runOption {
 	}
 }
 
-func newDiskCardRunOption() *runOption {
+func newDiscardRunOption() *runOption {
 	return &runOption{
 		reporter:     runner.NewDiscardTestReporter(),
 		reportWriter: runner.NewDiscardResultWriter(),
@@ -74,7 +74,7 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`,
 	flags.DurationVarP(&opt.duration, "duration", "", 0, "Running duration")
 	flags.DurationVarP(&opt.requestTimeout, "request-timeout", "", time.Minute, "Timeout for per request")
 	flags.BoolVarP(&opt.requestIgnoreError, "request-ignore-error", "", false, "Indicate if ignore the request error")
-	flags.StringVarP(&opt.report, "report", "", "", "The type of target report. Supported: markdown, md, discard, std")
+	flags.StringVarP(&opt.report, "report", "", "", "The type of target report. Supported: markdown, md, html, discard, std")
 	flags.StringVarP(&opt.reportFile, "report-file", "", "", "The file path of the report")
 	flags.BoolVarP(&opt.reportIgnore, "report-ignore", "", false, "Indicate if ignore the report output")
 	flags.Int64VarP(&opt.thread, "thread", "", 1, "Threads of the execution")
@@ -98,6 +98,8 @@ func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) {
 	switch o.report {
 	case "markdown", "md":
 		o.reportWriter = runner.NewMarkdownResultWriter(writer)
+	case "html":
+		o.reportWriter = runner.NewHTMLResultWriter(writer)
 	case "discard":
 		o.reportWriter = runner.NewDiscardResultWriter()
 	case "", "std":
@@ -178,7 +180,7 @@ func (o *runOption) runSuiteWithDuration(suite string) (err error) {
 				defer sem.Release(1)
 				defer wait.Done()
 				defer func() {
-					fmt.Println("routing end with", time.Now().Sub(now))
+					fmt.Println("routing end with", time.Since(now))
 				}()
 
 				dataContext := getDefaultContext()

cmd/run_test.go

@@ -53,7 +53,7 @@ func TestRunSuite(t *testing.T) {
 			defer gock.Clean()
 			util.MakeSureNotNil(tt.prepare)()
 			ctx := getDefaultContext()
-			opt := newDiskCardRunOption()
+			opt := newDiscardRunOption()
 			opt.requestTimeout = 30 * time.Second
 			opt.limiter = limit.NewDefaultRateLimiter(0, 0)
 			stopSingal := make(chan struct{}, 1)

pkg/render/template.go

@@ -2,7 +2,9 @@ package render
 
 import (
 	"bytes"
+	"fmt"
 	"html/template"
+	"io"
 	"strings"
 
 	"github.com/Masterminds/sprig/v3"
@@ -31,3 +33,12 @@ func FuncMap() template.FuncMap {
 	}
 	return funcs
 }
+
+// RenderThenPrint renders the template then prints the result
+func RenderThenPrint(name, text string, ctx interface{}, w io.Writer) (err error) {
+	var report string
+	if report, err = Render(name, text, ctx); err == nil {
+		fmt.Fprint(w, report)
+	}
+	return
+}

pkg/render/template_test.go

@@ -1,6 +1,7 @@
 package render
 
 import (
+	"bytes"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -56,3 +57,32 @@ func TestRender(t *testing.T) {
 		})
 	}
 }
+
+func TestRenderThenPrint(t *testing.T) {
+	tests := []struct {
+		name    string
+		tplText string
+		ctx     interface{}
+		buf     *bytes.Buffer
+		expect  string
+	}{{
+		name:    "simple",
+		tplText: `{{max 1 2 3}}`,
+		ctx:     nil,
+		buf:     new(bytes.Buffer),
+		expect:  `3`,
+	}, {
+		name:    "with a map as context",
+		tplText: `{{.name}}`,
+		ctx:     map[string]string{"name": "linuxsuren"},
+		buf:     new(bytes.Buffer),
+		expect:  "linuxsuren",
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := RenderThenPrint(tt.name, tt.tplText, tt.ctx, tt.buf)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expect, tt.buf.String())
+		})
+	}
+}

pkg/runner/data/html.html

@@ -0,0 +1,33 @@
+<!DOCTYPE>
+<html lang="zh">
+<head>
+    <title>API Testing Report</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style type="text/css">
+    [leading-7=""] {
+        line-height: 1.75rem;
+    }
+    .text-center, [text-center=""] {
+        text-align: center;
+    }
+    footer {
+    position: fixed;
+    bottom: 0;
+    width: 100%;
+    height: 60px;
+    }
+    </style>
+</head>
+<body>
+    <table>
+        <caption>API Testing Report</caption>
+        <tr><th>API</th><th>Average</th><th>Max</th><th>Min</th><th>Count</th><th>Error</th></tr>
+        {{- range $val := .}}
+        <tr><td>{{$val.API}}</td><td>{{$val.Average}}</td><td>{{$val.Max}}</td><td>{{$val.Min}}</td><td>{{$val.Count}}</td><td>{{$val.Error}}</td></tr>
+        {{- end}}
+    </table>
+    <footer text-center="" leading-7="">
+        <p text-sm=""><a href="https://github.com/LinuxSuRen/api-testing" target="_blank" rel="noopener">Powered by API Testing</a></p>
+    </footer>
+</body>
+</html>

pkg/runner/reporter_memory.go

@@ -58,12 +58,8 @@ func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice,
 			item.Total += duration
 			item.Count += 1
 
-			if record.EndTime.After(item.Last) {
-				item.Last = record.EndTime
-			}
-			if record.BeginTime.Before(item.First) {
-				item.First = record.BeginTime
-			}
+			item.Last = getLaterTime(record.EndTime, item.Last)
+			item.LastErrorMessage = getOriginalStringWhenEmpty(item.LastErrorMessage, record.GetErrorMessage())
 		} else {
 			resultWithTotal[api] = &ReportResultWithTotal{
 				ReportResult: ReportResult{
@@ -77,6 +73,7 @@ func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice,
 				Last:  record.EndTime,
 				Total: duration,
 			}
+			resultWithTotal[api].LastErrorMessage = record.GetErrorMessage()
 		}
 	}
 
@@ -91,3 +88,17 @@ func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice,
 	sort.Sort(result)
 	return
 }
+
+func getLaterTime(a, b time.Time) time.Time {
+	if a.After(b) {
+		return a
+	}
+	return b
+}
+
+func getOriginalStringWhenEmpty(a, b string) string {
+	if b == "" {
+		return a
+	}
+	return b
+}

pkg/runner/reporter_memory_test.go

@@ -38,6 +38,7 @@ func TestExportAllReportResults(t *testing.T) {
 			BeginTime: now,
 			EndTime:   now.Add(time.Second * 4),
 			Error:     errors.New("fake"),
+			Body:      "fake",
 		}, {
 			API:       urlFoo,
 			Method:    http.MethodGet,
@@ -68,6 +69,7 @@ func TestExportAllReportResults(t *testing.T) {
 			Min:              time.Second * 2,
 			Count:            3,
 			Error:            1,
+			LastErrorMessage: "fake",
 		}, {
 			API:     "GET http://bar",
 			Average: time.Second,
@@ -77,6 +79,25 @@ func TestExportAllReportResults(t *testing.T) {
 			Count:   1,
 			Error:   0,
 		}},
+	}, {
+		name: "first record has error",
+		records: []*runner.ReportRecord{{
+			API:       urlFoo,
+			Method:    http.MethodGet,
+			BeginTime: now,
+			EndTime:   now.Add(time.Second * 4),
+			Error:     errors.New("fake"),
+			Body:      "fake",
+		}},
+		expect: runner.ReportResultSlice{{
+			API:              "GET http://foo",
+			Average:          time.Second * 4,
+			Max:              time.Second * 4,
+			Min:              time.Second * 4,
+			Count:            1,
+			Error:            1,
+			LastErrorMessage: "fake",
+		}},
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/runner/simple.go

@@ -102,6 +102,15 @@ func (r *ReportRecord) ErrorCount() int {
 	return 1
 }
 
+// GetErrorMessage returns the error message
+func (r *ReportRecord) GetErrorMessage() string {
+	if r.ErrorCount() > 0 {
+		return r.Body
+	} else {
+		return ""
+	}
+}
+
 // NewReportRecord creates a record, and set the begin time to be now
 func NewReportRecord() *ReportRecord {
 	return &ReportRecord{
@@ -118,6 +127,7 @@ type ReportResult struct {
 	Min              time.Duration
 	QPS              int
 	Error            int
+	LastErrorMessage string
 }
 
 // ReportResultSlice is the alias type of ReportResult slice
@@ -202,10 +212,6 @@ func (r *simpleTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataConte
 	}(record)
 
 	defer func() {
-		if testcase.Clean.CleanPrepare {
-			err = r.doCleanPrepare(testcase)
-		}
-
 		if err == nil {
 			err = runJob(testcase.After)
 		}

pkg/runner/simple_test.go

@@ -72,9 +72,6 @@ func TestTestCase(t *testing.T) {
 			Before: atest.Job{
 				Items: []string{"sleep(1)"},
 			},
-			Clean: atest.Clean{
-				CleanPrepare: true,
-			},
 		},
 		execer: fakeruntime.FakeExecer{},
 		prepare: func() {

pkg/runner/testdata/report.html

@@ -0,0 +1,31 @@
+<!DOCTYPE>
+<html lang="zh">
+<head>
+    <title>API Testing Report</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style type="text/css">
+    [leading-7=""] {
+        line-height: 1.75rem;
+    }
+    .text-center, [text-center=""] {
+        text-align: center;
+    }
+    footer {
+    position: fixed;
+    bottom: 0;
+    width: 100%;
+    height: 60px;
+    }
+    </style>
+</head>
+<body>
+    <table>
+        <caption>API Testing Report</caption>
+        <tr><th>API</th><th>Average</th><th>Max</th><th>Min</th><th>Count</th><th>Error</th></tr>
+        <tr><td>/foo</td><td>3ns</td><td>3ns</td><td>3ns</td><td>1</td><td>0</td></tr>
+    </table>
+    <footer text-center="" leading-7="">
+        <p text-sm=""><a href="https://github.com/LinuxSuRen/api-testing" target="_blank" rel="noopener">Powered by API Testing</a></p>
+    </footer>
+</body>
+</html>

pkg/runner/writer_html.go

@@ -0,0 +1,25 @@
+package runner
+
+import (
+	_ "embed"
+	"io"
+
+	"github.com/linuxsuren/api-testing/pkg/render"
+)
+
+type htmlResultWriter struct {
+	writer io.Writer
+}
+
+// NewHTMLResultWriter creates a new htmlResultWriter
+func NewHTMLResultWriter(writer io.Writer) ReportResultWriter {
+	return &htmlResultWriter{writer: writer}
+}
+
+// Output writes the HTML base report to target writer
+func (w *htmlResultWriter) Output(result []ReportResult) (err error) {
+	return render.RenderThenPrint("html-report", htmlReport, result, w.writer)
+}
+
+//go:embed data/html.html
+var htmlReport string

pkg/runner/writer_html_test.go

@@ -0,0 +1,43 @@
+package runner_test
+
+import (
+	"bytes"
+	"testing"
+
+	_ "embed"
+
+	"github.com/linuxsuren/api-testing/pkg/runner"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTMLResultWriter(t *testing.T) {
+	tests := []struct {
+		name    string
+		buf     *bytes.Buffer
+		results []runner.ReportResult
+		expect  string
+	}{{
+		name: "simple",
+		buf:  new(bytes.Buffer),
+		results: []runner.ReportResult{{
+			API:     "/foo",
+			Max:     3,
+			Min:     3,
+			Average: 3,
+			Error:   0,
+			Count:   1,
+		}},
+		expect: htmlReportExpect,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			w := runner.NewHTMLResultWriter(tt.buf)
+			err := w.Output(tt.results)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expect, tt.buf.String())
+		})
+	}
+}
+
+//go:embed testdata/report.html
+var htmlReportExpect string

pkg/runner/writer_markdown.go

@@ -0,0 +1,25 @@
+package runner
+
+import (
+	_ "embed"
+	"io"
+
+	"github.com/linuxsuren/api-testing/pkg/render"
+)
+
+type markdownResultWriter struct {
+	writer io.Writer
+}
+
+// NewMarkdownResultWriter creates the Markdown writer
+func NewMarkdownResultWriter(writer io.Writer) ReportResultWriter {
+	return &markdownResultWriter{writer: writer}
+}
+
+// Output writes the Markdown based report to target writer
+func (w *markdownResultWriter) Output(result []ReportResult) (err error) {
+	return render.RenderThenPrint("md-report", markdownReport, result, w.writer)
+}
+
+//go:embed data/report.md
+var markdownReport string

pkg/runner/writer_markdown_test.go

@@ -0,0 +1,35 @@
+package runner_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/linuxsuren/api-testing/pkg/runner"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarkdownWriter(t *testing.T) {
+	buf := new(bytes.Buffer)
+	writer := runner.NewMarkdownResultWriter(buf)
+
+	err := writer.Output([]runner.ReportResult{{
+		API:     "api",
+		Average: 3,
+		Max:     4,
+		Min:     2,
+		Count:   3,
+		Error:   0,
+	}, {
+		API:     "api",
+		Average: 3,
+		Max:     4,
+		Min:     2,
+		Count:   3,
+		Error:   0,
+	}})
+	assert.Nil(t, err)
+	assert.Equal(t, `| API | Average | Max | Min | Count | Error |
+|---|---|---|---|---|---|
+| api | 3ns | 4ns | 2ns | 3 | 0 |
+| api | 3ns | 4ns | 2ns | 3 | 0 |`, buf.String())
+}

pkg/runner/writer_std.go

@@ -1,11 +1,9 @@
 package runner
 
 import (
-	"bytes"
 	_ "embed"
 	"fmt"
 	"io"
-	"text/template"
 )
 
 type stdResultWriter struct {
@@ -23,36 +21,19 @@ func NewDiscardResultWriter() ReportResultWriter {
 }
 
 // Output writer the report to target writer
-func (w *stdResultWriter) Output(result []ReportResult) error {
+func (w *stdResultWriter) Output(results []ReportResult) error {
+	var errResults []ReportResult
 	fmt.Fprintf(w.writer, "API Average Max Min QPS Count Error\n")
-	for _, r := range result {
+	for _, r := range results {
 		fmt.Fprintf(w.writer, "%s %v %v %v %d %d %d\n", r.API, r.Average, r.Max,
 			r.Min, r.QPS, r.Count, r.Error)
+		if r.Error > 0 && r.LastErrorMessage != "" {
+			errResults = append(errResults, r)
+		}
+	}
+
+	for _, r := range errResults {
+		fmt.Fprintf(w.writer, "%s error: %s\n", r.API, r.LastErrorMessage)
 	}
 	return nil
 }
-
-type markdownResultWriter struct {
-	writer io.Writer
-}
-
-// NewMarkdownResultWriter creates the Markdown writer
-func NewMarkdownResultWriter(writer io.Writer) ReportResultWriter {
-	return &markdownResultWriter{writer: writer}
-}
-
-// Output writer the Markdown based report to target writer
-func (w *markdownResultWriter) Output(result []ReportResult) (err error) {
-	var tpl *template.Template
-	if tpl, err = template.New("report").Parse(markDownReport); err == nil {
-		buf := new(bytes.Buffer)
-
-		if err = tpl.Execute(buf, result); err == nil {
-			fmt.Fprint(w.writer, buf.String())
-		}
-	}
-	return
-}
-
-//go:embed data/report.md
-var markDownReport string

pkg/runner/writer_std_test.go

@@ -8,33 +8,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestMarkdownWriter(t *testing.T) {
-	buf := new(bytes.Buffer)
-	writer := runner.NewMarkdownResultWriter(buf)
-
-	err := writer.Output([]runner.ReportResult{{
-		API:     "api",
-		Average: 3,
-		Max:     4,
-		Min:     2,
-		Count:   3,
-		Error:   0,
-	}, {
-		API:     "api",
-		Average: 3,
-		Max:     4,
-		Min:     2,
-		Count:   3,
-		Error:   0,
-	}})
-	assert.Nil(t, err)
-	assert.Equal(t, `| API | Average | Max | Min | Count | Error |
-|---|---|---|---|---|---|
-| api | 3ns | 4ns | 2ns | 3 | 0 |
-| api | 3ns | 4ns | 2ns | 3 | 0 |
-`, buf.String())
-}
-
 func TestNewStdResultWriter(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -61,6 +34,39 @@ func TestNewStdResultWriter(t *testing.T) {
 		}},
 		expect: `API Average Max Min QPS Count Error
 api 1ns 1ns 1ns 10 1 0
+`,
+	}, {
+		name: "have errors",
+		buf:  new(bytes.Buffer),
+		results: []runner.ReportResult{{
+			API:              "api",
+			Average:          1,
+			Max:              1,
+			Min:              1,
+			QPS:              10,
+			Count:            1,
+			Error:            1,
+			LastErrorMessage: "error",
+		}},
+		expect: `API Average Max Min QPS Count Error
+api 1ns 1ns 1ns 10 1 1
+api error: error
+`,
+	}, {
+		name: "have no errors but with message",
+		buf:  new(bytes.Buffer),
+		results: []runner.ReportResult{{
+			API:              "api",
+			Average:          1,
+			Max:              1,
+			Min:              1,
+			QPS:              10,
+			Count:            1,
+			Error:            0,
+			LastErrorMessage: "message",
+		}},
+		expect: `API Average Max Min QPS Count Error
+api 1ns 1ns 1ns 10 1 0
 `,
 	}}
 	for _, tt := range tests {

pkg/testing/case.go

@@ -15,7 +15,6 @@ type TestCase struct {
 	After   Job      `yaml:"after,omitempty" json:"after"`
 	Request Request  `yaml:"request" json:"request"`
 	Expect  Response `yaml:"expect,omitempty" json:"expect"`
-	Clean   Clean    `yaml:"clean,omitempty" json:"-"`
 }
 
 // InScope returns true if the test case is in scope with the given items.
@@ -57,8 +56,3 @@ type Response struct {
 	Verify           []string               `yaml:"verify,omitempty" json:"verify,omitempty"`
 	Schema           string                 `yaml:"schema,omitempty" json:"schema,omitempty"`
 }
-
-// Clean represents the clean work after testing
-type Clean struct {
-	CleanPrepare bool `yaml:"cleanPrepare"`
-}