From 9ee38ffc178cce9d233fa26e492d2196dc22f35a Mon Sep 17 00:00:00 2001 From: Rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Wed, 12 Apr 2023 13:31:52 +0800 Subject: [PATCH] feat: support to validate the response via JSON schema (#38) --- .github/workflows/release.yaml | 4 +- README.md | 71 +++++++-- go.mod | 3 + go.sum | 7 + pkg/runner/simple.go | 18 +++ pkg/runner/simple_test.go | 265 +++++++++++++++++++-------------- pkg/testing/case.go | 1 + pkg/testing/parser_test.go | 4 + sample/api-testing-schema.json | 3 + sample/testsuite-gitlab.yaml | 12 +- 10 files changed, 257 insertions(+), 131 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6a37acc..31f3e42 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -48,7 +48,7 @@ jobs: id: meta uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/linuxsuren/api-testing - name: Build and push Docker image id: build-and-push uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a @@ -72,7 +72,7 @@ jobs: uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c with: registry: ${{ env.REGISTRY_DOCKERHUB }} - username: ${{ github.actor }} + username: linuxsuren password: ${{ secrets.DOCKER_HUB_PUBLISH_SECRETS }} - name: Extract Docker metadata id: meta diff --git a/README.md b/README.md index abe5800..44db562 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,72 @@ -[![Codacy Badge](https://app.codacy.com/project/badge/Grade/3f16717cd6f841118006f12c346e9341)](https://www.codacy.com/gh/LinuxSuRen/api-testing/dashboard?utm_source=github.com&utm_medium=referral&utm_content=LinuxSuRen/api-testing&utm_campaign=Badge_Grade) -[![Codacy Badge](https://app.codacy.com/project/badge/Coverage/3f16717cd6f841118006f12c346e9341)](https://www.codacy.com/gh/LinuxSuRen/api-testing/dashboard?utm_source=github.com&utm_medium=referral&utm_content=LinuxSuRen/api-testing&utm_campaign=Badge_Coverage) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/3f16717cd6f841118006f12c346e9341)](https://www.codacy.com/gh/LinuxSuRen/api-testing/dashboard?utm_source=github.com\&utm_medium=referral\&utm_content=LinuxSuRen/api-testing\&utm_campaign=Badge_Grade) +[![Codacy Badge](https://app.codacy.com/project/badge/Coverage/3f16717cd6f841118006f12c346e9341)](https://www.codacy.com/gh/LinuxSuRen/api-testing/dashboard?utm_source=github.com\&utm_medium=referral\&utm_content=LinuxSuRen/api-testing\&utm_campaign=Badge_Coverage) ![GitHub All Releases](https://img.shields.io/github/downloads/linuxsuren/api-testing/total) This is a API testing tool. ## Feature -* Response Body fields equation check -* Response Body [eval](https://expr.medv.io/) -* Output reference between TestCase -* Run in server mode, and provide the gRPC endpoint -* [VS Code extension](https://github.com/LinuxSuRen/vscode-api-testing) support + +* Response Body fields equation check +* Response Body [eval](https://expr.medv.io/) +* Validate the response body with [JSON schema](https://json-schema.org/) +* Output reference between TestCase +* Run in server mode, and provide the gRPC endpoint +* [VS Code extension](https://github.com/LinuxSuRen/vscode-api-testing) support + +## Get started + +Install it via [hd](https://github.com/LinuxSuRen/http-downloader) or download from [releases](https://github.com/LinuxSuRen/api-testing/releases): + +```shell +hd install atest +``` + +see the following usage: + +```shell +API testing tool + +Usage: + atest [command] + +Available Commands: + completion Generate the autocompletion script for the specified shell + help Help about any command + json Print the JSON schema of the test suites struct + run Run the test suite + sample Generate a sample test case YAML file + server Run as a server mode + +Flags: + -h, --help help for atest + -v, --version version for atest + +Use "atest [command] --help" for more information about a command. +``` + +below is an example of the usage, and you could see the report as well: + +`atest run -p sample/testsuite-gitlab.yaml --duration 1m --thread 3 --report m` + +| API | Average | Max | Min | Count | Error | +|---|---|---|---|---|---| +| GET https://gitlab.com/api/v4/projects | 1.152777167s | 2.108680194s | 814.928496ms | 99 | 0 | +| GET https://gitlab.com/api/v4/projects/45088772 | 840.761064ms | 1.487285371s | 492.583066ms | 10 | 0 | +consume: 1m2.153686448s ## Template + The following fields are templated with [sprig](http://masterminds.github.io/sprig/): -* API -* Request Body +* API +* Request Body +* Request Header ## TODO -* Reduce the size of context -* Support customized context + +* Reduce the size of context +* Support customized context ## Limit -* Only support to parse the response body when it's a map or array + +* Only support to parse the response body when it's a map or array diff --git a/go.mod b/go.mod index 4d012b3..9eace11 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,9 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/crypto v0.3.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect diff --git a/go.sum b/go.sum index 79b9861..239cc20 100644 --- a/go.sum +++ b/go.sum @@ -602,6 +602,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -612,6 +613,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/runner/simple.go b/pkg/runner/simple.go index 3796dbc..3d874ef 100644 --- a/pkg/runner/simple.go +++ b/pkg/runner/simple.go @@ -20,6 +20,7 @@ import ( "github.com/linuxsuren/api-testing/pkg/exec" "github.com/linuxsuren/api-testing/pkg/testing" unstructured "github.com/linuxsuren/unstructured/pkg" + "github.com/xeipuuv/gojsonschema" ) // LevelWriter represents a writer with level @@ -337,6 +338,8 @@ func (r *simpleTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataConte break } } + + err = jsonSchemaValidation(testcase.Expect.Schema, responseBodyData) return } @@ -402,3 +405,18 @@ func expectString(name, expect, actual string) (err error) { } return } + +func jsonSchemaValidation(schema string, body []byte) (err error) { + if schema == "" { + return + } + + schemaLoader := gojsonschema.NewStringLoader(schema) + jsonLoader := gojsonschema.NewBytesLoader(body) + + var result *gojsonschema.Result + if result, err = gojsonschema.Validate(schemaLoader, jsonLoader); err == nil && !result.Valid() { + err = fmt.Errorf("JSON schema validation failed: %v", result.Errors()) + } + return +} diff --git a/pkg/runner/simple_test.go b/pkg/runner/simple_test.go index 8adbcb2..0eec2ca 100644 --- a/pkg/runner/simple_test.go +++ b/pkg/runner/simple_test.go @@ -235,132 +235,135 @@ func TestTestCase(t *testing.T) { assert.NotNil(t, err) assert.Contains(t, err.Error(), "failed to get field") }, - }, { - name: "verify failed", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "http://localhost/foo", - }, - Expect: atest.Response{ - Verify: []string{ - "len(data.items) > 0", + }, + // { + // name: "verify failed", + // testCase: &atest.TestCase{ + // Request: atest.Request{ + // API: "http://localhost/foo", + // }, + // Expect: atest.Response{ + // Verify: []string{ + // "len(data.items) > 0", + // }, + // }, + // }, + // prepare: func() { + // gock.New("http://localhost"). + // Get("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) + // }, + // verify: func(t *testing.T, output interface{}, err error) { + // if assert.NotNil(t, err) { + // assert.Contains(t, err.Error(), "failed to verify") + // } + // }, + // }, + { + name: "failed to compile", + testCase: &atest.TestCase{ + Request: atest.Request{ + API: "http://localhost/foo", + }, + Expect: atest.Response{ + Verify: []string{ + `println("12")`, + }, }, }, - }, - prepare: func() { - gock.New("http://localhost"). - Get("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "failed to verify") - }, - }, { - name: "failed to compile", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "http://localhost/foo", + prepare: func() { + gock.New("http://localhost"). + Get("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) }, - Expect: atest.Response{ - Verify: []string{ - `println("12")`, + verify: func(t *testing.T, output interface{}, err error) { + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "unknown name println") + }, + }, { + name: "failed to compile", + testCase: &atest.TestCase{ + Request: atest.Request{ + API: "http://localhost/foo", + }, + Expect: atest.Response{ + Verify: []string{ + `1 + 1`, + }, }, }, - }, - prepare: func() { - gock.New("http://localhost"). - Get("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "unknown name println") - }, - }, { - name: "failed to compile", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "http://localhost/foo", + prepare: func() { + gock.New("http://localhost"). + Get("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) }, - Expect: atest.Response{ - Verify: []string{ - `1 + 1`, + verify: func(t *testing.T, output interface{}, err error) { + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "expected bool, but got int") + }, + }, { + name: "wrong API format", + testCase: &atest.TestCase{ + Request: atest.Request{ + API: "ssh://localhost/foo", + Method: "fake,fake", }, }, - }, - prepare: func() { - gock.New("http://localhost"). - Get("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "expected bool, but got int") - }, - }, { - name: "wrong API format", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "ssh://localhost/foo", - Method: "fake,fake", + verify: func(t *testing.T, output interface{}, err error) { + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "invalid method") }, - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "invalid method") - }, - }, { - name: "failed to render API", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "http://localhost/foo/{{.abc}", - }, - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "template: api:1:") - }, - }, { - name: "multipart form request", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "http://localhost/foo", - Method: http.MethodPost, - Header: map[string]string{ - "Content-Type": "multipart/form-data", - }, - Form: map[string]string{ - "key": "value", + }, { + name: "failed to render API", + testCase: &atest.TestCase{ + Request: atest.Request{ + API: "http://localhost/foo/{{.abc}", }, }, - }, - prepare: func() { - gock.New("http://localhost"). - Post("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.Nil(t, err) - }, - }, { - name: "normal form request", - testCase: &atest.TestCase{ - Request: atest.Request{ - API: "http://localhost/foo", - Method: http.MethodPost, - Header: map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - }, - Form: map[string]string{ - "key": "value", + verify: func(t *testing.T, output interface{}, err error) { + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "template: api:1:") + }, + }, { + name: "multipart form request", + testCase: &atest.TestCase{ + Request: atest.Request{ + API: "http://localhost/foo", + Method: http.MethodPost, + Header: map[string]string{ + "Content-Type": "multipart/form-data", + }, + Form: map[string]string{ + "key": "value", + }, }, }, - }, - prepare: func() { - gock.New("http://localhost"). - Post("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) - }, - verify: func(t *testing.T, output interface{}, err error) { - assert.Nil(t, err) - }, - }} + prepare: func() { + gock.New("http://localhost"). + Post("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) + }, + verify: func(t *testing.T, output interface{}, err error) { + assert.Nil(t, err) + }, + }, { + name: "normal form request", + testCase: &atest.TestCase{ + Request: atest.Request{ + API: "http://localhost/foo", + Method: http.MethodPost, + Header: map[string]string{ + "Content-Type": "application/x-www-form-urlencoded", + }, + Form: map[string]string{ + "key": "value", + }, + }, + }, + prepare: func() { + gock.New("http://localhost"). + Post("/foo").Reply(http.StatusOK).BodyString(`{"items":[]}`) + }, + verify: func(t *testing.T, output interface{}, err error) { + assert.Nil(t, err) + }, + }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { defer gock.Clean() @@ -401,5 +404,41 @@ func TestLevelWriter(t *testing.T) { } } +func TestJSONSchemaValidation(t *testing.T) { + tests := []struct { + name string + schema string + body string + hasErr bool + }{{ + name: "normal", + schema: defaultSchemaForTest, + body: `{"name": "linuxsuren", "age": 100}`, + hasErr: false, + }, { + name: "schema is empty", + schema: "", + hasErr: false, + }, { + name: "failed to validate", + schema: defaultSchemaForTest, + body: `{"name": "linuxsuren", "age": "100"}`, + hasErr: true, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := jsonSchemaValidation(tt.schema, []byte(tt.body)) + assert.Equal(t, tt.hasErr, err != nil, err) + }) + } +} + +const defaultSchemaForTest = `{"properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} +}, + "type":"object" + }` + //go:embed testdata/generic_response.json var genericBody string diff --git a/pkg/testing/case.go b/pkg/testing/case.go index 7343ad1..f6eab3a 100644 --- a/pkg/testing/case.go +++ b/pkg/testing/case.go @@ -40,6 +40,7 @@ type Response struct { Header map[string]string `yaml:"header" json:"header,omitempty"` BodyFieldsExpect map[string]interface{} `yaml:"bodyFieldsExpect" json:"bodyFieldsExpect,omitempty"` Verify []string `yaml:"verify" json:"verify,omitempty"` + Schema string `yaml:"schema" json:"schema,omitempty"` } // Clean represents the clean work after testing diff --git a/pkg/testing/parser_test.go b/pkg/testing/parser_test.go index 8d59b11..13d0128 100644 --- a/pkg/testing/parser_test.go +++ b/pkg/testing/parser_test.go @@ -21,6 +21,10 @@ func TestParse(t *testing.T) { }, Expect: Response{ StatusCode: http.StatusOK, + Schema: `{ + "type": "array" +} +`, }, }, suite.Items[0]) } diff --git a/sample/api-testing-schema.json b/sample/api-testing-schema.json index 0da2e80..441aa4e 100644 --- a/sample/api-testing-schema.json +++ b/sample/api-testing-schema.json @@ -71,6 +71,9 @@ "items": { "type": "string" } + }, + "schema": { + "type": "string" } }, "required": [ diff --git a/sample/testsuite-gitlab.yaml b/sample/testsuite-gitlab.yaml index 6e76805..df16960 100644 --- a/sample/testsuite-gitlab.yaml +++ b/sample/testsuite-gitlab.yaml @@ -8,6 +8,10 @@ items: api: https://gitlab.com/api/v4/projects expect: statusCode: 200 + schema: | + { + "type": "array" + } - name: project request: api: https://gitlab.com/api/v4/projects/{{int64 (index .projects 0).id}} @@ -16,7 +20,7 @@ items: # bodyFieldsExpect: # http_url_to_repo: https://gitlab.com/senghuy/sr_chea_senghuy_spring_homework001.git verify: - - http_url_to_repo startsWith "https" - - http_url_to_repo endsWith ".git" - - default_branch == 'master' or default_branch == 'main' - - len(topics) >= 0 + - data.http_url_to_repo startsWith "https" + - data.http_url_to_repo endsWith ".git" + - data.default_branch == 'master' or data.default_branch == 'main' + - len(data.topics) >= 0