From 08f2907c3c0c90c340ebff7be045e457b191d944 Mon Sep 17 00:00:00 2001 From: rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Tue, 13 Jun 2023 17:00:24 +0800 Subject: [PATCH 1/2] feat: support to calculate the API coverage --- cmd/function.go | 3 +- cmd/run.go | 12 ++++ cmd/run_test.go | 20 +++++- pkg/apispec/swagger.go | 98 +++++++++++++++++++++++++ pkg/apispec/swagger_test.go | 111 +++++++++++++++++++++++++++++ pkg/apispec/testdata/swagger.json | 34 +++++++++ pkg/runner/simple.go | 2 + pkg/runner/writer_html.go | 10 ++- pkg/runner/writer_html_test.go | 1 + pkg/runner/writer_markdown.go | 10 ++- pkg/runner/writer_markdown_test.go | 1 + pkg/runner/writer_std.go | 27 ++++++- pkg/runner/writer_std_test.go | 1 + 13 files changed, 324 insertions(+), 6 deletions(-) create mode 100644 pkg/apispec/swagger.go create mode 100644 pkg/apispec/swagger_test.go create mode 100644 pkg/apispec/testdata/swagger.json diff --git a/cmd/function.go b/cmd/function.go index 46593c71c..6be2c8489 100644 --- a/cmd/function.go +++ b/cmd/function.go @@ -1,7 +1,6 @@ package cmd import ( - "fmt" "go/ast" "go/doc" "go/parser" @@ -26,7 +25,7 @@ func createFunctionCmd() (c *cobra.Command) { cmd.Println(reflect.TypeOf(fn)) desc := FuncDescription(fn) if desc != "" { - fmt.Println(desc) + cmd.Println(desc) } } else { cmd.Println("No such function") diff --git a/cmd/run.go b/cmd/run.go index 56a4403b9..0902acd21 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/linuxsuren/api-testing/pkg/apispec" "github.com/linuxsuren/api-testing/pkg/limit" "github.com/linuxsuren/api-testing/pkg/render" "github.com/linuxsuren/api-testing/pkg/runner" @@ -35,6 +36,7 @@ type runOption struct { reportWriter runner.ReportResultWriter report string reportIgnore bool + swaggerURL string level string caseItems []string } @@ -77,6 +79,7 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`, flags.StringVarP(&opt.report, "report", "", "", "The type of target report. Supported: markdown, md, html, discard, std") flags.StringVarP(&opt.reportFile, "report-file", "", "", "The file path of the report") flags.BoolVarP(&opt.reportIgnore, "report-ignore", "", false, "Indicate if ignore the report output") + flags.StringVarP(&opt.swaggerURL, "swagger-url", "", "", "The URL of swagger") flags.Int64VarP(&opt.thread, "thread", "", 1, "Threads of the execution") flags.Int32VarP(&opt.qps, "qps", "", 5, "QPS") flags.Int32VarP(&opt.burst, "burst", "", 5, "burst") @@ -108,6 +111,15 @@ func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) { err = fmt.Errorf("not supported report type: '%s'", o.report) } + if err == nil { + var swaggerAPI apispec.APIConverage + if o.swaggerURL != "" { + if swaggerAPI, err = apispec.ParseURLToSwagger(o.swaggerURL); err == nil { + o.reportWriter.WithAPIConverage(swaggerAPI) + } + } + } + o.caseItems = args return } diff --git a/cmd/run_test.go b/cmd/run_test.go index 516eec20e..4e0e0a64e 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -115,6 +115,14 @@ func TestRunCommand(t *testing.T) { prepare: fooPrepare, args: []string{"-p", simpleSuite, "--report", "md", "--report-file", tmpFile.Name()}, hasErr: false, + }, { + name: "report with swagger URL", + prepare: func() { + fooPrepare() + fooPrepare() + }, + args: []string{"-p", simpleSuite, "--swagger-url", urlFoo + "/bar"}, + hasErr: false, }, { name: "report file with error", prepare: fooPrepare, @@ -124,9 +132,10 @@ func TestRunCommand(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { defer gock.Clean() + buf := new(bytes.Buffer) util.MakeSureNotNil(tt.prepare)() root := &cobra.Command{Use: "root"} - root.SetOut(&bytes.Buffer{}) + root.SetOut(buf) root.AddCommand(createRunCommand()) root.SetArgs(append([]string{"run"}, tt.args...)) @@ -184,6 +193,15 @@ func TestPreRunE(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, ro.reportWriter) }, + }, { + name: "html report", + opt: &runOption{ + report: "html", + }, + verify: func(t *testing.T, ro *runOption, err error) { + assert.Nil(t, err) + assert.NotNil(t, ro.reportWriter) + }, }, { name: "empty report", opt: &runOption{ diff --git a/pkg/apispec/swagger.go b/pkg/apispec/swagger.go new file mode 100644 index 000000000..f5e3d7ee9 --- /dev/null +++ b/pkg/apispec/swagger.go @@ -0,0 +1,98 @@ +package apispec + +import ( + "encoding/json" + "io" + "net/http" + "regexp" + "strings" +) + +type Swagger struct { + Swagger string `json:"swagger"` + Paths map[string]map[string]SwaggerAPI `json:"paths"` + Info SwaggerInfo `json:"info"` +} + +type SwaggerAPI struct { + OperationId string `json:"operationId"` + Summary string `json:"summary"` +} + +type SwaggerInfo struct { + Description string `json:"description"` + Title string `json:"title"` + Version string `json:"version"` +} + +type APIConverage interface { + HaveAPI(path, method string) (exist bool) + APICount() (count int) +} + +// HaveAPI check if the swagger has the API. +// If the path is /api/v1/names/linuxsuren, then will match /api/v1/names/{name} +func (s *Swagger) HaveAPI(path, method string) (exist bool) { + method = strings.ToLower(method) + for item := range s.Paths { + if matchAPI(path, item) { + for m := range s.Paths[item] { + if strings.ToLower(m) == method { + exist = true + return + } + } + } + } + return +} + +func matchAPI(particularAPI, swaggerAPI string) (matched bool) { + result := swaggerAPIConvert(swaggerAPI) + reg, err := regexp.Compile(result) + if err == nil { + matched = reg.MatchString(particularAPI) + } + return +} + +func swaggerAPIConvert(text string) (result string) { + result = text + reg, err := regexp.Compile("{.*}") + if err == nil { + result = reg.ReplaceAllString(text, ".*") + } + return +} + +// APICount return the count of APIs +func (s *Swagger) APICount() (count int) { + for path := range s.Paths { + for range s.Paths[path] { + count++ + } + } + return +} + +func ParseToSwagger(data []byte) (swagger *Swagger, err error) { + swagger = &Swagger{} + err = json.Unmarshal(data, swagger) + return +} + +func ParseURLToSwagger(swaggerURL string) (swagger *Swagger, err error) { + var resp *http.Response + if resp, err = http.Get(swaggerURL); err == nil && resp != nil && resp.StatusCode == http.StatusOK { + swagger, err = ParseStreamToSwagger(resp.Body) + } + return +} + +func ParseStreamToSwagger(stream io.Reader) (swagger *Swagger, err error) { + var data []byte + if data, err = io.ReadAll(stream); err == nil { + swagger, err = ParseToSwagger(data) + } + return +} diff --git a/pkg/apispec/swagger_test.go b/pkg/apispec/swagger_test.go new file mode 100644 index 000000000..692eb1887 --- /dev/null +++ b/pkg/apispec/swagger_test.go @@ -0,0 +1,111 @@ +package apispec_test + +import ( + "net/http" + "testing" + + _ "embed" + + "github.com/h2non/gock" + "github.com/linuxsuren/api-testing/pkg/apispec" + "github.com/stretchr/testify/assert" +) + +func TestParseURLToSwagger(t *testing.T) { + tests := []struct { + name string + swaggerURL string + verify func(t *testing.T, swagger *apispec.Swagger, err error) + }{{ + name: "normal", + swaggerURL: "http://foo", + verify: func(t *testing.T, swagger *apispec.Swagger, err error) { + assert.NoError(t, err) + assert.Equal(t, "2.0", swagger.Swagger) + assert.Equal(t, apispec.SwaggerInfo{ + Description: "sample", + Title: "sample", + Version: "1.0.0", + }, swagger.Info) + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gock.New(tt.swaggerURL).Get("/").Reply(200).BodyString(testdataSwaggerJSON) + defer gock.Off() + + s, err := apispec.ParseURLToSwagger(tt.swaggerURL) + tt.verify(t, s, err) + }) + } +} + +func TestHaveAPI(t *testing.T) { + tests := []struct { + name string + swaggerURL string + path, method string + expectExist bool + }{{ + name: "normal, exist", + swaggerURL: "http://foo", + path: "/api/v1/users", + method: http.MethodGet, + expectExist: true, + }, { + name: "create user, exist", + swaggerURL: "http://foo", + path: "/api/v1/users", + method: http.MethodPost, + expectExist: true, + }, { + name: "get a user, exist", + swaggerURL: "http://foo", + path: "/api/v1/users/linuxsuren", + method: http.MethodGet, + expectExist: true, + }, { + name: "normal, not exist", + swaggerURL: "http://foo", + path: "/api/v1/users", + method: http.MethodDelete, + expectExist: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gock.New(tt.swaggerURL).Get("/").Reply(200).BodyString(testdataSwaggerJSON) + defer gock.Off() + + swagger, err := apispec.ParseURLToSwagger(tt.swaggerURL) + assert.NoError(t, err) + exist := swagger.HaveAPI(tt.path, tt.method) + assert.Equal(t, tt.expectExist, exist) + }) + } +} + +func TestAPICount(t *testing.T) { + tests := []struct { + name string + swaggerURL string + expectCount int + }{{ + name: "normal", + swaggerURL: "http://foo", + expectCount: 5, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gock.New(tt.swaggerURL).Get("/").Reply(200).BodyString(testdataSwaggerJSON) + defer gock.Off() + + swagger, err := apispec.ParseURLToSwagger(tt.swaggerURL) + assert.NoError(t, err) + count := swagger.APICount() + assert.Equal(t, tt.expectCount, count) + }) + } +} + +//go:embed testdata/swagger.json +var testdataSwaggerJSON string diff --git a/pkg/apispec/testdata/swagger.json b/pkg/apispec/testdata/swagger.json new file mode 100644 index 000000000..391c3a875 --- /dev/null +++ b/pkg/apispec/testdata/swagger.json @@ -0,0 +1,34 @@ +{ + "swagger": "2.0", + "info": { + "description": "sample", + "title": "sample", + "version": "1.0.0" + }, + "paths": { + "/api/v1/users": { + "get": { + "summary": "summary", + "operationId": "getUsers" + }, + "post": { + "summary": "summary", + "operationId": "createUser" + } + }, + "/api/v1/users/{user}": { + "get": { + "summary": "summary", + "operationId": "getUser" + }, + "delete": { + "summary": "summary", + "operationId": "deleteUser" + }, + "put": { + "summary": "summary", + "operationId": "updateUser" + } + } + } +} diff --git a/pkg/runner/simple.go b/pkg/runner/simple.go index 8006e39b5..e3323af00 100644 --- a/pkg/runner/simple.go +++ b/pkg/runner/simple.go @@ -14,6 +14,7 @@ import ( "github.com/andreyvit/diff" "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/linuxsuren/api-testing/pkg/apispec" "github.com/linuxsuren/api-testing/pkg/runner/kubernetes" "github.com/linuxsuren/api-testing/pkg/testing" fakeruntime "github.com/linuxsuren/go-fake-runtime" @@ -153,6 +154,7 @@ func (r ReportResultSlice) Swap(i, j int) { // ReportResultWriter is the interface of the report writer type ReportResultWriter interface { Output([]ReportResult) error + WithAPIConverage(apiConverage apispec.APIConverage) ReportResultWriter } // TestReporter is the interface of the report diff --git a/pkg/runner/writer_html.go b/pkg/runner/writer_html.go index a1d181cf6..8957381cb 100644 --- a/pkg/runner/writer_html.go +++ b/pkg/runner/writer_html.go @@ -4,11 +4,13 @@ import ( _ "embed" "io" + "github.com/linuxsuren/api-testing/pkg/apispec" "github.com/linuxsuren/api-testing/pkg/render" ) type htmlResultWriter struct { - writer io.Writer + writer io.Writer + apiConverage apispec.APIConverage } // NewHTMLResultWriter creates a new htmlResultWriter @@ -21,5 +23,11 @@ func (w *htmlResultWriter) Output(result []ReportResult) (err error) { return render.RenderThenPrint("html-report", htmlReport, result, w.writer) } +// WithAPIConverage sets the api coverage +func (w *htmlResultWriter) WithAPIConverage(apiConverage apispec.APIConverage) ReportResultWriter { + w.apiConverage = apiConverage + return w +} + //go:embed data/html.html var htmlReport string diff --git a/pkg/runner/writer_html_test.go b/pkg/runner/writer_html_test.go index 341355430..4fa49fcf7 100644 --- a/pkg/runner/writer_html_test.go +++ b/pkg/runner/writer_html_test.go @@ -32,6 +32,7 @@ func TestHTMLResultWriter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { w := runner.NewHTMLResultWriter(tt.buf) + w.WithAPIConverage(nil) err := w.Output(tt.results) assert.NoError(t, err) assert.Equal(t, tt.expect, tt.buf.String()) diff --git a/pkg/runner/writer_markdown.go b/pkg/runner/writer_markdown.go index 69d24f5fe..14bb8b4be 100644 --- a/pkg/runner/writer_markdown.go +++ b/pkg/runner/writer_markdown.go @@ -4,11 +4,13 @@ import ( _ "embed" "io" + "github.com/linuxsuren/api-testing/pkg/apispec" "github.com/linuxsuren/api-testing/pkg/render" ) type markdownResultWriter struct { - writer io.Writer + writer io.Writer + apiConverage apispec.APIConverage } // NewMarkdownResultWriter creates the Markdown writer @@ -21,5 +23,11 @@ func (w *markdownResultWriter) Output(result []ReportResult) (err error) { return render.RenderThenPrint("md-report", markdownReport, result, w.writer) } +// WithAPIConverage sets the api coverage +func (w *markdownResultWriter) WithAPIConverage(apiConverage apispec.APIConverage) ReportResultWriter { + w.apiConverage = apiConverage + return w +} + //go:embed data/report.md var markdownReport string diff --git a/pkg/runner/writer_markdown_test.go b/pkg/runner/writer_markdown_test.go index 1a152f90f..5cff1e202 100644 --- a/pkg/runner/writer_markdown_test.go +++ b/pkg/runner/writer_markdown_test.go @@ -11,6 +11,7 @@ import ( func TestMarkdownWriter(t *testing.T) { buf := new(bytes.Buffer) writer := runner.NewMarkdownResultWriter(buf) + writer.WithAPIConverage(nil) err := writer.Output([]runner.ReportResult{{ API: "api", diff --git a/pkg/runner/writer_std.go b/pkg/runner/writer_std.go index 2a34bd0d5..1b71db0c4 100644 --- a/pkg/runner/writer_std.go +++ b/pkg/runner/writer_std.go @@ -4,10 +4,13 @@ import ( _ "embed" "fmt" "io" + + "github.com/linuxsuren/api-testing/pkg/apispec" ) type stdResultWriter struct { - writer io.Writer + writer io.Writer + apiConverage apispec.APIConverage } // NewResultWriter creates a result writer with the specific io.Writer @@ -35,5 +38,27 @@ func (w *stdResultWriter) Output(results []ReportResult) error { for _, r := range errResults { fmt.Fprintf(w.writer, "%s error: %s\n", r.API, r.LastErrorMessage) } + + apiConveragePrint(results, w.apiConverage, w.writer) return nil } + +// WithAPIConverage sets the api coverage +func (w *stdResultWriter) WithAPIConverage(apiConverage apispec.APIConverage) ReportResultWriter { + w.apiConverage = apiConverage + return w +} + +func apiConveragePrint(result []ReportResult, apiConverage apispec.APIConverage, w io.Writer) { + if apiConverage == nil { + return + } + + var covered int + for _, item := range result { + if apiConverage.HaveAPI(item.API, "GET") { + covered++ + } + } + fmt.Fprintf(w, "\nAPI Coverage: %d/%d\n", covered, apiConverage.APICount()) +} diff --git a/pkg/runner/writer_std_test.go b/pkg/runner/writer_std_test.go index f7d2c40e7..9a7561e6d 100644 --- a/pkg/runner/writer_std_test.go +++ b/pkg/runner/writer_std_test.go @@ -76,6 +76,7 @@ api 1ns 1ns 1ns 10 1 0 return } + writer.WithAPIConverage(nil) err := writer.Output(tt.results) assert.Nil(t, err) assert.Equal(t, tt.expect, tt.buf.String()) From 5ba30aede790b29f58f9db3a558cd0397274bbd4 Mon Sep 17 00:00:00 2001 From: rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:23:49 +0800 Subject: [PATCH 2/2] feat: add brace expansion support --- README.md | 2 ++ cmd/run.go | 24 ++++++++++++++++-------- pkg/util/expand.go | 25 +++++++++++++++++++++++++ pkg/util/expand_test.go | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 pkg/util/expand.go create mode 100644 pkg/util/expand_test.go diff --git a/README.md b/README.md index 78e2cad52..84a0c4284 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,13 @@ Usage: Available Commands: completion Generate the autocompletion script for the specified shell + func Print all the supported functions help Help about any command json Print the JSON schema of the test suites struct run Run the test suite sample Generate a sample test case YAML file server Run as a server mode + service Install atest as a Linux service Flags: -h, --help help for atest diff --git a/cmd/run.go b/cmd/run.go index 0902acd21..37c76d592 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -16,6 +16,7 @@ import ( "github.com/linuxsuren/api-testing/pkg/render" "github.com/linuxsuren/api-testing/pkg/runner" "github.com/linuxsuren/api-testing/pkg/testing" + "github.com/linuxsuren/api-testing/pkg/util" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" ) @@ -71,7 +72,7 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`, // set flags flags := cmd.Flags() flags.StringVarP(&opt.pattern, "pattern", "p", "test-suite-*.yaml", - "The file pattern which try to execute the test cases") + "The file pattern which try to execute the test cases. Brace expansion is supported, such as: test-suite-{1,2}.yaml") flags.StringVarP(&opt.level, "level", "l", "info", "Set the output log level") flags.DurationVarP(&opt.duration, "duration", "", 0, "Running duration") flags.DurationVarP(&opt.requestTimeout, "request-timeout", "", time.Minute, "Timeout for per request") @@ -125,7 +126,6 @@ func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) { } func (o *runOption) runE(cmd *cobra.Command, args []string) (err error) { - var files []string o.startTime = time.Now() o.context = cmd.Context() o.limiter = limit.NewDefaultRateLimiter(o.qps, o.burst) @@ -134,12 +134,20 @@ func (o *runOption) runE(cmd *cobra.Command, args []string) (err error) { o.limiter.Stop() }() - if files, err = filepath.Glob(o.pattern); err == nil { - for i := range files { - item := files[i] - if err = o.runSuiteWithDuration(item); err != nil { - break - } + var suites []string + for _, pattern := range util.Expand(o.pattern) { + var files []string + if files, err = filepath.Glob(pattern); err == nil { + suites = append(suites, files...) + } + } + + cmd.Println("found suites:", len(suites)) + for i := range suites { + item := suites[i] + cmd.Println("run suite:", item) + if err = o.runSuiteWithDuration(item); err != nil { + break } } diff --git a/pkg/util/expand.go b/pkg/util/expand.go new file mode 100644 index 000000000..88357b7e6 --- /dev/null +++ b/pkg/util/expand.go @@ -0,0 +1,25 @@ +package util + +import ( + "regexp" + "strings" +) + +// Expand the text with brace syntax. +// Such as: /home/{good,bad} -> [/home/good, /home/bad] +func Expand(text string) (result []string) { + reg := regexp.MustCompile(`\{.*\}`) + if reg.MatchString(text) { + brace := reg.FindString(text) + braceItem := strings.TrimPrefix(brace, "{") + braceItem = strings.TrimSuffix(braceItem, "}") + items := strings.Split(braceItem, ",") + + for _, item := range items { + result = append(result, strings.ReplaceAll(text, brace, item)) + } + } else { + result = []string{text} + } + return +} diff --git a/pkg/util/expand_test.go b/pkg/util/expand_test.go new file mode 100644 index 000000000..ca1d60659 --- /dev/null +++ b/pkg/util/expand_test.go @@ -0,0 +1,34 @@ +package util_test + +import ( + "testing" + + "github.com/linuxsuren/api-testing/pkg/util" + "github.com/stretchr/testify/assert" +) + +func TestExpand(t *testing.T) { + tests := []struct { + name string + input string + expect []string + }{{ + name: "without brace", + input: "/home", + expect: []string{"/home"}, + }, { + name: "with brace", + input: "/home/{good,bad}", + expect: []string{"/home/good", "/home/bad"}, + }, { + name: "with brace, have suffix", + input: "/home/{good,bad}.yaml", + expect: []string{"/home/good.yaml", "/home/bad.yaml"}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := util.Expand(tt.input) + assert.Equal(t, tt.expect, got, got) + }) + } +}