feature: refactor api parse to g4 (zeromicro#365)

* feature: refactor api parse to g4

* new g4 parser

* add CHANGE_LOG.MD

* refactor

* fix byte bug

* refactor

* optimized

* optimized

* revert

* update readme.md

* update readme.md

* update readme.md

* update readme.md

* remove no need

* fix java gen

* add upgrade

* resolve confilits

Co-authored-by: anqiansong <anqiansong@xiaoheiban.cn>

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/ClickHouse/clickhouse-go v1.4.3
 	github.com/DATA-DOG/go-sqlmock v1.4.1
 	github.com/alicebob/miniredis/v2 v2.14.1
+	github.com/antlr/antlr4 v0.0.0-20210105212045-464bcbc32de2
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/dchest/siphash v1.2.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible

go.sum

@@ -16,6 +16,8 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGn
 github.com/alicebob/miniredis/v2 v2.14.1 h1:GjlbSeoJ24bzdLRs13HoMEeaRZx9kg5nHoRW7QV/nCs=
 github.com/alicebob/miniredis/v2 v2.14.1/go.mod h1:uS970Sw5Gs9/iK3yBg0l9Uj9s25wXxSpQUE9EaJ/Blg=
 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
+github.com/antlr/antlr4 v0.0.0-20210105212045-464bcbc32de2 h1:rL2miklL5rhxUaZO7hntBcy/VHaiyuPQ4EJoy/NMwaM=
+github.com/antlr/antlr4 v0.0.0-20210105212045-464bcbc32de2/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

tools/goctl/CHANGE_LOG.MD

@@ -0,0 +1,15 @@
+# 2020-01-08
+
+## features ![](https://img.shields.io/static/v1?label=&message=new&color=red)
+
+* reactor api parse by g4
+* add syntax lexer for api
+* support java-style documentation comments
+* support parsing doc and comment
+* support import group
+
+> original: [api grammar document](./api/parser/readme.md)
+
+# 2020-01-08
+
+* add change log

tools/goctl/api/apigen/gen.go

@@ -12,18 +12,21 @@ import (
 	"github.com/urfave/cli"
 )
 
-const apiTemplate = `info(
+const apiTemplate = `
+syntax = "v1"
+
+info(
 	title: // TODO: add title
 	desc: // TODO: add description
 	author: "{{.gitUser}}"
 	email: "{{.gitEmail}}"
 )
 
-type request struct {
+type request {
 	// TODO: add members here and delete this comment
 }
 
-type response struct {
+type response {
 	// TODO: add members here and delete this comment
 }
 

tools/goctl/api/dartgen/gen.go

@@ -19,11 +19,7 @@ func DartCommand(c *cli.Context) error {
 		return errors.New("missing -dir")
 	}
 
-	p, err := parser.NewParser(apiFile)
-	if err != nil {
-		return err
-	}
-	api, err := p.Parse()
+	api, err := parser.Parse(apiFile)
 	if err != nil {
 		return err
 	}

tools/goctl/api/dartgen/gendata.go

@@ -59,7 +59,6 @@ func genData(dir string, api *spec.ApiSpec) error {
 		return e
 	}
 
-	convertMemberType(api)
 	return t.Execute(file, api)
 }
 

tools/goctl/api/dartgen/util.go

@@ -1,12 +1,10 @@
 package dartgen
 
 import (
-	"log"
 	"os"
 	"reflect"
 	"strings"
 
-	"github.com/tal-tech/go-zero/tools/goctl/api/spec"
 	"github.com/tal-tech/go-zero/tools/goctl/api/util"
 )
 
@@ -39,47 +37,6 @@ func tagGet(tag, k string) (reflect.Value, error) {
 	return reflect.ValueOf(out), nil
 }
 
-func convertMemberType(api *spec.ApiSpec) {
-	for i, t := range api.Types {
-		for j, mem := range t.Members {
-			api.Types[i].Members[j].Type = goTypeToDart(mem.Type)
-		}
-	}
-}
-
-func goTypeToDart(t string) string {
-	t = strings.Replace(t, "*", "", -1)
-	if strings.HasPrefix(t, "[]") {
-		return "List<" + goTypeToDart(t[2:]) + ">"
-	}
-
-	if strings.HasPrefix(t, "map") {
-		tys, e := util.DecomposeType(t)
-		if e != nil {
-			log.Fatal(e)
-		}
-
-		if len(tys) != 2 {
-			log.Fatal("Map type number !=2")
-		}
-
-		return "Map<String," + goTypeToDart(tys[1]) + ">"
-	}
-
-	switch t {
-	case "string":
-		return "String"
-	case "int", "int32", "int64":
-		return "int"
-	case "float", "float32", "float64":
-		return "double"
-	case "bool":
-		return "bool"
-	default:
-		return t
-	}
-}
-
 func isDirectType(s string) bool {
 	return isAtomicType(s) || isListType(s) && isAtomicType(getCoreType(s))
 }

tools/goctl/api/docgen/doc.go

@@ -41,7 +41,7 @@ func genDoc(api *spec.ApiSpec, dir string, filename string) error {
 
 	var builder strings.Builder
 	for index, route := range api.Service.Routes() {
-		routeComment, _ := util.GetAnnotationValue(route.Annotations, "doc", "summary")
+		routeComment := route.JoinedDoc()
 		if len(routeComment) == 0 {
 			routeComment = "N/A"
 		}
@@ -58,8 +58,8 @@ func genDoc(api *spec.ApiSpec, dir string, filename string) error {
 			"routeComment":    routeComment,
 			"method":          strings.ToUpper(route.Method),
 			"uri":             route.Path,
-			"requestType":     "`" + stringx.TakeOne(route.RequestType.Name, "-") + "`",
-			"responseType":    "`" + stringx.TakeOne(route.ResponseType.Name, "-") + "`",
+			"requestType":     "`" + stringx.TakeOne(route.RequestTypeName(), "-") + "`",
+			"responseType":    "`" + stringx.TakeOne(route.ResponseTypeName(), "-") + "`",
 			"responseContent": responseContent,
 		})
 		if err != nil {
@@ -73,10 +73,28 @@ func genDoc(api *spec.ApiSpec, dir string, filename string) error {
 }
 
 func responseBody(api *spec.ApiSpec, route spec.Route) (string, error) {
-	tps := util.GetLocalTypes(api, route)
+	if len(route.ResponseTypeName()) == 0 {
+		return "", nil
+	}
+
+	var tps = make([]spec.Type, 0)
+	tps = append(tps, route.ResponseType)
+	if definedType, ok := route.ResponseType.(spec.DefineStruct); ok {
+		associatedTypes(definedType, &tps)
+	}
 	value, err := gogen.BuildTypes(tps)
 	if err != nil {
 		return "", err
 	}
+
 	return fmt.Sprintf("\n\n```golang\n%s\n```\n", value), nil
 }
+
+func associatedTypes(tp spec.DefineStruct, tps *[]spec.Type) {
+	*tps = append(*tps, tp)
+	for _, item := range tp.Members {
+		if definedType, ok := item.Type.(spec.DefineStruct); ok {
+			associatedTypes(definedType, tps)
+		}
+	}
+}

tools/goctl/api/docgen/gen.go

@@ -29,14 +29,11 @@ func DocCommand(c *cli.Context) error {
 		return err
 	}
 	for _, f := range files {
-		p, err := parser.NewParser(f)
+		api, err := parser.Parse(f)
 		if err != nil {
 			return errors.New(fmt.Sprintf("parse file: %s, err: %s", f, err.Error()))
 		}
-		api, err := p.Parse()
-		if err != nil {
-			return err
-		}
+
 		index := strings.Index(f, dir)
 		if index < 0 {
 			continue

tools/goctl/api/format/format.go

@@ -97,7 +97,7 @@ func ApiFormatByPath(apiFilePath string) error {
 }
 
 func apiFormat(data string) (string, error) {
-	_, err := parser.ParseApi(data)
+	_, err := parser.ParseContent(data)
 	if err != nil {
 		return "", err
 	}
@@ -208,5 +208,9 @@ func mayInsertStructKeyword(line string, token *int) string {
 	if strings.HasSuffix(noCommentLine, leftParenthesis) {
 		*token++
 	}
+
+	if strings.Contains(noCommentLine, "`") {
+		return util.UpperFirst(strings.TrimSpace(line))
+	}
 	return line
 }

tools/goctl/api/format/format_test.go

@@ -9,13 +9,11 @@ import (
 const (
 	notFormattedStr = `
 type Request struct {
-  Name string 
+  Name string ` + "`" + `path:"name,options=you|me"` + "`" + `  
 }
-
 type Response struct {
-  Message string
+  Message string ` + "`" + `json:"message"` + "`" + `
 }
-
 service A-api {
 @server(
 handler: GreetHandler
@@ -25,13 +23,11 @@ handler: GreetHandler
 `
 
 	formattedStr = `type Request {
-	Name string
+	Name string ` + "`" + `path:"name,options=you|me"` + "`" + `
 }
-
 type Response {
-	Message string
+	Message string ` + "`" + `json:"message"` + "`" + `
 }
-
 service A-api {
 	@server(
 		handler: GreetHandler

tools/goctl/api/gogen/gen.go

@@ -41,11 +41,7 @@ func GoCommand(c *cli.Context) error {
 }
 
 func DoGenProject(apiFile, dir, style string) error {
-	p, err := parser.NewParser(apiFile)
-	if err != nil {
-		return err
-	}
-	api, err := p.Parse()
+	api, err := parser.Parse(apiFile)
 	if err != nil {
 		return err
 	}