From d0018c3142e8f8e347915cfa1b8c89d1de0b3bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=AF=E9=A3=8E?= Date: Tue, 16 Jun 2020 11:25:48 +0800 Subject: [PATCH] 1.Improve the template project directory structure 2.Increase the Protoc toolset 2.1 Support to generate Pb.go files 2.2 Support to generate GRPC server implementations from proto files --- go.mod | 1 + ...15\347\275\256\350\257\264\346\230\216.md" | 7 + ...45\345\277\227\345\237\213\347\202\271.md" | 86 ++++++ tools/jupiter/README.MD | 272 ++++++++++++++++- tools/jupiter/common/util.go | 35 +++ tools/jupiter/common/util_test.go | 35 +++ tools/jupiter/main.go | 21 +- tools/jupiter/new/new_cmd.go | 20 ++ tools/jupiter/new/option.go | 3 +- tools/jupiter/new/templates/.gitignore | 8 + tools/jupiter/new/templates/README.md | 278 +++++++++++++++++- tools/jupiter/new/templates/cmd/main.go.tmpl | 15 +- .../jupiter/new/templates/config/config.toml | 16 +- .../internal/app/engine/engine.go.tmpl | 44 ++- .../grpc}/greeter/greeter.go.tmpl | 0 .../internal/app/handler/user.go.tmpl | 20 ++ .../templates/internal/app/model/init.go.tmpl | 11 + .../internal/app/service/init.go.tmpl | 15 + .../app/service/user/impl/mysqlimpl.go.tmpl | 21 ++ .../app/service/user/repository.go.tmpl | 9 + .../new/templates/pb/hello/hello.proto | 14 + tools/jupiter/new/templates/sql/user.sql | 9 + tools/jupiter/new/tools.go | 58 ++-- tools/jupiter/protoc/grpc.go | 74 +++++ tools/jupiter/protoc/option.go | 14 + tools/jupiter/protoc/protoc_cmd.go | 42 +++ tools/jupiter/protoc/server.go | 133 +++++++++ tools/jupiter/protoc/template/server_tmpl.go | 22 ++ tools/jupiter/protoc/tools.go | 41 +++ tools/jupiter/protoc/usage.go | 37 +++ tools/jupiter/protoc/util.go | 19 ++ tools/jupiter/protoc/util_test.go | 51 ++++ 32 files changed, 1343 insertions(+), 88 deletions(-) create mode 100644 "tools/ast_codes/\351\205\215\347\275\256\350\257\264\346\230\216.md" create mode 100644 "tools/ast_codes/\351\224\231\350\257\257\346\227\245\345\277\227\345\237\213\347\202\271.md" create mode 100644 tools/jupiter/common/util.go create mode 100644 tools/jupiter/common/util_test.go create mode 100644 tools/jupiter/new/new_cmd.go create mode 100644 tools/jupiter/new/templates/.gitignore rename tools/jupiter/new/templates/internal/{demo => app/grpc}/greeter/greeter.go.tmpl (100%) create mode 100644 tools/jupiter/new/templates/internal/app/handler/user.go.tmpl create mode 100644 tools/jupiter/new/templates/internal/app/model/init.go.tmpl create mode 100644 tools/jupiter/new/templates/internal/app/service/init.go.tmpl create mode 100644 tools/jupiter/new/templates/internal/app/service/user/impl/mysqlimpl.go.tmpl create mode 100644 tools/jupiter/new/templates/internal/app/service/user/repository.go.tmpl create mode 100644 tools/jupiter/new/templates/pb/hello/hello.proto create mode 100644 tools/jupiter/new/templates/sql/user.sql create mode 100644 tools/jupiter/protoc/grpc.go create mode 100644 tools/jupiter/protoc/option.go create mode 100644 tools/jupiter/protoc/protoc_cmd.go create mode 100644 tools/jupiter/protoc/server.go create mode 100644 tools/jupiter/protoc/template/server_tmpl.go create mode 100644 tools/jupiter/protoc/tools.go create mode 100644 tools/jupiter/protoc/usage.go create mode 100644 tools/jupiter/protoc/util.go create mode 100644 tools/jupiter/protoc/util_test.go diff --git a/go.mod b/go.mod index fb14667d67..a541311d51 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/davecgh/go-spew v1.1.1 + github.com/emicklei/proto v1.9.0 github.com/fatih/structtag v1.2.0 github.com/flosch/pongo2 v0.0.0-20200518135938-dfb43dbdc22a github.com/fsnotify/fsnotify v1.4.9 diff --git "a/tools/ast_codes/\351\205\215\347\275\256\350\257\264\346\230\216.md" "b/tools/ast_codes/\351\205\215\347\275\256\350\257\264\346\230\216.md" new file mode 100644 index 0000000000..eb0d1f0cdd --- /dev/null +++ "b/tools/ast_codes/\351\205\215\347\275\256\350\257\264\346\230\216.md" @@ -0,0 +1,7 @@ + +# mienrva 配置结构列表 + +> 本文档是由github.com/douyu/jupiter/script/ast_codes自动生成 +> 配置格式有变更时，请执行: cd script/ast_codes; go run main.go --dir=../../; cd - + + diff --git "a/tools/ast_codes/\351\224\231\350\257\257\346\227\245\345\277\227\345\237\213\347\202\271.md" "b/tools/ast_codes/\351\224\231\350\257\257\346\227\245\345\277\227\345\237\213\347\202\271.md" new file mode 100644 index 0000000000..24c09442af --- /dev/null +++ "b/tools/ast_codes/\351\224\231\350\257\257\346\227\245\345\277\227\345\237\213\347\202\271.md" @@ -0,0 +1,86 @@ + + +## pkg/client/redis +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| addr empty stub config | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\redis\config.go#L110)| +| cluster addr empty stub config | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\redis\config.go#L133)| +| start redis | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\redis\redis.go#L47)| +| start cluster redis | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\redis\cluster.go#L50)| +| start redis | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\redis\redis.go#L49)| +| start cluster redis | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\redis\cluster.go#L52)| + + +## pkg/server/xecho +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| access | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xecho\middleware.go#L67)| +| access | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xecho\middleware.go#L70)| +| echo add route | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xecho\server.go#L47)| +| http server parse config panic | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xecho\config.go#L60)| + + +## pkg/server/xgrpc +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| grpc server parse config panic | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xgrpc\config.go#L49)| +| new grpc server err | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xgrpc\server.go#L46)| +| grpc server recover | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\server\xgrpc\interceptor.go#L174)| + + +## pkg/store/gorm +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| mysql err | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\interceptor.go#L42)| +| slow | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\interceptor.go#L49)| +| | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\config.go#L136)| +| record not found | Warn|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\interceptor.go#L44)| +| open mysql | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\config.go#L122)| +| ping mysql | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\config.go#L126)| +| | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\store\gorm\config.go#L134)| + + +## ./ +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| stop register close err | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L175)| +| stop governor close err | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L179)| +| graceful stop register close err | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L201)| +| graceful stop governor close err | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L205)| +| leaving jupiter, bye.... | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L224)| +| start governor | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L228)| +| start servers | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L246)| +| exit server | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L247)| +| load remote config | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L322)| +| load local file | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L329)| +| auto max procs | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L367)| +| start governor | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L231)| +| load remote config | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L320)| +| load local file | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L327)| +| auto max procs | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L363)| +| no config ... | Warn|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\jupiter.go#L312)| + + +## pkg/client/etcdv3 +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| handle watch update | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\watch.go#L49)| +| handle watch block | Warn|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\watch.go#L57)| +| dial etcd server | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\client.go#L114)| +| client etcd parse config panic | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\config.go#L64)| +| client etcd endpoints empty | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\client.go#L58)| +| parse CaCert failed | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\client.go#L78)| +| load CertFile or KeyFile failed | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\client.go#L93)| +| client etcd start panic | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\etcdv3\client.go#L106)| + + +## pkg/client/grpc +| 错误 | 级别 | 行数 | +|:--------------|:-----|:-------------------| +| dial grpc server | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\grpc\client.go#L53)| +| client grpc parse config panic | Panic|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\grpc\config.go#L69)| +| dial grpc server | Error|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\grpc\client.go#L55)| +| start grpc client | Info|[代码地址](https://github.com/douyu/jupiter/blob/master/\..\pkg\client\grpc\client.go#L58)| + + + diff --git a/tools/jupiter/README.MD b/tools/jupiter/README.MD index 6697091897..13306e0c18 100644 --- a/tools/jupiter/README.MD +++ b/tools/jupiter/README.MD @@ -1,14 +1,272 @@ # jupiter -## Project introduction -jupiter Scaffold tools +## 脚手架介绍 +### jupiter脚手架工具集 +1. 快速生成模板项目 +2. 基于proto文件生成pb.go +3. 基于proto文件生成服务端实现 # go version - GO > 1.14 + GO >= 1.13 -## install +## 脚手架工具获取 +windows : +```shell script +set GO111MODULE=on +set GOPROXY=https://mirrors.aliyun.com/goproxy/,direct +``` +linux : +```shell script +export GO111MODULE=on +export GOPROXY=https://mirrors.aliyun.com/goproxy/,direct +``` +`go get -u -v github.com/douyu/jupiter/tools/jupiter` +* windows 用户: + 会在${GOPATH}/src/bin 目录下生成jupiter.exe 文件,若想方便的在任何地方使用jupiter命令,请将该 + 命令配置在系统的环境变量中 +* Linux 用户: +* Mac 用户 : + + +## 如何使用 +* jupiter -h +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>jupiter -h +NAME: + jupiter - jupiter tools + +USAGE: + jupiter [global options] command [command options] [arguments...] + +VERSION: + 0.1.0 + +COMMANDS: + new, n Create Jupiter template project + protoc, p jupiter protoc tools + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --help, -h show help + --version, -v print the version +``` +* jupiter new -h + +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>jupiter new -h +NAME: + jupiter new - Create Jupiter template project + +USAGE: + +jupiter [commands|flags] + +The commands & flags are: + new Create Jupiter template project + + -d Build the specified directory for the template project + +Examples: + # Build the specified directory for the template project + jupiter new (your project name) -d (project dir) +``` +* jupiter protoc -h +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>jupiter protoc -h +NAME: + jupiter protoc - jupiter protoc tools + +USAGE: + +jupiter [commands|flags] + +The commands & flags are: + protoc jupiter protoc tools + -g,--grpc whether to generate GRPC code + -s,--server whether to generate grpc server code + -f,--file path of proto file + -o,--out path of code generation + -p,--prefix prefix(current project name) +Examples: + # Generate GRPC code from the Proto file + # -f: Proto file address -o: Code generation path -g: Whether to generate GRPC code + jupiter protoc -f ./pb/hello/hello.proto -o ./pb/hello -g + # According to the proto file, generate the server implementation + # -f: Proto file address -o: Code generation path -p:prefix(Current project name) -g: Whether to generate Server code + jupiter protoc -f ./pb/hello/hello.proto -o ./internal/app/grpc -p jupiter-demo -s +``` +## 开始实战 + 接下来我们会一步一步的带着大家从无到有开发jupiter应用!(gopher Let's go) +### 快速创建jupiter模板项目 +```shell script +cd ${GOPATH}/src +jupiter new jupiter-demo -d ./ +``` +命令解释: +* new :创建jupiter模板项目 +* jupiter-demo: 项目名称 +* -d: 生成项目所在路径 + +然后就会在${GOPATH}/src 下生成jupiter-demo 项目 +项目目录结构: +```go +build 编译目录 +cmd 应用启动目录 +config 应用配置目录 +internal +├─app 应用目录 +│ ├─engine +│ │ ├─engine.go 核心编排引擎(启动HTTP,GRPC,JOB等服务) +│ ├─grpc grpc服务实现目录 +│ ├─handler 控制器目录（接收用户请求） +│ │ ├─user.go 控制器文件 +│ ├─model model目录（定义持久层结构体） +│ │ ├─db +│ │ │ ├─user.go +│ │ ├─init.go 初始化全局数据库句柄 +│ ├─service service层 +│ │ ├─user 模块 +│ │ │ ├─impl +│ │ │ │ ├─mysqlImpl.go 实现 +│ │ │ ├─repository.go service 接口 +│ │ ├─init.go +pb proto文件 +sql sql脚本 +.gitignore +go.mod +Makefile +``` +### 运行jupiter应用 +* 数据库环境准备 +1. 请先在mysql中创建test库，然后将sql中的user.sql在该库中执行 +2. 修改mysql配置: 在config/config.toml中 找到 `[jupiter.mysql.test]` + 将`dsn` 改成你所在的环境 +* 下载go.mod依赖 +* 编译运行 + 在项目跟目录下执行以下命令 +```shell script +cd cmd +go run main.go --config=../config/config.toml +``` + 接下来你将会看到以下日志 +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>go run main.go --config=../config/config.toml + + (_)_ _ _ __ (_) |_ ___ _ __ + | | | | | '_ \| | __/ _ \ '__| + | | |_| | |_) | | || __/ | + _/ |\__,_| .__/|_|\__\___|_| + |__/ |_| + + Welcome to jupiter, starting application ... + +1592274902 INFO load local file {"mod": "config", "addr": "../config/config.toml"} +1592274902 INFO auto max procs {"mod": "proc", "procs": 4} +1592274902 INFO set global tracer {"mod": "trace"} +1592274902 INFO add job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:delete` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m +?[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:update` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m +?[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:create` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m +?[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:query` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m ? +[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:row_query` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31; +1m ?[0m +1592274902 INFO client mysql start {"mod": "gorm", "addr": "127.0.0.1:3306", "name": "test"} +1592274902 INFO run worker {"mod": "worker.cron", "number of scheduled jobs": 1} +1592274902 INFO cron start {"mod": "worker.cron"} +1592274902 INFO cron schedule {"mod": "worker.cron", "now": 1592274902, "entry": 1, "next": 1592274902} +1592274902 INFO cron wake {"mod": "worker.cron", "now": 1592274902} +1592274902 INFO cron run {"mod": "worker.cron", "now": 1592274902, "entry": 1, "next": 1592274912} +1592274902 INFO start servers {"mod": "app", "addr": "http://127.0.0.1:20105"} +1592274902 INFO start servers {"mod": "app", "addr": "grpc://127.0.0.1:20102"} +1592274902 INFO start governor {"mod": "app", "addr": "http://127.0.0.1:9990"} +1592274902 INFO exec job {"info": "print info"} +1592274902 WARN exec job {"warn": "print warning"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "GET", "path": "/jupiter"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "GET", "path": "/api/user/:id"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "GET", "path": "/grpc/get"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "POST", "path": "/grpc/post"} +1592274902 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +⇨ http server started on 127.0.0.1:20105 +1592274912 INFO cron wake {"mod": "worker.cron", "now": 1592274912} +1592274912 INFO cron run {"mod": "worker.cron", "now": 1592274912, "entry": 1, "next": 1592274922} +1592274912 INFO exec job {"info": "print info"} +1592274912 WARN exec job {"warn": "print warning"} +1592274912 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +1592274922 INFO cron wake {"mod": "worker.cron", "now": 1592274922} +1592274922 INFO cron run {"mod": "worker.cron", "now": 1592274922, "entry": 1, "next": 1592274932} +1592274922 INFO exec job {"info": "print info"} +1592274922 WARN exec job {"warn": "print warning"} +1592274922 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +1592274932 INFO cron wake {"mod": "worker.cron", "now": 1592274932} +1592274932 INFO cron run {"mod": "worker.cron", "now": 1592274932, "entry": 1, "next": 1592274942} +1592274932 INFO exec job {"info": "print info"} +1592274932 WARN exec job {"warn": "print warning"} +1592274932 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +``` +模板项目中 我们默认帮你开启了HTTP,GRPC,JOB等服务。当然你也可以根据自己的需求进行取舍 +接下来打开浏览器输入: +http://127.0.0.1:20105/jupiter +你将会看到 `welcome to jupiter` + +在开始运行项目前我们不是配置过数据库环境么? 接下来我们可以验证查询是否正常: +打开浏览器输入: http://127.0.0.1:20105/api/user/1 +你将会看到 `{"id":1,"username":"admin","password":"123456","nickname":"rose","address":"WUHAN"}` + +如果以上操作你都正常完成，恭喜你，你已经完成了基础操作，接下来我们将介绍脚手架工具的 protoc模块 + +### protoc 工具集介绍 + +jupiter protoc工具集目前可用于根据pb文件中的proto文件，生成pb.go，以及生成服务端实现， +你只需要去完善对应服务端实现的业务逻辑即可。 +* 环境准备 + 请确保当前系统安装protoc 编译工具 + go版本的 Protobuf 编译器插件,工具内部采用的是性能更好的 protoc-gen-gofast ,若检测到当前系统安装会自动进行安装 +* 实操演示 +脚手架中默认提供的 `pb/hello/hello.proto` 文件,以下操作都是基于该文件 +进入到项目根目录下，执行以下命令 +1. 生成pb.go 文件 +```shell script + jupiter protoc -f ./pb/hello/hello.proto -o ./pb/hello -g +``` +命令解释: +-f : proto文件路径 +-o :生成文件目录(建议生成目录与proto文件同级，方便管理) +-g :是否生成pb.go 文件 + +正常情况下会在 `./pb/hello/`中 生成 `hello.pb.go` 文件 + +2. 生成grpc服务端实现 +``` +jupiter protoc -f ./pb/hello/hello.proto -o ./internal/app/grpc -p jupiter-demo -s +``` +命令解释: +-f : proto文件路径 +-o : grpc服务端代码生成文件目录(建议生成至./internal/app/grpc 中) +-p : grpc服务端代码实现依赖pb.go的前缀(请使用当前项目名) +-s : 是否生成grpc服务端实现代码 + +正常情况下会在 `./internal/app/grpc/hello` 目录中 生成 `helloServiceServer.go` 文件 +请注意: 我们在grpc包中会默认以proto的package的包名来管理 grpc服务端实现代码,从而让代码结构更加清晰 +接下来 你只需要去该文件中完成对应的业务逻辑实现即可 + +快速测试grpc服务端代码: +框架内部我们支持 代理http到grpc控制器,并且在engine.go中我们也提供了相关的实例 +```go + //support proxy for http to grpc controller + g := greeter.Greeter{} + group2 := server.Group("/grpc") + group2.GET("/get", xecho.GRPCProxyWrapper(g.SayHello)) + group2.POST("/post", xecho.GRPCProxyWrapper(g.SayHello)) +``` +你可以参照该示例,将自己的grpc服务端实现注册在路由中，然后通过HTTP请求即可访问 -`go get -u github.com/douyu/jupiter/tools/jupiter` -## instruction for use -jupiter -h diff --git a/tools/jupiter/common/util.go b/tools/jupiter/common/util.go new file mode 100644 index 0000000000..d3c6df6f9e --- /dev/null +++ b/tools/jupiter/common/util.go @@ -0,0 +1,35 @@ +package common + +import ( + "fmt" + "github.com/douyu/jupiter/pkg/util/xregexp" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +// GetModPath ... +func GetModPath(projectPath string) (modPath string) { + dir := filepath.Dir(projectPath) + for { + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + content, _ := ioutil.ReadFile(filepath.Join(dir, "go.mod")) + mod := xregexp.RegexpReplace(`module\s+(?P[\S]+)`, string(content), "$name") + name := strings.TrimPrefix(filepath.Dir(projectPath), dir) + name = strings.TrimPrefix(name, string(os.PathSeparator)) + if name == "" { + return fmt.Sprintf("%s/", mod) + } + return fmt.Sprintf("%s/%s/", mod, name) + } + parent := filepath.Dir(dir) + if dir == parent { + return "" + } + dir = parent + } + } + +} diff --git a/tools/jupiter/common/util_test.go b/tools/jupiter/common/util_test.go new file mode 100644 index 0000000000..9294c2b4d8 --- /dev/null +++ b/tools/jupiter/common/util_test.go @@ -0,0 +1,35 @@ +package common + +import ( + "log" + "path/filepath" + "testing" +) + +func TestGetModPath(t *testing.T) { + type args struct { + projectPath string + } + tests := []struct { + name string + args args + wantModPath string + }{ + { + name: "getModPath", + args: args{projectPath: "./"}, + wantModPath: "asdasd/asdas", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + projectPath, err := filepath.Abs(tt.args.projectPath) + if err != nil { + log.Fatalf("get absolute path failed:%v\n", err) + } + if gotModPath := GetModPath(projectPath); gotModPath != tt.wantModPath { + t.Errorf("GetModPath() = %v, want %v", gotModPath, tt.wantModPath) + } + }) + } +} diff --git a/tools/jupiter/main.go b/tools/jupiter/main.go index 5c92166a84..e3710e6802 100644 --- a/tools/jupiter/main.go +++ b/tools/jupiter/main.go @@ -15,9 +15,10 @@ package main import ( + "github.com/douyu/jupiter/tools/jupiter/new" + "github.com/douyu/jupiter/tools/jupiter/protoc" "os" - new "github.com/douyu/jupiter/tools/jupiter/new" "github.com/urfave/cli" ) @@ -29,22 +30,8 @@ func main() { app.Usage = "jupiter tools" app.Version = Version app.Commands = []cli.Command{ - { - Name: "new", - Aliases: []string{"n"}, - Usage: "Create Jupiter template project", - Action: new.CreateProject, - SkipFlagParsing: false, - UsageText: new.NewProjectHelpTemplate, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "d", - Value: "", - Usage: "Specify the directory of the project", - Destination: &new.Project.Path, - }, - }, - }, + new.Cmd, + protoc.Cmd, } err := app.Run(os.Args) if err != nil { diff --git a/tools/jupiter/new/new_cmd.go b/tools/jupiter/new/new_cmd.go new file mode 100644 index 0000000000..82a0e325ac --- /dev/null +++ b/tools/jupiter/new/new_cmd.go @@ -0,0 +1,20 @@ +package new + +import "github.com/urfave/cli" + +var Cmd = cli.Command{ + Name: "new", + Aliases: []string{"n"}, + Usage: "Create Jupiter template project", + Action: CreateProject, + SkipFlagParsing: false, + UsageText: NewProjectHelpTemplate, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "d", + Value: "", + Usage: "Specify the directory of the project", + Destination: &project.Path, + }, + }, +} diff --git a/tools/jupiter/new/option.go b/tools/jupiter/new/option.go index 4a719a09df..74ec93b424 100644 --- a/tools/jupiter/new/option.go +++ b/tools/jupiter/new/option.go @@ -14,6 +14,7 @@ package new +//ProjectInfo ... type ProjectInfo struct { // project dir Path string @@ -23,6 +24,6 @@ type ProjectInfo struct { } var ( - Project ProjectInfo + project ProjectInfo DefaultProjectName = "jupiter-demo" ) diff --git a/tools/jupiter/new/templates/.gitignore b/tools/jupiter/new/templates/.gitignore new file mode 100644 index 0000000000..387b921169 --- /dev/null +++ b/tools/jupiter/new/templates/.gitignore @@ -0,0 +1,8 @@ +.idea/ +*.iml +*.exe +*.bak* +.history/ +.vscode/ +db/ +vendor/ diff --git a/tools/jupiter/new/templates/README.md b/tools/jupiter/new/templates/README.md index 2ecfc26fe3..13306e0c18 100644 --- a/tools/jupiter/new/templates/README.md +++ b/tools/jupiter/new/templates/README.md @@ -1,12 +1,272 @@ -# Directory structure description +# jupiter -- cmd: -program entry file -- config: -configuration file -- internal/demo/engine: core engine, profile loading, HTTP, grpc and other services are enabled -- internal/demo/greeter:GRPC service demo -- internal/pkg: +## 脚手架介绍 +### jupiter脚手架工具集 +1. 快速生成模板项目 +2. 基于proto文件生成pb.go +3. 基于proto文件生成服务端实现 # go version - GO > 1.14 \ No newline at end of file + GO >= 1.13 + +## 脚手架工具获取 +windows : +```shell script +set GO111MODULE=on +set GOPROXY=https://mirrors.aliyun.com/goproxy/,direct +``` +linux : +```shell script +export GO111MODULE=on +export GOPROXY=https://mirrors.aliyun.com/goproxy/,direct +``` +`go get -u -v github.com/douyu/jupiter/tools/jupiter` +* windows 用户: + 会在${GOPATH}/src/bin 目录下生成jupiter.exe 文件,若想方便的在任何地方使用jupiter命令,请将该 + 命令配置在系统的环境变量中 +* Linux 用户: +* Mac 用户 : + + +## 如何使用 +* jupiter -h +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>jupiter -h +NAME: + jupiter - jupiter tools + +USAGE: + jupiter [global options] command [command options] [arguments...] + +VERSION: + 0.1.0 + +COMMANDS: + new, n Create Jupiter template project + protoc, p jupiter protoc tools + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --help, -h show help + --version, -v print the version +``` +* jupiter new -h + +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>jupiter new -h +NAME: + jupiter new - Create Jupiter template project + +USAGE: + +jupiter [commands|flags] + +The commands & flags are: + new Create Jupiter template project + + -d Build the specified directory for the template project + +Examples: + # Build the specified directory for the template project + jupiter new (your project name) -d (project dir) +``` +* jupiter protoc -h +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>jupiter protoc -h +NAME: + jupiter protoc - jupiter protoc tools + +USAGE: + +jupiter [commands|flags] + +The commands & flags are: + protoc jupiter protoc tools + -g,--grpc whether to generate GRPC code + -s,--server whether to generate grpc server code + -f,--file path of proto file + -o,--out path of code generation + -p,--prefix prefix(current project name) +Examples: + # Generate GRPC code from the Proto file + # -f: Proto file address -o: Code generation path -g: Whether to generate GRPC code + jupiter protoc -f ./pb/hello/hello.proto -o ./pb/hello -g + # According to the proto file, generate the server implementation + # -f: Proto file address -o: Code generation path -p:prefix(Current project name) -g: Whether to generate Server code + jupiter protoc -f ./pb/hello/hello.proto -o ./internal/app/grpc -p jupiter-demo -s +``` +## 开始实战 + 接下来我们会一步一步的带着大家从无到有开发jupiter应用!(gopher Let's go) +### 快速创建jupiter模板项目 +```shell script +cd ${GOPATH}/src +jupiter new jupiter-demo -d ./ +``` +命令解释: +* new :创建jupiter模板项目 +* jupiter-demo: 项目名称 +* -d: 生成项目所在路径 + +然后就会在${GOPATH}/src 下生成jupiter-demo 项目 +项目目录结构: +```go +build 编译目录 +cmd 应用启动目录 +config 应用配置目录 +internal +├─app 应用目录 +│ ├─engine +│ │ ├─engine.go 核心编排引擎(启动HTTP,GRPC,JOB等服务) +│ ├─grpc grpc服务实现目录 +│ ├─handler 控制器目录（接收用户请求） +│ │ ├─user.go 控制器文件 +│ ├─model model目录（定义持久层结构体） +│ │ ├─db +│ │ │ ├─user.go +│ │ ├─init.go 初始化全局数据库句柄 +│ ├─service service层 +│ │ ├─user 模块 +│ │ │ ├─impl +│ │ │ │ ├─mysqlImpl.go 实现 +│ │ │ ├─repository.go service 接口 +│ │ ├─init.go +pb proto文件 +sql sql脚本 +.gitignore +go.mod +Makefile +``` +### 运行jupiter应用 +* 数据库环境准备 +1. 请先在mysql中创建test库，然后将sql中的user.sql在该库中执行 +2. 修改mysql配置: 在config/config.toml中 找到 `[jupiter.mysql.test]` + 将`dsn` 改成你所在的环境 +* 下载go.mod依赖 +* 编译运行 + 在项目跟目录下执行以下命令 +```shell script +cd cmd +go run main.go --config=../config/config.toml +``` + 接下来你将会看到以下日志 +```shell script +E:\go\goworkspace\src\jupiter-demo\cmd>go run main.go --config=../config/config.toml + + (_)_ _ _ __ (_) |_ ___ _ __ + | | | | | '_ \| | __/ _ \ '__| + | | |_| | |_) | | || __/ | + _/ |\__,_| .__/|_|\__\___|_| + |__/ |_| + + Welcome to jupiter, starting application ... + +1592274902 INFO load local file {"mod": "config", "addr": "../config/config.toml"} +1592274902 INFO auto max procs {"mod": "proc", "procs": 4} +1592274902 INFO set global tracer {"mod": "trace"} +1592274902 INFO add job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:delete` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m +?[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:update` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m +?[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:create` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m +?[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:query` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31;1m ? +[0m + +?[33m[2020-06-16 10:35:02]?[0m ?[35m[info] replacing callback `gorm:row_query` from E:/go/goworkspace/src/jupiter-demo/vendor/github.com/douyu/jupiter/pkg/store/gorm/orm.go:118?[0m ?[31; +1m ?[0m +1592274902 INFO client mysql start {"mod": "gorm", "addr": "127.0.0.1:3306", "name": "test"} +1592274902 INFO run worker {"mod": "worker.cron", "number of scheduled jobs": 1} +1592274902 INFO cron start {"mod": "worker.cron"} +1592274902 INFO cron schedule {"mod": "worker.cron", "now": 1592274902, "entry": 1, "next": 1592274902} +1592274902 INFO cron wake {"mod": "worker.cron", "now": 1592274902} +1592274902 INFO cron run {"mod": "worker.cron", "now": 1592274902, "entry": 1, "next": 1592274912} +1592274902 INFO start servers {"mod": "app", "addr": "http://127.0.0.1:20105"} +1592274902 INFO start servers {"mod": "app", "addr": "grpc://127.0.0.1:20102"} +1592274902 INFO start governor {"mod": "app", "addr": "http://127.0.0.1:9990"} +1592274902 INFO exec job {"info": "print info"} +1592274902 WARN exec job {"warn": "print warning"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "GET", "path": "/jupiter"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "GET", "path": "/api/user/:id"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "GET", "path": "/grpc/get"} +1592274902 INFO echo add route {"mod": "server.echo", "method": "POST", "path": "/grpc/post"} +1592274902 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +⇨ http server started on 127.0.0.1:20105 +1592274912 INFO cron wake {"mod": "worker.cron", "now": 1592274912} +1592274912 INFO cron run {"mod": "worker.cron", "now": 1592274912, "entry": 1, "next": 1592274922} +1592274912 INFO exec job {"info": "print info"} +1592274912 WARN exec job {"warn": "print warning"} +1592274912 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +1592274922 INFO cron wake {"mod": "worker.cron", "now": 1592274922} +1592274922 INFO cron run {"mod": "worker.cron", "now": 1592274922, "entry": 1, "next": 1592274932} +1592274922 INFO exec job {"info": "print info"} +1592274922 WARN exec job {"warn": "print warning"} +1592274922 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +1592274932 INFO cron wake {"mod": "worker.cron", "now": 1592274932} +1592274932 INFO cron run {"mod": "worker.cron", "now": 1592274932, "entry": 1, "next": 1592274942} +1592274932 INFO exec job {"info": "print info"} +1592274932 WARN exec job {"warn": "print warning"} +1592274932 INFO run job {"mod": "worker.cron", "name": "jupiter-demo/internal/app/engine.(*Engine).execJob-fm"} +``` +模板项目中 我们默认帮你开启了HTTP,GRPC,JOB等服务。当然你也可以根据自己的需求进行取舍 +接下来打开浏览器输入: +http://127.0.0.1:20105/jupiter +你将会看到 `welcome to jupiter` + +在开始运行项目前我们不是配置过数据库环境么? 接下来我们可以验证查询是否正常: +打开浏览器输入: http://127.0.0.1:20105/api/user/1 +你将会看到 `{"id":1,"username":"admin","password":"123456","nickname":"rose","address":"WUHAN"}` + +如果以上操作你都正常完成，恭喜你，你已经完成了基础操作，接下来我们将介绍脚手架工具的 protoc模块 + +### protoc 工具集介绍 + +jupiter protoc工具集目前可用于根据pb文件中的proto文件，生成pb.go，以及生成服务端实现， +你只需要去完善对应服务端实现的业务逻辑即可。 +* 环境准备 + 请确保当前系统安装protoc 编译工具 + go版本的 Protobuf 编译器插件,工具内部采用的是性能更好的 protoc-gen-gofast ,若检测到当前系统安装会自动进行安装 +* 实操演示 +脚手架中默认提供的 `pb/hello/hello.proto` 文件,以下操作都是基于该文件 +进入到项目根目录下，执行以下命令 +1. 生成pb.go 文件 +```shell script + jupiter protoc -f ./pb/hello/hello.proto -o ./pb/hello -g +``` +命令解释: +-f : proto文件路径 +-o :生成文件目录(建议生成目录与proto文件同级，方便管理) +-g :是否生成pb.go 文件 + +正常情况下会在 `./pb/hello/`中 生成 `hello.pb.go` 文件 + +2. 生成grpc服务端实现 +``` +jupiter protoc -f ./pb/hello/hello.proto -o ./internal/app/grpc -p jupiter-demo -s +``` +命令解释: +-f : proto文件路径 +-o : grpc服务端代码生成文件目录(建议生成至./internal/app/grpc 中) +-p : grpc服务端代码实现依赖pb.go的前缀(请使用当前项目名) +-s : 是否生成grpc服务端实现代码 + +正常情况下会在 `./internal/app/grpc/hello` 目录中 生成 `helloServiceServer.go` 文件 +请注意: 我们在grpc包中会默认以proto的package的包名来管理 grpc服务端实现代码,从而让代码结构更加清晰 +接下来 你只需要去该文件中完成对应的业务逻辑实现即可 + +快速测试grpc服务端代码: +框架内部我们支持 代理http到grpc控制器,并且在engine.go中我们也提供了相关的实例 +```go + //support proxy for http to grpc controller + g := greeter.Greeter{} + group2 := server.Group("/grpc") + group2.GET("/get", xecho.GRPCProxyWrapper(g.SayHello)) + group2.POST("/post", xecho.GRPCProxyWrapper(g.SayHello)) +``` +你可以参照该示例,将自己的grpc服务端实现注册在路由中，然后通过HTTP请求即可访问 + + diff --git a/tools/jupiter/new/templates/cmd/main.go.tmpl b/tools/jupiter/new/templates/cmd/main.go.tmpl index 4be3f6944b..90bf7a0209 100644 --- a/tools/jupiter/new/templates/cmd/main.go.tmpl +++ b/tools/jupiter/new/templates/cmd/main.go.tmpl @@ -1,14 +1,23 @@ package main import ( + "fmt" "log" "{{.ModPrefix}}{{.Name}}/internal/app/engine" + "{{.ModPrefix}}{{.Name}}/internal/app/model" + "{{.ModPrefix}}{{.Name}}/internal/app/service" ) func main() { eng := engine.NewEngine() - if err := eng.Run(); err != nil { - log.Fatal(err) - } + eng.Defer(func() error { + fmt.Println("exit jupiter app ...") + return nil + }) + model.Init() + service.Init() + if err := eng.Run(); err != nil { + log.Fatal(err) + } } diff --git a/tools/jupiter/new/templates/config/config.toml b/tools/jupiter/new/templates/config/config.toml index 0d34b1f8ad..e632f1a8ca 100644 --- a/tools/jupiter/new/templates/config/config.toml +++ b/tools/jupiter/new/templates/config/config.toml @@ -15,4 +15,18 @@ timeout = "1s" [jupiter.cron.demo] immediatelyRun = true - concurrentDelay = -1 + concurrentDelay = 5 +[jupiter.mysql.test] + connMaxLifetime = "300s" + debug = true + dsn = "user:pwd@tcp(127.0.0.1:3306)/test?charset=utf8&parseTime=True&loc=Local" + level = "panic" + maxIdleConns = 50 + maxOpenConns = 100 + disableTrace = true + disableMetric = true +[jupiter.trace.jaeger] + enableRPCMetrics = false + [jupiter.trace.jaeger.sampler] + type = "const" + param = 0.001 \ No newline at end of file diff --git a/tools/jupiter/new/templates/internal/app/engine/engine.go.tmpl b/tools/jupiter/new/templates/internal/app/engine/engine.go.tmpl index 113ddd9ee6..f9a6d44bb6 100644 --- a/tools/jupiter/new/templates/internal/app/engine/engine.go.tmpl +++ b/tools/jupiter/new/templates/internal/app/engine/engine.go.tmpl @@ -2,13 +2,16 @@ package engine import ( "github.com/douyu/jupiter" - "{{.ModPrefix}}{{.Name}}/internal/demo/greeter" + "{{.ModPrefix}}{{.Name}}/internal/app/grpc/greeter" + "{{.ModPrefix}}{{.Name}}/internal/app/handler" "github.com/douyu/jupiter/pkg/server/xecho" + "github.com/douyu/jupiter/pkg/worker/xcron" "github.com/douyu/jupiter/pkg/server/xgrpc" "github.com/douyu/jupiter/pkg/util/xgo" "github.com/douyu/jupiter/pkg/xlog" "github.com/labstack/echo/v4" "google.golang.org/grpc/examples/helloworld/helloworld" + "time" ) type Engine struct { @@ -19,26 +22,47 @@ func NewEngine() *Engine { eng := &Engine{} if err := eng.Startup( xgo.ParallelWithError( - eng.serveHTTP, eng.serveGRPC, + eng.serveHTTP, + eng.startJobs, ), ); err != nil { xlog.Panic("startup engine", xlog.Any("err", err)) } return eng } + func (eng *Engine) serveHTTP() error { server := xecho.StdConfig("http").Build() - server.GET("/ping", func(ctx echo.Context) error { - return ctx.JSON(200, "pong") - }) - eng.Serve(server) - return nil + server.GET("/jupiter", func(ctx echo.Context) error { + return ctx.JSON(200, "welcome to jupiter") + }) + // Specify routing group + group := server.Group("/api") + group.GET("/user/:id",handler.GetUser) + + //support proxy for http to grpc controller + g := greeter.Greeter{} + group2 := server.Group("/grpc") + group2.GET("/get", xecho.GRPCProxyWrapper(g.SayHello)) + group2.POST("/post", xecho.GRPCProxyWrapper(g.SayHello)) + return eng.Serve(server) } func (eng *Engine) serveGRPC() error { server := xgrpc.StdConfig("grpc").Build() - helloworld.RegisterGreeterServer(server.Server, new(greeter.Greeter)) - eng.Serve(server) - return nil + helloworld.RegisterGreeterServer(server.Server, new(greeter.Greeter)) + return eng.Serve(server) +} + +func (eng *Engine) startJobs() error { + cron := xcron.StdConfig("demo").Build() + cron.Schedule(xcron.Every(time.Second*10), xcron.FuncJob(eng.execJob)) + return eng.Schedule(cron) } + +func (eng *Engine) execJob() error { + xlog.Info("exec job", xlog.String("info", "print info")) + xlog.Warn("exec job", xlog.String("warn", "print warning")) + return nil +} \ No newline at end of file diff --git a/tools/jupiter/new/templates/internal/demo/greeter/greeter.go.tmpl b/tools/jupiter/new/templates/internal/app/grpc/greeter/greeter.go.tmpl similarity index 100% rename from tools/jupiter/new/templates/internal/demo/greeter/greeter.go.tmpl rename to tools/jupiter/new/templates/internal/app/grpc/greeter/greeter.go.tmpl diff --git a/tools/jupiter/new/templates/internal/app/handler/user.go.tmpl b/tools/jupiter/new/templates/internal/app/handler/user.go.tmpl new file mode 100644 index 0000000000..9f860bb0e6 --- /dev/null +++ b/tools/jupiter/new/templates/internal/app/handler/user.go.tmpl @@ -0,0 +1,20 @@ +package handler + +import ( + "github.com/labstack/echo/v4" + "{{.ModPrefix}}{{.Name}}/internal/app/service" + "net/http" + "strconv" +) + +func GetUser(c echo.Context) (err error) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + return c.JSON(http.StatusInternalServerError,"parameter error") + } + user,err := service.UserRepository.Get(id) + if err != nil { + return c.JSON(http.StatusInternalServerError,err.Error()) + } + return c.JSON(http.StatusOK, user) +} diff --git a/tools/jupiter/new/templates/internal/app/model/init.go.tmpl b/tools/jupiter/new/templates/internal/app/model/init.go.tmpl new file mode 100644 index 0000000000..7fe24bcbd5 --- /dev/null +++ b/tools/jupiter/new/templates/internal/app/model/init.go.tmpl @@ -0,0 +1,11 @@ +package model + +import "github.com/douyu/jupiter/pkg/store/gorm" + +var ( + MysqlHandler *gorm.DB +) +//Init ... +func Init() { + MysqlHandler = gorm.StdConfig("test").Build() +} \ No newline at end of file diff --git a/tools/jupiter/new/templates/internal/app/service/init.go.tmpl b/tools/jupiter/new/templates/internal/app/service/init.go.tmpl new file mode 100644 index 0000000000..182ad68691 --- /dev/null +++ b/tools/jupiter/new/templates/internal/app/service/init.go.tmpl @@ -0,0 +1,15 @@ +package service + +import ( + "{{.ModPrefix}}{{.Name}}/internal/app/model" + "{{.ModPrefix}}{{.Name}}/internal/app/service/user" + "{{.ModPrefix}}{{.Name}}/internal/app/service/user/impl" +) + +var ( + UserRepository user.Repository +) +//Init instantiate the service +func Init() { + UserRepository = impl.NewMysqlImpl(model.MysqlHandler) +} \ No newline at end of file diff --git a/tools/jupiter/new/templates/internal/app/service/user/impl/mysqlimpl.go.tmpl b/tools/jupiter/new/templates/internal/app/service/user/impl/mysqlimpl.go.tmpl new file mode 100644 index 0000000000..2abcd25884 --- /dev/null +++ b/tools/jupiter/new/templates/internal/app/service/user/impl/mysqlimpl.go.tmpl @@ -0,0 +1,21 @@ +package impl + +import ( + "github.com/douyu/jupiter/pkg/store/gorm" + "{{.ModPrefix}}{{.Name}}/internal/app/model/db" + "{{.ModPrefix}}{{.Name}}/internal/app/service/user" +) +type mysqlImpl struct { + gh *gorm.DB +} +// NewMysqlImpl construct an instance of mysqlImpl +func NewMysqlImpl(gh *gorm.DB) user.Repository { + return &mysqlImpl{ + gh: gh, + } +} +func (m *mysqlImpl)Get(id int) (user db.User,err error){ + user = db.User{} + err = m.gh.Where("id = ?", id).Find(&user).Error + return +} diff --git a/tools/jupiter/new/templates/internal/app/service/user/repository.go.tmpl b/tools/jupiter/new/templates/internal/app/service/user/repository.go.tmpl new file mode 100644 index 0000000000..12a5040d1a --- /dev/null +++ b/tools/jupiter/new/templates/internal/app/service/user/repository.go.tmpl @@ -0,0 +1,9 @@ +package user + +import ( + "{{.ModPrefix}}{{.Name}}/internal/app/model/db" +) + +type Repository interface { + Get(id int) (user db.User,err error) +} diff --git a/tools/jupiter/new/templates/pb/hello/hello.proto b/tools/jupiter/new/templates/pb/hello/hello.proto new file mode 100644 index 0000000000..1dc3ab4d1c --- /dev/null +++ b/tools/jupiter/new/templates/pb/hello/hello.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package hello; +option go_package = ".;hello"; +message HelloRequest { + string name = 1; +} + +message HelloResponse { + string reply = 1; +} + +service HelloService { + rpc SayHello(HelloRequest) returns (HelloResponse){} +} \ No newline at end of file diff --git a/tools/jupiter/new/templates/sql/user.sql b/tools/jupiter/new/templates/sql/user.sql new file mode 100644 index 0000000000..c6d12a26ee --- /dev/null +++ b/tools/jupiter/new/templates/sql/user.sql @@ -0,0 +1,9 @@ +CREATE TABLE `user` ( +`id` int NOT NULL AUTO_INCREMENT , +`username` varchar(255) NOT NULL DEFAULT '' , +`password` varchar(255) NOT NULL DEFAULT '' , +`nickname` varchar(255) NOT NULL DEFAULT '' , +`address` varchar(255) NOT NULL DEFAULT '' , +PRIMARY KEY (`id`) +)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +insert into user (id,username,password,nickname,address)VALUES(null,"admin","123456","rose","WUHAN"); \ No newline at end of file diff --git a/tools/jupiter/new/tools.go b/tools/jupiter/new/tools.go index f190c8a91c..742d6dce4c 100644 --- a/tools/jupiter/new/tools.go +++ b/tools/jupiter/new/tools.go @@ -17,6 +17,7 @@ package new import ( "bytes" "fmt" + "github.com/douyu/jupiter/tools/jupiter/common" "io/ioutil" "os" "path/filepath" @@ -24,7 +25,6 @@ import ( "text/template" "github.com/douyu/jupiter/pkg/util/xcolor" - "github.com/douyu/jupiter/pkg/util/xregexp" "github.com/gobuffalo/packr/v2" "github.com/urfave/cli" ) @@ -38,73 +38,51 @@ func CreateProject(cli *cli.Context) (err error) { } name := newArgs[0] if name == "" { - Project.Name = DefaultProjectName + project.Name = DefaultProjectName } else { - Project.Name = name + project.Name = name } - if Project.Path != "" { - if Project.Path, err = filepath.Abs(Project.Path); err != nil { + if project.Path != "" { + if project.Path, err = filepath.Abs(project.Path); err != nil { return } - Project.Path = filepath.Join(Project.Path, Project.Name) + project.Path = filepath.Join(project.Path, project.Name) } else { pwd, _ := os.Getwd() - Project.Path = filepath.Join(pwd, Project.Name) + project.Path = filepath.Join(pwd, project.Name) } - modPath := getModPath(Project.Path) - Project.ModPrefix = modPath + //GetModPath + modPath := common.GetModPath(project.Path) + fmt.Println("new project modPrefix:", modPath) + project.ModPrefix = modPath if err = doCreateProject(); err != nil { return } - fmt.Println(xcolor.Greenf("Project dir:", Project.Path)) + fmt.Println(xcolor.Greenf("Project dir:", project.Path)) fmt.Println(xcolor.Green("Project created successfully")) return } - -func getModPath(projectPath string) (modPath string) { - dir := filepath.Dir(projectPath) - for { - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - content, _ := ioutil.ReadFile(filepath.Join(dir, "go.mod")) - mod := xregexp.RegexpReplace(`module\s+(?P[\S]+)`, string(content), "$name") - name := strings.TrimPrefix(filepath.Dir(projectPath), dir) - name = strings.TrimPrefix(name, string(os.PathSeparator)) - if name == "" { - return fmt.Sprintf("%s/", mod) - } - return fmt.Sprintf("%s/%s/", mod, name) - } - parent := filepath.Dir(dir) - if dir == parent { - return "" - } - dir = parent - } - } - -} func doCreateProject() (err error) { box := packr.New("all", "./templates") - if err = os.MkdirAll(Project.Path, 0755); err != nil { + if err = os.MkdirAll(project.Path, 0755); err != nil { return } for _, name := range box.List() { - if Project.ModPrefix != "" && name == "go.mod.tmpl" { + if project.ModPrefix != "" && name == "go.mod.tmpl" { continue } tmpl, _ := box.FindString(name) i := strings.LastIndex(name, string(os.PathSeparator)) if i > 0 { dir := name[:i] - if err = os.MkdirAll(filepath.Join(Project.Path, dir), 0755); err != nil { + if err = os.MkdirAll(filepath.Join(project.Path, dir), 0755); err != nil { return } } if strings.HasSuffix(name, ".tmpl") { name = strings.TrimSuffix(name, ".tmpl") } - if err = doWriteFile(filepath.Join(Project.Path, name), tmpl); err != nil { + if err = doWriteFile(filepath.Join(project.Path, name), tmpl); err != nil { return } } @@ -118,7 +96,7 @@ func doWriteFile(path, tmpl string) (err error) { return } fmt.Println(xcolor.Greenf("File generated----------------------->", path)) - return ioutil.WriteFile(path, data, 0644) + return ioutil.WriteFile(path, data, 0755) } func parseTmpl(tmpl string) ([]byte, error) { @@ -127,7 +105,7 @@ func parseTmpl(tmpl string) ([]byte, error) { return nil, err } var buf bytes.Buffer - if err = tmp.Execute(&buf, Project); err != nil { + if err = tmp.Execute(&buf, project); err != nil { return nil, err } return buf.Bytes(), nil diff --git a/tools/jupiter/protoc/grpc.go b/tools/jupiter/protoc/grpc.go new file mode 100644 index 0000000000..e68eb4cd2a --- /dev/null +++ b/tools/jupiter/protoc/grpc.go @@ -0,0 +1,74 @@ +package protoc + +import ( + "errors" + "fmt" + "github.com/douyu/jupiter/pkg/util/xcolor" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + _genGoFastAddress = "go get -u -v github.com/gogo/protobuf/protoc-gen-gofast" + _grpcProtocCmd = "protoc --gofast_out=plugins=grpc:%s %s" +) + +func generateGRPC() (err error) { + if err = installGRPCGen(); err != nil { + return + } + if err = doGenerate(); err != nil { + return + } + return +} +func installGRPCGen() (err error) { + gofastPath := "" + if gofastPath, err = exec.LookPath("protoc-gen-gofast"); err != nil { + fmt.Println(xcolor.Green("start installing protoc-gen-gofast")) + if err = executeGoGet(_genGoFastAddress); err != nil { + return + } + } + fmt.Println(xcolor.Greenf("protoc-gen-gofast installation was successful, the installation path is:", gofastPath)) + return +} +func executeGoGet(address string) error { + args := strings.Split(address, " ") + cmd := exec.Command(args[0], args[1:]...) + cmd.Env = os.Environ() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} +func protocEnvCheck() (err error) { + protocPath := "" + if protocPath, err = exec.LookPath("protoc"); err != nil { + err = errors.New("You haven't installed Protobuf yet，Please visit this page to install with your own system：https://github.com/protocolbuffers/protobuf/releases") + return + } + fmt.Println(xcolor.Greenf("Protoc environment monitoring is successful , the installation path is:", protocPath)) + return +} +func doGenerate() (err error) { + if err = os.MkdirAll(option.outputDir, 0755); err != nil { + return + } + cmdLine := fmt.Sprintf(_grpcProtocCmd, option.outputDir, option.protoFilePath) + fmt.Println("protocCmdLine:", cmdLine) + args := strings.Split(cmdLine, " ") + cmd := exec.Command(args[0], args[1:]...) + cmd.Env = os.Environ() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + return + } + if genAbsPath, err := filepath.Abs(option.outputDir); err == nil { + fmt.Println(xcolor.Greenf("pb.go file generated successfully. The path is as follows:", genAbsPath)) + } + return +} diff --git a/tools/jupiter/protoc/option.go b/tools/jupiter/protoc/option.go new file mode 100644 index 0000000000..b419ed2b16 --- /dev/null +++ b/tools/jupiter/protoc/option.go @@ -0,0 +1,14 @@ +package protoc + +//Option ... +type Option struct { + withGRPC bool + withServer bool + protoFilePath string + outputDir string + prefix string +} + +var ( + option Option +) diff --git a/tools/jupiter/protoc/protoc_cmd.go b/tools/jupiter/protoc/protoc_cmd.go new file mode 100644 index 0000000000..1e2411aa13 --- /dev/null +++ b/tools/jupiter/protoc/protoc_cmd.go @@ -0,0 +1,42 @@ +package protoc + +import "github.com/urfave/cli" + +var Cmd = cli.Command{ + Name: "protoc", + Aliases: []string{"p"}, + Usage: "jupiter protoc tools", + Action: Run, + SkipFlagParsing: false, + UsageText: ProtocHelpTemplate, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "grpc,g", + Usage: "whether to generate GRPC code", + Destination: &option.withGRPC, + }, + &cli.BoolFlag{ + Name: "server,s", + Usage: "whether to generate grpc server code", + Destination: &option.withServer, + }, + &cli.StringFlag{ + Name: "file,f", + Usage: "Path of proto file", + Required: true, + Destination: &option.protoFilePath, + }, + &cli.StringFlag{ + Name: "out,o", + Usage: "Path of code generation", + Required: true, + Destination: &option.outputDir, + }, + &cli.StringFlag{ + Name: "prefix,p", + Usage: "prefix(current project name)", + Required: false, + Destination: &option.prefix, + }, + }, +} diff --git a/tools/jupiter/protoc/server.go b/tools/jupiter/protoc/server.go new file mode 100644 index 0000000000..6e501ccd6a --- /dev/null +++ b/tools/jupiter/protoc/server.go @@ -0,0 +1,133 @@ +package protoc + +import ( + "fmt" + "github.com/douyu/jupiter/pkg/util/xcolor" + template2 "github.com/douyu/jupiter/tools/jupiter/protoc/template" + "github.com/emicklei/proto" + "html/template" + "os" + "path" + "path/filepath" + "runtime" + "strings" +) + +// GRPCServerGen ... +type GRPCServerGen struct { + rpcMeta *RPCMeta +} + +//RPCMeta ... +type RPCMeta struct { + Service *proto.Service + Message []*proto.Message + RPC []*proto.RPC + Package *proto.Package + Prefix string //生成的GRPC server代码 import pb.go文件时的前缀 +} + +//NewGRPCServerGen construct a GRPCServerGen instance +func NewGRPCServerGen() *GRPCServerGen { + return &GRPCServerGen{rpcMeta: &RPCMeta{}} +} + +// Parse IDL files +func (server *GRPCServerGen) parseProtoFile(protoFilePath string) (err error) { + reader, err := os.Open(protoFilePath) + defer reader.Close() + if err != nil { + return + } + parser := proto.NewParser(reader) + definition, err := parser.Parse() + if err != nil { + return + } + proto.Walk(definition, + proto.WithService(server.handleService), + proto.WithMessage(server.handleMessage), + proto.WithRPC(server.handleRPC), + proto.WithPackage(server.handlePackage)) + return +} +func (server *GRPCServerGen) handlePackage(pkg *proto.Package) { + server.rpcMeta.Package = pkg +} +func (server *GRPCServerGen) handleRPC(rpc *proto.RPC) { + server.rpcMeta.RPC = append(server.rpcMeta.RPC, rpc) +} +func (server *GRPCServerGen) handleService(s *proto.Service) { + server.rpcMeta.Service = s +} +func (server *GRPCServerGen) handleMessage(m *proto.Message) { + server.rpcMeta.Message = append(server.rpcMeta.Message, m) +} +func (server *GRPCServerGen) generateServer() (err error) { + if server.rpcMeta == nil { + return + } + if err = server.initPrefix(); err != nil { + return + } + if err = server.parseProtoFile(option.protoFilePath); err != nil { + return + } + outPut := filepath.Join(option.outputDir, server.rpcMeta.Package.Name) + if err = os.MkdirAll(outPut, 0755); err != nil { + return + } + fileName := fmt.Sprintf("%sServer.go", server.rpcMeta.Service.Name) + fileName = Lcfirst(fileName) + filePath := path.Join(outPut, fileName) + file, err1 := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err1 != nil { + err = err1 + return + } + defer file.Close() + filePath, _ = filepath.Abs(filePath) + err1 = server.render(file, template2.GRPCServerTemplate, server.rpcMeta) + if err != nil { + err = err1 + return + } + fmt.Println(xcolor.Greenf("GRCP server file generate success ,the path :", filePath)) + return +} +func (server *GRPCServerGen) initPrefix() (err error) { + if option.prefix != "" { + server.rpcMeta.Prefix = option.prefix + return + } + goPath := os.Getenv("GOPATH") + execFilePath, err := filepath.Abs(os.Args[0]) + if err != nil { + return + } + if runtime.GOOS == "windows" { + execFilePath = strings.Replace(execFilePath, "\\", "/", -1) + goPath = strings.Replace(goPath, "\\", "/", -1) + } + lastIdx := strings.LastIndex(execFilePath, "/") + if lastIdx < 0 { + return + } + output := strings.ToLower(execFilePath[0:lastIdx]) + srcPath := path.Join(goPath, "src/") + if srcPath[len(srcPath)-1] != '/' { + srcPath = fmt.Sprintf("%s/", srcPath) + } + server.rpcMeta.Prefix = strings.Replace(output, strings.ToLower(srcPath), "", -1) + fmt.Printf("rpcMeta.Prefix:%s\n", server.rpcMeta.Prefix) + return +} +func (server *GRPCServerGen) render(file *os.File, data string, rpcMeta *RPCMeta) (err error) { + t := template.New("main") + t, err = t.Parse(data) + err = t.Execute(file, rpcMeta) + if err != nil { + return + } + return +} diff --git a/tools/jupiter/protoc/template/server_tmpl.go b/tools/jupiter/protoc/template/server_tmpl.go new file mode 100644 index 0000000000..5dfc6c3b7a --- /dev/null +++ b/tools/jupiter/protoc/template/server_tmpl.go @@ -0,0 +1,22 @@ +package template + +var GRPCServerTemplate = ` +package {{.Package.Name}} + +import ( + "context" + {{if not .Prefix}} + "pb/{{.Package.Name}}" + {{else}} + "{{.Prefix}}/pb/{{.Package.Name}}" + {{end}} +) + +type {{.Service.Name}}Server struct{} +{{range .RPC}} +func (server *{{$.Service.Name}}Server) {{.Name}}(context context.Context, request *{{$.Package.Name}}.{{.RequestType}}) (response *{{$.Package.Name}}.{{.ReturnsType}}, err error) { + panic("implement me") + return +} +{{end}} +` diff --git a/tools/jupiter/protoc/tools.go b/tools/jupiter/protoc/tools.go new file mode 100644 index 0000000000..01484d7203 --- /dev/null +++ b/tools/jupiter/protoc/tools.go @@ -0,0 +1,41 @@ +package protoc + +import ( + "errors" + "github.com/urfave/cli" +) + +//Run ... +func Run(cli *cli.Context) (err error) { + // 校验OS是否安装 protoc 工具 + if err = protocEnvCheck(); err != nil { + return + } + if option.protoFilePath == "" { + err = errors.New("Please specify the proto file path and use jupiter protoc - h to view the detailed prompt") + return + } + if option.outputDir == "" { + err = errors.New("Please specify the code generation path and use jupiter protoc - h to view the detailed prompt") + return + } + // 默认生成全部 + if !option.withGRPC && !option.withServer { + option.withGRPC = true + option.withServer = true + } + // 根据指定目录下的proto 文件 生成pb.go 文件 + if option.withGRPC { + if err = generateGRPC(); err != nil { + return + } + } + // 生成 grpc server 服务端代码 + if option.withServer { + serverGen := NewGRPCServerGen() + if err = serverGen.generateServer(); err != nil { + return + } + } + return +} diff --git a/tools/jupiter/protoc/usage.go b/tools/jupiter/protoc/usage.go new file mode 100644 index 0000000000..fc152b00e7 --- /dev/null +++ b/tools/jupiter/protoc/usage.go @@ -0,0 +1,37 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoc + +//ProtocHelpTemplate ... +const ProtocHelpTemplate = ` +jupiter [commands|flags] + +The commands & flags are: + protoc jupiter protoc tools + -g,--grpc whether to generate GRPC code + -s,--server whether to generate grpc server code + -c,--client generate grpc server code + -f,--file path of proto file + -o,--out path of code generation + -p,--prefix prefix(current project name) +Examples: + # Generate GRPC code from the Proto file + # -f: Proto file address -o: Code generation path -g: Whether to generate GRPC code + jupiter protoc -f ./pb/hello/hello.proto -o ./pb/hello -g + # According to the proto file, generate the server implementation + # -f: Proto file address -o: Code generation path -p:prefix(Current project name) -g: Whether to generate Server code + jupiter protoc -f ./pb/hello/hello.proto -o ./internal/app/grpc -p jupiter-demo -s + +` diff --git a/tools/jupiter/protoc/util.go b/tools/jupiter/protoc/util.go new file mode 100644 index 0000000000..f2a15384cf --- /dev/null +++ b/tools/jupiter/protoc/util.go @@ -0,0 +1,19 @@ +package protoc + +import "unicode" + +// Ucfirst capitalize the first letter of the string +func Ucfirst(str string) string { + for i, v := range str { + return string(unicode.ToUpper(v)) + str[i+1:] + } + return "" +} + +// Lcfirst the string begins with a lowercase letter +func Lcfirst(str string) string { + for i, v := range str { + return string(unicode.ToLower(v)) + str[i+1:] + } + return "" +} diff --git a/tools/jupiter/protoc/util_test.go b/tools/jupiter/protoc/util_test.go new file mode 100644 index 0000000000..d89f75cc50 --- /dev/null +++ b/tools/jupiter/protoc/util_test.go @@ -0,0 +1,51 @@ +package protoc + +import "testing" + +func TestUcfirst(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "TestUcfirst", + args: args{"helloService"}, + want: "HelloService", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Ucfirst(tt.args.str); got != tt.want { + t.Errorf("Ucfirst() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLcfirst(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "TestLcfirst", + args: args{"HelloService"}, + want: "helloService", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Lcfirst(tt.args.str); got != tt.want { + t.Errorf("Lcfirst() = %v, want %v", got, tt.want) + } + }) + } +}