update: search & load commands

README.md

@@ -56,7 +56,7 @@ EXPIRED_TIME: 432000
 ./ZoomEye-go init -apikey "XXXXXXXX-XXXX-XXXXX-XXXX-XXXXXXXXXXX"
 succeed to initialize
 
-[Resources Info]
+[ZoomEye Resources Info]
 
   Role:  developer
   Quota: 10000
@@ -75,7 +75,7 @@ succeed to initialize
 ./ZoomEye-go info
 succeed to query
 
-[Resources Info]
+[ZoomEye Resources Info]
 
   Role:  developer
   Quota: 10000
@@ -87,7 +87,6 @@ succeed to query
 搜索是 `ZoomEye-go` 最核心的功能，通过 `search` 命令指定搜索关键词进行使用，支持的参数说明如下：
 
 ```text
--dork [KEYWORDS]     设置搜索关键字，必填（如：-dork "port:80 nginx"）
 -num [NUM]           设置显示/搜索的数据条数，默认为 20（建议设置20的倍数，因为ZoomEye一次接口查询为20条）
 -type [host/web]     设置搜索资源类型，默认为 host（如：-type "web"）
 -force               强制调用 ZoomEye API 查询，忽略本地数据和缓存
@@ -111,15 +110,15 @@ succeed to query
 使用示例：
 
 ```bash
-./ZoomEye-go search -dork "telnet" -count
+./ZoomEye-go search "telnet" -count
 succeed to search (in 272.080753ms)
 
-[Total Count]
+[ZoomEye Total]
 
   Count: 57003299
 
 
-./ZoomEye-go search -dork "telnet" -num 1
+./ZoomEye-go search "telnet" -num 1
 succeed to search (in 370.930383ms)
 
 [Host Search Result]
@@ -133,12 +132,12 @@ succeed to search (in 370.930383ms)
   +-----------------------+----------------------+----------------------+------------------------------------------+----------------------+
 
 
-./ZoomEye-go search -dork "weblogic" -facet "country" -figure "hist"
+./ZoomEye-go search "weblogic" -facet "country" -figure "hist"
 succeed to search (in 177.088662ms)
 
-[Facets Histogram]
+[ZoomEye Facets - HIST]
 
-  COUNTRY
+  Type: country
   
   United States  [232751]  ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉
           Japan  [ 45285]  ▉▉▉▉▉▉▉
@@ -163,7 +162,7 @@ succeed to search (in 177.088662ms)
 
 #### 加载分析本地数据
 
-`ZoomEye-go` 也可以通过 `load` 命令加载本地数据，并将它解析成搜索结果数据类型，支持与 `search` 命令类似的 `-count` 、 `-facet` 、 `-stat` 、 `-figure` 和 `-filter` 参数对数据进行统计分析，不同的是，`-save` 参数仅会保存 `-filter` 的执行结果。
+`ZoomEye-go` 也可以通过 `load` 命令加载本地数据文件，并将它解析成搜索结果数据类型，支持与 `search` 命令类似的 `-count` 、 `-facet` 、 `-stat` 、 `-figure` 和 `-filter` 参数对数据进行统计分析。不同的是，`-save` 参数仅会保存 `-filter` 的执行结果。
 
 可以通过 `load -h` 获取帮助。
 

cmd.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -19,6 +20,13 @@ func cmdInit(agent *ZoomEyeAgent) {
 	flag.StringVar(&args.apiKey, "apikey", "", "ZoomEye API-Key")
 	flag.StringVar(&args.username, "username", "", "ZoomEye account username")
 	flag.StringVar(&args.password, "password", "", "ZoomEye account password")
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "\nUsage of %s (init):\n", filepath.Base(os.Args[0]))
+		flag.PrintDefaults()
+		fmt.Fprintf(flag.CommandLine.Output(),
+			"Example:\n  ./ZoomEye-go init -apikey \"XXXXXXXX-XXXX-XXXXX-XXXX-XXXXXXXXXXX\"\n"+
+				"  ./ZoomEye-go init -username \"username@zoomeye.org\" -password \"password\"\n\n")
+	}
 	flag.Parse()
 	var (
 		result *zoomeye.ResourcesInfoResult
@@ -39,7 +47,7 @@ func cmdInit(agent *ZoomEyeAgent) {
 		return
 	}
 	successf("succeed to initialize")
-	infof("Resources Info", "Role:  %s\nQuota: %d", result.Plan, result.Resources.Search)
+	infof("ZoomEye Resources Info", "Role:  %s\nQuota: %d", result.Plan, result.Resources.Search)
 }
 
 func cmdInfo(agent *ZoomEyeAgent) {
@@ -56,12 +64,25 @@ func cmdInfo(agent *ZoomEyeAgent) {
 		return
 	}
 	successf("succeed to query")
-	infof("Resources Info", "Role:  %s\nQuota: %d", result.Plan, result.Resources.Search)
+	infof("ZoomEye Resources Info", "Role:  %s\nQuota: %d", result.Plan, result.Resources.Search)
+}
+
+func commandVal() string {
+	var v string
+	if len(os.Args) > 1 && !strings.HasPrefix(os.Args[1], "-") {
+		v = os.Args[1]
+		os.Args = append(os.Args[0:1], os.Args[2:]...)
+	}
+	return v
 }
 
 func cmdSearch(agent *ZoomEyeAgent) {
+	dork := commandVal()
+	if dork == "" {
+		warnf("search keyword missing, please run <zoomeye search -h> for help")
+		return
+	}
 	var args struct {
-		dork     string
 		num      int
 		resource string
 		force    bool
@@ -72,7 +93,6 @@ func cmdSearch(agent *ZoomEyeAgent) {
 		filter   string
 		save     bool
 	}
-	flag.StringVar(&args.dork, "dork", "", "[REQUIRED] The ZoomEye search keyword or ZoomEye exported file")
 	flag.IntVar(&args.num, "num", 20, "The number of search results that should be returned, multiple of 20")
 	flag.StringVar(&args.resource, "type", "host", "Specify the type of resource to search")
 	flag.BoolVar(&args.force, "force", false, "Ignore local and cache data")
@@ -82,13 +102,16 @@ func cmdSearch(agent *ZoomEyeAgent) {
 	flag.StringVar(&args.figure, "figure", "", "Output Pie or bar chart only be used under facet and stat")
 	flag.StringVar(&args.filter, "filter", "", "Output more clearer search results by set filter field")
 	flag.BoolVar(&args.save, "save", false, "Save the search results in JSON format")
-	if flag.Parse(); args.dork == "" {
-		warnf("required parameter missing, please run <zoomeye search -h> for help")
-		return
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "\nUsage of %s (search):\n", filepath.Base(os.Args[0]))
+		flag.PrintDefaults()
+		fmt.Fprintf(flag.CommandLine.Output(),
+			"Example:\n  ./Zoomeye-go search \"weblogic\" -facet \"app\" -count\n\n")
 	}
+	flag.Parse()
 	var (
 		start       = time.Now()
-		result, err = agent.Search(args.dork, args.num, args.resource, args.force)
+		result, err = agent.Search(dork, args.num, args.resource, args.force)
 		since       = time.Since(start)
 	)
 	if err != nil {
@@ -104,7 +127,7 @@ func cmdSearch(agent *ZoomEyeAgent) {
 	}
 	successf("succeed to search (in %v)", since)
 	if args.count {
-		infof("Total Count", "Count: %d", result.Total)
+		infof("ZoomEye Total", "Count: %d", result.Total)
 	}
 	if args.figure != "" {
 		if args.figure = strings.ToLower(args.figure); args.figure != "pie" {
@@ -124,7 +147,7 @@ func cmdSearch(agent *ZoomEyeAgent) {
 		printData(result)
 	}
 	if args.save {
-		name := fmt.Sprintf("%s_%s_%d", args.resource, url.QueryEscape(args.dork), args.num)
+		name := fmt.Sprintf("%s_%s_%d", args.resource, url.QueryEscape(dork), args.num)
 		if path, err := agent.Save(result, name); err != nil {
 			errorf("failed to save: %v", err)
 		} else {
@@ -134,34 +157,40 @@ func cmdSearch(agent *ZoomEyeAgent) {
 }
 
 func cmdLoad(agent *ZoomEyeAgent) {
+	file := commandVal()
+	if file == "" {
+		warnf("path of local data file missing, please run <zoomeye load -h> for help")
+		return
+	}
 	var args struct {
-		file   string
 		count  bool
 		facet  string
 		stat   string
 		figure string
 		filter string
 		save   bool
 	}
-	flag.StringVar(&args.file, "file", "", "[REQUIRED] The path of local data")
 	flag.BoolVar(&args.count, "count", false, "The total number of results in ZoomEye database in local data")
 	flag.StringVar(&args.facet, "facet", "", "Perform statistics on ZoomEye database in local data")
 	flag.StringVar(&args.stat, "stat", "", "Perform statistics in local data")
 	flag.StringVar(&args.figure, "figure", "", "Output Pie or bar chart only be used under facet and stat")
 	flag.StringVar(&args.filter, "filter", "", "Output more clearer results by set filter field in local data")
 	flag.BoolVar(&args.save, "save", false, "Save the filter data in JSON format")
-	if flag.Parse(); args.file == "" {
-		warnf("required parameter missing, please run <zoomeye load -h> for help")
-		return
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "\nUsage of %s (load):\n", filepath.Base(os.Args[0]))
+		flag.PrintDefaults()
+		fmt.Fprintf(flag.CommandLine.Output(),
+			"Example:\n  ./Zoomeye-go load \"data/host_weblogic_20.json\" -facet \"app\" -count\n\n")
 	}
-	result, err := agent.Load(args.file)
+	flag.Parse()
+	result, err := agent.Load(file)
 	if err != nil {
 		errorf("invalid local data: %v", err)
 		return
 	}
 	successf("succeed to load")
 	if args.count {
-		infof("Total Count", "Count: %d", result.Total)
+		infof("ZoomEye Total", "Count: %d", result.Total)
 	}
 	if args.figure != "" {
 		if args.figure = strings.ToLower(args.figure); args.figure != "pie" {
@@ -182,8 +211,8 @@ func cmdLoad(agent *ZoomEyeAgent) {
 	}
 	if args.save {
 		var (
-			ext  = filepath.Ext(args.file)
-			path = strings.TrimSuffix(args.file, ext) + "_filtered" + ext
+			ext  = filepath.Ext(file)
+			path = strings.TrimSuffix(file, ext) + "_filtered" + ext
 		)
 		if err := agent.SaveFilterData(result.FilterCache, path); err != nil {
 			errorf("failed to save: %v", err)

main.go

@@ -1,20 +1,12 @@
 package main
 
 import (
-	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 )
 
-func init() {
-	flag.Usage = func() {
-		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", filepath.Base(os.Args[0]))
-		flag.PrintDefaults()
-	}
-}
-
 func help() {
 	fmt.Printf("Usage of %s:\n"+
 		"  init\n        Initialize ZoomEye by username/password or API-Key\n"+
@@ -29,18 +21,13 @@ func help() {
 func main() {
 	var (
 		agent = NewAgent()
-		n     = len(os.Args)
+		cmd   string
 	)
-	if n == 1 {
-		// TODO: user interact mode
-		warnf("User-Interact mode is coming soon, please run <zoomeye -h> for help")
-		return
-	}
-	cmd := strings.ToLower(os.Args[1])
-	if n > 2 {
+	if len(os.Args) > 1 {
+		cmd = os.Args[1]
 		os.Args = append(os.Args[0:1], os.Args[2:]...)
 	}
-	switch cmd {
+	switch strings.ToLower(cmd) {
 	case "init":
 		cmdInit(agent)
 	case "info":
@@ -53,6 +40,8 @@ func main() {
 		cmdClean(agent)
 	case "help", "-help", "--help", "-h", "?":
 		help()
+	case "":
+		warnf("Cli-User-Interact mode is coming soon, please run <zoomeye -h> for help")
 	default:
 		warnf("input parameter error, please run <zoomeye -h> for help")
 	}

print.go

@@ -65,15 +65,15 @@ func print(s, color string) {
 }
 
 func errorf(format string, a ...interface{}) {
-	print(fmt.Sprintf(format, a...), "red")
+	print(fmt.Sprintf(format, a...), colorRed)
 }
 
 func successf(format string, a ...interface{}) {
-	print(fmt.Sprintf(format, a...), "green")
+	print(fmt.Sprintf(format, a...), colorGreen)
 }
 
 func warnf(format string, a ...interface{}) {
-	print(fmt.Sprintf(format, a...), "yellow")
+	print(fmt.Sprintf(format, a...), colorYellow)
 }
 
 func infof(title, format string, a ...interface{}) {
@@ -238,7 +238,7 @@ func pief(title string, body map[string][][]interface{}) {
 				}
 				if i > 0 {
 					if i == 1 {
-						builder.WriteString(c + "   " + colorf(strings.ToUpper(k), colorLightGreen) + "\n")
+						builder.WriteString(c + "   " + colorf("Type: "+k, colorLightGreen) + "\n")
 					} else if n := i - 3; n >= 0 && n < len(v) {
 						builder.WriteString(c + "   " +
 							colorf(fmt.Sprintf("%5.2f%%%% - %s", v[n][2].(float64)*100, v[n][0]), pieColors[n]) + "\n")
@@ -263,7 +263,7 @@ func histf(title string, body map[string][][]interface{}) {
 			} else {
 				first = false
 			}
-			builder.WriteString(colorf(strings.ToUpper(k), colorLightGreen) + "\n\n")
+			builder.WriteString(colorf("Type: "+k, colorLightGreen) + "\n\n")
 			var (
 				maxNameLen  int
 				maxCountLen int
@@ -324,13 +324,15 @@ func printFacet(result *zoomeye.SearchResult, facets []string, figure string) {
 	var (
 		head = [][2]interface{}{
 			[2]interface{}{"Type", 10},
-			[2]interface{}{"Facet", 35},
+			[2]interface{}{"Name", 35},
 			[2]interface{}{"Count", 20},
 		}
 		body = make(map[string][][]interface{})
 	)
-	for _, s := range facets {
-		if s = strings.ToLower(strings.TrimSpace(s)); result.Type == "host" && s == "app" {
+	for _, f := range facets {
+		f = strings.ToLower(strings.TrimSpace(f))
+		s := f
+		if result.Type == "host" && f == "app" {
 			s = "product"
 		}
 		if facet, ok := result.Facets[s]; ok {
@@ -350,16 +352,16 @@ func printFacet(result *zoomeye.SearchResult, facets []string, figure string) {
 				group = append(group, []interface{}{omitStr(name, 35), v.Count,
 					float64(v.Count) / float64(result.Total)})
 			}
-			body[s] = group
+			body[f] = group
 		}
 	}
 	switch figure {
 	case "":
-		tablef("Facets Info", head, body, false)
+		tablef("ZoomEye Facets", head, body, false)
 	case "pie":
-		pief("Facets Pie", body)
+		pief("ZoomEye Facets - PIE", body)
 	case "hist":
-		histf("Facets Histogram", body)
+		histf("ZoomEye Facets - HIST", body)
 	}
 }
 
@@ -384,11 +386,11 @@ func printStat(result *zoomeye.SearchResult, keys []string, figure string) {
 	}
 	switch figure {
 	case "":
-		tablef("Statistics Info", head, body, false)
+		tablef("Result Statistics", head, body, false)
 	case "pie":
-		pief("Statistics Pie", body)
+		pief("Result Statistics - PIE", body)
 	case "hist":
-		histf("Statistics Histogram", body)
+		histf("Result Statistics - HIST", body)
 	}
 }
 
@@ -438,7 +440,7 @@ func printFilter(result *zoomeye.SearchResult, keys []string) {
 			"items": items,
 		}
 	}
-	htablef("Filtered Data", body, [3]int{30, 15, 75}, true)
+	htablef("Result Filtered", body, [3]int{30, 15, 75}, true)
 }
 
 func withVersion(o interface{}) string {