v5: New multi-json loader (v2fly#451)

* scalable commands column

* new multi-json loader
For both internal & external json loader

This commit also:
* applies -confdir to other formats, e.g. "yaml" in the future
* multiple assign of -confdir is accepted
* add flag to load confdir recursively
* config loader can have alias name
* json loader also accepts .jsonc
* add merge command
* add help topics for json merge, format loader
* format loaders don't panic

* apply lint style

* add merge test

* merge same tag in array, solve v2fly/discussion#97

* apply lint style

* merge code optimize

* fix merge cmdarg.Arg

* update cmd description

* improve merge logic
* fix zero value overwrite
* fix "null" lost after array merge

* code optimize

* fix merged slices not sorted

* code optimize

* add package doc

* fix a typo

commands/all/commands.go

@@ -13,5 +13,10 @@ func init() {
 		cmdTLS,
 		cmdUUID,
 		cmdVerify,
+		cmdMerge,
+
+		// documents
+		docFormat,
+		docMerge,
 	)
 }

commands/all/convert.go

@@ -2,58 +2,60 @@ package all
 
 import (
 	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
 	"os"
-	"strings"
-	"time"
 
 	"google.golang.org/protobuf/proto"
 	"v2ray.com/core/commands/base"
-	"v2ray.com/core/common"
-	"v2ray.com/core/common/buf"
-	"v2ray.com/core/infra/conf"
+	"v2ray.com/core/infra/conf/merge"
 	"v2ray.com/core/infra/conf/serial"
 )
 
 var cmdConvert = &base.Command{
-	UsageLine: "{{.Exec}} convert [json file] [json file] ...",
+	UsageLine: "{{.Exec}} convert [-r] [c1.json] [<url>.json] [dir1] ...",
 	Short:     "Convert multiple json config to protobuf",
 	Long: `
-Convert multiple json config to protobuf.
+Convert JSON config to protobuf.
+
+If multiple JSON files or folders specified, it merges them first, then convert.
+
+Arguments:
+
+	-r
+		Load confdir recursively.
 
 Examples:
 
-    {{.Exec}} {{.LongName}} config.json c1.json c2.json <url>.json
+	{{.Exec}} {{.LongName}} config.json
+	{{.Exec}} {{.LongName}} c1.json c2.json
+	{{.Exec}} {{.LongName}} c1.json https://url.to/c2.json
+	{{.Exec}} {{.LongName}} "path/to/json_dir"
 `,
 }
 
 func init() {
 	cmdConvert.Run = executeConvert // break init loop
 }
 
+var convertReadDirRecursively = cmdConvert.Flag.Bool("r", false, "")
+
 func executeConvert(cmd *base.Command, args []string) {
-	unnamedArgs := cmdConvert.Flag.Args()
-	if len(unnamedArgs) < 1 {
+	unnamed := cmd.Flag.Args()
+	files := resolveFolderToFiles(unnamed, *convertReadDirRecursively)
+	if len(files) == 0 {
 		base.Fatalf("empty config list")
 	}
 
-	conf := &conf.Config{}
-	for _, arg := range unnamedArgs {
-		fmt.Fprintf(os.Stderr, "Read config: %s", arg)
-		r, err := loadArg(arg)
-		common.Must(err)
-		c, err := serial.DecodeJSONConfig(r)
-		if err != nil {
-			base.Fatalf(err.Error())
-		}
-		conf.Override(c, arg)
+	data, err := merge.FilesToJSON(files)
+	if err != nil {
+		base.Fatalf("failed to load json: %s", err)
+	}
+	r := bytes.NewReader(data)
+	cf, err := serial.DecodeJSONConfig(r)
+	if err != nil {
+		base.Fatalf("failed to decode json: %s", err)
 	}
 
-	pbConfig, err := conf.Build()
+	pbConfig, err := cf.Build()
 	if err != nil {
 		base.Fatalf(err.Error())
 	}
@@ -67,60 +69,3 @@ func executeConvert(cmd *base.Command, args []string) {
 		base.Fatalf("failed to write proto config: %s", err)
 	}
 }
-
-// loadArg loads one arg, maybe an remote url, or local file path
-func loadArg(arg string) (out io.Reader, err error) {
-	var data []byte
-	switch {
-	case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"):
-		data, err = FetchHTTPContent(arg)
-
-	case arg == "stdin:":
-		data, err = ioutil.ReadAll(os.Stdin)
-
-	default:
-		data, err = ioutil.ReadFile(arg)
-	}
-
-	if err != nil {
-		return
-	}
-	out = bytes.NewBuffer(data)
-	return
-}
-
-// FetchHTTPContent dials https for remote content
-func FetchHTTPContent(target string) ([]byte, error) {
-	parsedTarget, err := url.Parse(target)
-	if err != nil {
-		return nil, newError("invalid URL: ", target).Base(err)
-	}
-
-	if s := strings.ToLower(parsedTarget.Scheme); s != "http" && s != "https" {
-		return nil, newError("invalid scheme: ", parsedTarget.Scheme)
-	}
-
-	client := &http.Client{
-		Timeout: 30 * time.Second,
-	}
-	resp, err := client.Do(&http.Request{
-		Method: "GET",
-		URL:    parsedTarget,
-		Close:  true,
-	})
-	if err != nil {
-		return nil, newError("failed to dial to ", target).Base(err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, newError("unexpected HTTP status code: ", resp.StatusCode)
-	}
-
-	content, err := buf.ReadAllToBytes(resp.Body)
-	if err != nil {
-		return nil, newError("failed to read HTTP response").Base(err)
-	}
-
-	return content, nil
-}

commands/all/format_doc.go

@@ -0,0 +1,50 @@
+package all
+
+import (
+	"v2ray.com/core/commands/base"
+)
+
+var docFormat = &base.Command{
+	UsageLine: "{{.Exec}} format-loader",
+	Short:     "config formats and loading",
+	Long: `
+{{.Exec}} supports different config formats:
+
+	* json (.json, .jsonc)
+	  The default loader, multiple config files support.
+
+	* yaml (.yml)
+	  The yaml loader (coming soon?), multiple config files support.
+
+	* protobuf / pb (.pb)
+	  Single conifg file support. If multiple files assigned, 
+	  only the first one is loaded.
+
+If "-format" is not explicitly specified, {{.Exec}} will choose 
+a loader by detecting the extension of the first config file, or 
+use the default loader.
+
+The following explains how format loaders behave with examples.
+
+Examples:
+
+	{{.Exec}} run -d dir                                  (1)
+	{{.Exec}} run -format=protobuf -d dir                 (2)
+	{{.Exec}} test -c c1.yml -d dir                       (3)
+	{{.Exec}} test -format=pb -c c1.json                  (4)
+
+(1) The default json loader is used, {{.Exec}} will try to load all 
+	json files in the "dir".
+
+(2) The protobuf loader is specified, {{.Exec}} will try to find 
+	all protobuf files in the "dir", but only the the first 
+	.pb file is loaded.
+
+(3) The yaml loader is selected because of the "c1.yml" file, 
+	{{.Exec}} will try to load "c1.yml" and all yaml files in 
+	the "dir".
+
+(4) The protobuf loader is specified, {{.Exec}} will load 
+	"c1.json" as protobuf, no matter its extension.
+`,
+}

commands/all/merge.go

@@ -0,0 +1,101 @@
+package all
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"v2ray.com/core/commands/base"
+	"v2ray.com/core/infra/conf/merge"
+)
+
+var cmdMerge = &base.Command{
+	UsageLine: "{{.Exec}} merge [-r] [c1.json] [url] [dir1] ...",
+	Short:     "Merge json files into one",
+	Long: `
+Merge JSON files into one.
+
+Arguments:
+
+	-r
+		Load confdir recursively.
+
+Examples:
+
+	{{.Exec}} {{.LongName}} c1.json c2.json 
+	{{.Exec}} {{.LongName}} c1.json https://url.to/c2.json 
+	{{.Exec}} {{.LongName}} "path/to/json_dir"
+`,
+}
+
+func init() {
+	cmdMerge.Run = executeMerge
+}
+
+var mergeReadDirRecursively = cmdMerge.Flag.Bool("r", false, "")
+
+func executeMerge(cmd *base.Command, args []string) {
+	unnamed := cmd.Flag.Args()
+	files := resolveFolderToFiles(unnamed, *mergeReadDirRecursively)
+	if len(files) == 0 {
+		base.Fatalf("empty config list")
+	}
+
+	data, err := merge.FilesToJSON(files)
+	if err != nil {
+		base.Fatalf(err.Error())
+	}
+	if _, err := os.Stdout.Write(data); err != nil {
+		base.Fatalf(err.Error())
+	}
+}
+
+// resolveFolderToFiles expands folder path (if any and it exists) to file paths.
+// Any other paths, like file, even URL, it returns them as is.
+func resolveFolderToFiles(paths []string, recursively bool) []string {
+	dirReader := readConfDir
+	if recursively {
+		dirReader = readConfDirRecursively
+	}
+	files := make([]string, 0)
+	for _, p := range paths {
+		i, err := os.Stat(p)
+		if err == nil && i.IsDir() {
+			files = append(files, dirReader(p)...)
+			continue
+		}
+		files = append(files, p)
+	}
+	return files
+}
+
+func readConfDir(dirPath string) []string {
+	confs, err := ioutil.ReadDir(dirPath)
+	if err != nil {
+		base.Fatalf("failed to read dir %s: %s", dirPath, err)
+	}
+	files := make([]string, 0)
+	for _, f := range confs {
+		ext := filepath.Ext(f.Name())
+		if ext == ".json" || ext == ".jsonc" {
+			files = append(files, filepath.Join(dirPath, f.Name()))
+		}
+	}
+	return files
+}
+
+// getFolderFiles get files in the folder and it's children
+func readConfDirRecursively(dirPath string) []string {
+	files := make([]string, 0)
+	err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
+		ext := filepath.Ext(path)
+		if ext == ".json" || ext == ".jsonc" {
+			files = append(files, path)
+		}
+		return nil
+	})
+	if err != nil {
+		base.Fatalf("failed to read dir %s: %s", dirPath, err)
+	}
+	return files
+}

commands/all/merge_doc.go

@@ -0,0 +1,66 @@
+package all
+
+import (
+	"v2ray.com/core/commands/base"
+)
+
+var docMerge = &base.Command{
+	UsageLine: "{{.Exec}} json-merge",
+	Short:     "json merge logic",
+	Long: `
+Merging of JSON configs is applied in following commands:
+
+	{{.Exec}} run -c c1.json -c c2.json ...
+	{{.Exec}} merge c1.json https://url.to/c2.json ...
+	{{.Exec}} convert c1.json dir1 ...
+
+Suppose we have 2 JSON files,
+
+The 1st one:
+
+	{
+	  "log": {"access": "some_value", "loglevel": "debug"},
+	  "inbounds": [{"tag": "in-1"}],
+	  "outbounds": [{"_priority": 100, "tag": "out-1"}],
+	  "routing": {"rules": [
+		{"_tag":"default_route","inboundTag":["in-1"],"outboundTag":"out-1"}
+	  ]}
+	}
+
+The 2nd one:
+
+	{
+	  "log": {"loglevel": "error"},
+	  "inbounds": [{"tag": "in-2"}],
+	  "outbounds": [{"_priority": -100, "tag": "out-2"}],
+	  "routing": {"rules": [
+		{"inboundTag":["in-2"],"outboundTag":"out-2"},
+		{"_tag":"default_route","inboundTag":["in-1.1"],"outboundTag":"out-1.1"}
+	  ]}
+	}
+
+Output:
+
+	{
+	  // loglevel is overwritten
+	  "log": {"access": "some_value", "loglevel": "error"},
+	  "inbounds": [{"tag": "in-1"}, {"tag": "in-2"}],
+	  "outbounds": [
+		{"tag": "out-2"}, // note the order is affected by priority
+		{"tag": "out-1"}
+	  ],
+	  "routing": {"rules": [
+		// note 3 rules are merged into 2, and outboundTag is overwritten,
+		// because 2 of them has same tag
+		{"inboundTag":["in-1","in-1.1"],"outboundTag":"out-1.1"}
+		{"inboundTag":["in-2"],"outboundTag":"out-2"}
+	  ]}
+	}
+
+Explained: 
+
+- Simple values (string, number, boolean) are overwritten, others are merged
+- Elements with same "tag" (or "_tag") in an array will be merged
+- Add "_priority" property to array elements will help sort the array
+`,
+}

commands/base/env.go

@@ -7,7 +7,10 @@ import (
 
 // CommandEnvHolder is a struct holds the environment info of commands
 type CommandEnvHolder struct {
+	// Excutable name of current binary
 	Exec string
+	// commands column width of current command
+	CommandsWidth int
 }
 
 // CommandEnv holds the environment info of commands