improve glob feature and also exclude hidden folders from the path walk

Author: Pantani

starport/pkg/cosmosanalysis/module/module.go

@@ -224,7 +224,7 @@ func (d *moduleDiscoverer) discover(pkg protoanalysis.Package) (Module, error) {
 
 func (d *moduleDiscoverer) findModuleProtoPkgs(ctx context.Context) ([]protoanalysis.Package, error) {
 	// find out all proto packages inside blockchain.
-	allprotopkgs, err := protoanalysis.Parse(ctx, protoanalysis.PatternRecursive(d.sourcePath))
+	allprotopkgs, err := protoanalysis.Parse(ctx, d.sourcePath)
 	if err != nil {
 		return nil, err
 	}

starport/pkg/cosmosgen/generate_go.go

@@ -35,7 +35,7 @@ func (g *generator) generateGo() error {
 
 	// discover proto packages in the app.
 	pp := filepath.Join(g.appPath, g.protoDir)
-	pkgs, err := protoanalysis.Parse(g.ctx, protoanalysis.PatternRecursive(pp))
+	pkgs, err := protoanalysis.Parse(g.ctx, pp)
 	if err != nil {
 		return err
 	}

starport/pkg/localfs/search.go

@@ -1,18 +1,49 @@
 package localfs
 
 import (
+	"os"
+	"path/filepath"
 	"sort"
-
-	"github.com/mattn/go-zglob"
+	"strings"
 )
 
 // Search searches for files in the fs with given glob pattern by ensuring that
 // returned file paths are sorted.
-func Search(pattern string) (paths []string, err error) {
-	files, err := zglob.Glob(pattern)
+func Search(path, pattern string) (paths []string, err error) {
+	files, err := glob(path, pattern)
 	if err != nil {
 		return nil, err
 	}
-	sort.Strings(files)
 	return files, nil
 }
+
+// glob returns the names of all files matching pattern or nil
+// if there is no matching file in all sub-paths recursively. The
+// syntax of patterns is the same as in Match. The pattern may
+// describe hierarchical names such as /usr/*/bin/ed (assuming
+// the Separator is '/').
+func glob(path, pattern string) ([]string, error) {
+	files := make([]string, 0)
+	err := filepath.Walk(path, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		base := filepath.Base(path)
+		// skip hidden folders
+		if f.IsDir() && strings.HasPrefix(base, ".") {
+			return filepath.SkipDir
+		}
+		// avoid check directories
+		if f.IsDir() {
+			return nil
+		}
+		// check if the file name and pattern matches
+		matched, err := filepath.Match(pattern, base)
+		if err == nil && matched {
+			files = append(files, path)
+		}
+		return nil
+	})
+	sort.Strings(files)
+	return files, err
+}

starport/pkg/localfs/search_test.go

@@ -0,0 +1,124 @@
+package localfs
+
+import (
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func setupGlobTests() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	tmpdir, err := ioutil.TempDir(dir, "glob-test")
+	if err != nil {
+		return "", err
+	}
+	dirs := []string{"foo/bar", "foo/baz"}
+	for _, dir := range dirs {
+		err = os.MkdirAll(filepath.Join(tmpdir, dir), 0755)
+		if err != nil {
+			return "", err
+		}
+	}
+	files := []string{
+		"foo/file.proto",
+		"foo/bar/file1.proto",
+		"foo/bar/file2.proto",
+		"foo/baz/file.proto",
+		"foo/file",
+		"foo/baz/file",
+	}
+	for _, file := range files {
+		err = ioutil.WriteFile(filepath.Join(tmpdir, file), []byte{}, 0644)
+		if err != nil {
+			return "", err
+		}
+	}
+	return tmpdir, nil
+}
+
+func Test_glob(t *testing.T) {
+	tmpdir, err := setupGlobTests()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	type args struct {
+		path    string
+		pattern string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+		err  error
+	}{
+		{
+			name: "get all proto files by pattern",
+			args: args{
+				path:    tmpdir,
+				pattern: "*.proto",
+			},
+			want: []string{
+				tmpdir + "/foo/bar/file1.proto",
+				tmpdir + "/foo/bar/file2.proto",
+				tmpdir + "/foo/baz/file.proto",
+				tmpdir + "/foo/file.proto",
+			},
+			err: nil,
+		}, {
+			name: "get only one proto file by name",
+			args: args{
+				path:    tmpdir,
+				pattern: "file1.proto",
+			},
+			want: []string{tmpdir + "/foo/bar/file1.proto"},
+			err:  nil,
+		}, {
+			name: "get two proto files by name",
+			args: args{
+				path:    tmpdir,
+				pattern: "file.proto",
+			},
+			want: []string{tmpdir + "/foo/baz/file.proto", tmpdir + "/foo/file.proto"},
+			err:  nil,
+		}, {
+			name: "get a specific file by name",
+			args: args{
+				path:    tmpdir,
+				pattern: "file",
+			},
+			want: []string{tmpdir + "/foo/baz/file", tmpdir + "/foo/file"},
+			err:  nil,
+		}, {
+			name: "error invalid directory",
+			args: args{
+				path:    "no-directory",
+				pattern: "file",
+			},
+			want: []string{tmpdir + "/foo/baz/file", tmpdir + "/foo/file"},
+			err:  &fs.PathError{Op: "lstat", Path: "no-directory", Err: syscall.ENOENT},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := glob(tt.args.path, tt.args.pattern)
+			if tt.err != nil {
+				require.Error(t, err)
+				assert.EqualValues(t, tt.err, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.EqualValues(t, tt.want, got)
+		})
+	}
+}

starport/pkg/protoanalysis/parser.go

@@ -18,10 +18,10 @@ type parser struct {
 
 // parse parses proto files in the fs that matches with pattern and returns
 // the low level representations of proto packages.
-func parse(ctx context.Context, pattern string) ([]*pkg, error) {
+func parse(ctx context.Context, path, pattern string) ([]*pkg, error) {
 	pr := &parser{}
 
-	paths, err := localfs.Search(pattern)
+	paths, err := localfs.Search(path, pattern)
 	if err != nil {
 		return nil, err
 	}

starport/pkg/protoanalysis/protoanalysis.go

@@ -7,9 +7,11 @@ import (
 	"github.com/tendermint/starport/starport/pkg/localfs"
 )
 
+const protoFilePattern = "*.proto"
+
 // Parse parses proto packages by finding them with given glob pattern.
-func Parse(ctx context.Context, pattern string) ([]Package, error) {
-	parsed, err := parse(ctx, pattern)
+func Parse(ctx context.Context, path string) ([]Package, error) {
+	parsed, err := parse(ctx, path, protoFilePattern)
 	if err != nil {
 		return nil, err
 	}
@@ -24,9 +26,6 @@ func Parse(ctx context.Context, pattern string) ([]Package, error) {
 }
 
 // SearchRecursive recursively finds all proto files under path.
-func SearchRecursive(dir string) ([]string, error) {
-	return localfs.Search(PatternRecursive(dir))
+func SearchRecursive(path string) ([]string, error) {
+	return localfs.Search(path, protoFilePattern)
 }
-
-// PatternRecursive returns a recursive glob search pattern to find all proto files under path.
-func PatternRecursive(dir string) string { return dir + "/**/*.proto" }

starport/pkg/protoanalysis/protoanalysis_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestLiquidity(t *testing.T) {
-	packages, err := Parse(context.Background(), PatternRecursive("testdata/liquidity"))
+	packages, err := Parse(context.Background(), "testdata/liquidity")
 	require.NoError(t, err)
 
 	expected := []Package{