feat(pkg/protoanalysis): ignore hidden files (ignite#1296)

* improve glob feature and also exclude hidden folders from the path walk

* fix the integration tests

Author: Pantani

go.mod

@@ -31,7 +31,6 @@ require (
 	github.com/jpillora/chisel v1.7.3
 	github.com/kr/pretty v0.2.1
 	github.com/manifoldco/promptui v0.8.0
-	github.com/mattn/go-zglob v0.0.3
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/otiai10/copy v1.4.2
@@ -57,5 +56,5 @@ require (
 
 replace (
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
- 	github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4
+	github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4
 )

go.sum

@@ -825,7 +825,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
 github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
-github.com/mattn/go-zglob v0.0.3 h1:6Ry4EYsScDyt5di4OI6xw1bYhOqfE5S33Z1OPy+d+To=
 github.com/mattn/go-zglob v0.0.3/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=

starport/pkg/cosmosanalysis/module/module.go

@@ -207,7 +207,7 @@ func (d *moduleDiscoverer) discover(pkg protoanalysis.Package) (Module, error) {
 
 func (d *moduleDiscoverer) findModuleProtoPkgs(ctx context.Context) ([]protoanalysis.Package, error) {
 	// find out all proto packages inside blockchain.
-	allprotopkgs, err := protoanalysis.Parse(ctx, protoanalysis.PatternRecursive(d.sourcePath))
+	allprotopkgs, err := protoanalysis.Parse(ctx, d.sourcePath)
 	if err != nil {
 		return nil, err
 	}

starport/pkg/cosmosgen/generate_go.go

@@ -35,7 +35,7 @@ func (g *generator) generateGo() error {
 
 	// discover proto packages in the app.
 	pp := filepath.Join(g.appPath, g.protoDir)
-	pkgs, err := protoanalysis.Parse(g.ctx, protoanalysis.PatternRecursive(pp))
+	pkgs, err := protoanalysis.Parse(g.ctx, pp)
 	if err != nil {
 		return err
 	}

starport/pkg/cosmosgen/generate_javascript.go

@@ -8,10 +8,10 @@ import (
 	"strings"
 
 	"github.com/iancoleman/strcase"
-	"github.com/mattn/go-zglob"
 	"github.com/tendermint/starport/starport/pkg/cosmosanalysis/module"
 	"github.com/tendermint/starport/starport/pkg/giturl"
 	"github.com/tendermint/starport/starport/pkg/gomodulepath"
+	"github.com/tendermint/starport/starport/pkg/localfs"
 	"github.com/tendermint/starport/starport/pkg/nodetime/programs/sta"
 	tsproto "github.com/tendermint/starport/starport/pkg/nodetime/programs/ts-proto"
 	"github.com/tendermint/starport/starport/pkg/nodetime/programs/tsc"
@@ -28,10 +28,10 @@ var (
 	jsOpenAPIOut = []string{
 		"--openapiv2_out=logtostderr=true,allow_merge=true,Mgoogle/protobuf/any.proto=github.com/cosmos/cosmos-sdk/codec/types:.",
 	}
-
-	vuexRootMarker = "vuex-root"
 )
 
+const vuexRootMarker = "vuex-root"
+
 type jsGenerator struct {
 	g *generator
 }
@@ -163,7 +163,7 @@ func (g *jsGenerator) generateModule(ctx context.Context, tsprotoPluginPath, app
 }
 
 func (g *jsGenerator) generateVuexModuleLoader() error {
-	modulePaths, err := zglob.Glob(filepath.Join(g.g.o.vuexStoreRootPath, "/**/"+vuexRootMarker))
+	modulePaths, err := localfs.Search(g.g.o.vuexStoreRootPath, vuexRootMarker)
 	if err != nil {
 		return err
 	}

starport/pkg/localfs/search.go

@@ -1,18 +1,46 @@
 package localfs
 
 import (
+	"os"
+	"path/filepath"
 	"sort"
-
-	"github.com/mattn/go-zglob"
+	"strings"
 )
 
 // Search searches for files in the fs with given glob pattern by ensuring that
 // returned file paths are sorted.
-func Search(pattern string) (paths []string, err error) {
-	files, err := zglob.Glob(pattern)
-	if err != nil {
+func Search(path, pattern string) ([]string, error) {
+	files := make([]string, 0)
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return files, nil
+		}
 		return nil, err
 	}
+
+	err := filepath.Walk(path, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		base := filepath.Base(path)
+		// skip hidden folders
+		if f.IsDir() && strings.HasPrefix(base, ".") {
+			return filepath.SkipDir
+		}
+		// avoid check directories
+		if f.IsDir() {
+			return nil
+		}
+		// check if the file name and pattern matches
+		matched, err := filepath.Match(pattern, base)
+		if err != nil {
+			return err
+		}
+		if matched {
+			files = append(files, path)
+		}
+		return nil
+	})
 	sort.Strings(files)
-	return files, nil
+	return files, err
 }

starport/pkg/localfs/search_test.go

@@ -0,0 +1,111 @@
+package localfs
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func setupGlobTests(t *testing.T, files []string) string {
+	t.Helper()
+	tmpdir, err := os.MkdirTemp("", "glob-test")
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		os.RemoveAll(tmpdir)
+	})
+	for _, file := range files {
+		fileDir := filepath.Dir(file)
+		fileDir = filepath.Join(tmpdir, fileDir)
+		err = os.MkdirAll(fileDir, 0755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(tmpdir, file), []byte{}, 0644)
+		require.NoError(t, err)
+	}
+	return tmpdir
+}
+
+func TestSearch(t *testing.T) {
+	files := []string{
+		"foo/file.proto",
+		"foo/bar/file1.proto",
+		"foo/bar/file2.proto",
+		"foo/baz/file.proto",
+		"foo/file",
+		"foo/baz/file",
+	}
+	tmpdir := setupGlobTests(t, files)
+	type args struct {
+		path    string
+		pattern string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+		err  error
+	}{
+		{
+			name: "get all proto files by pattern",
+			args: args{
+				path:    tmpdir,
+				pattern: "*.proto",
+			},
+			want: []string{
+				filepath.Join(tmpdir, "foo/bar/file1.proto"),
+				filepath.Join(tmpdir, "foo/bar/file2.proto"),
+				filepath.Join(tmpdir, "foo/baz/file.proto"),
+				filepath.Join(tmpdir, "foo/file.proto"),
+			},
+		}, {
+			name: "get only one proto file by name",
+			args: args{
+				path:    tmpdir,
+				pattern: "file1.proto",
+			},
+			want: []string{filepath.Join(tmpdir, "foo/bar/file1.proto")},
+		}, {
+			name: "get two proto files by name",
+			args: args{
+				path:    tmpdir,
+				pattern: "file.proto",
+			},
+			want: []string{
+				filepath.Join(tmpdir, "foo/baz/file.proto"),
+				filepath.Join(tmpdir, "foo/file.proto"),
+			},
+		}, {
+			name: "get a specific file by name",
+			args: args{
+				path:    tmpdir,
+				pattern: "file",
+			},
+			want: []string{
+				filepath.Join(tmpdir, "foo/baz/file"),
+				filepath.Join(tmpdir, "foo/file"),
+			},
+		}, {
+			name: "not found directory",
+			args: args{
+				path:    "no-directory",
+				pattern: "file",
+			},
+			want: []string{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Search(tt.args.path, tt.args.pattern)
+			if tt.err != nil {
+				require.Error(t, err)
+				assert.EqualValues(t, tt.err, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.EqualValues(t, tt.want, got)
+		})
+	}
+}

starport/pkg/protoanalysis/parser.go

@@ -18,10 +18,10 @@ type parser struct {
 
 // parse parses proto files in the fs that matches with pattern and returns
 // the low level representations of proto packages.
-func parse(ctx context.Context, pattern string) ([]*pkg, error) {
+func parse(ctx context.Context, path, pattern string) ([]*pkg, error) {
 	pr := &parser{}
 
-	paths, err := localfs.Search(pattern)
+	paths, err := localfs.Search(path, pattern)
 	if err != nil {
 		return nil, err
 	}

starport/pkg/protoanalysis/protoanalysis.go

@@ -7,9 +7,11 @@ import (
 	"github.com/tendermint/starport/starport/pkg/localfs"
 )
 
+const protoFilePattern = "*.proto"
+
 // Parse parses proto packages by finding them with given glob pattern.
-func Parse(ctx context.Context, pattern string) ([]Package, error) {
-	parsed, err := parse(ctx, pattern)
+func Parse(ctx context.Context, path string) ([]Package, error) {
+	parsed, err := parse(ctx, path, protoFilePattern)
 	if err != nil {
 		return nil, err
 	}
@@ -24,9 +26,6 @@ func Parse(ctx context.Context, pattern string) ([]Package, error) {
 }
 
 // SearchRecursive recursively finds all proto files under path.
-func SearchRecursive(dir string) ([]string, error) {
-	return localfs.Search(PatternRecursive(dir))
+func SearchRecursive(path string) ([]string, error) {
+	return localfs.Search(path, protoFilePattern)
 }
-
-// PatternRecursive returns a recursive glob search pattern to find all proto files under path.
-func PatternRecursive(dir string) string { return dir + "/**/*.proto" }

starport/pkg/protoanalysis/protoanalysis_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestLiquidity(t *testing.T) {
-	packages, err := Parse(context.Background(), PatternRecursive("testdata/liquidity"))
+	packages, err := Parse(context.Background(), "testdata/liquidity")
 	require.NoError(t, err)
 
 	expected := []Package{