From 999b4479dc488df0de8bf028a716004e2cff8ce6 Mon Sep 17 00:00:00 2001 From: Thibault Jamet Date: Tue, 1 Jun 2021 10:41:11 +0200 Subject: [PATCH 1/3] utils: add Walk function to walk over a filesystem Adapted from the official filepath.Walk, this function allows to walk trough a whole filesystem and discover all its files --- util/walk.go | 70 +++++++++++++++++ util/walk_test.go | 194 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 util/walk.go create mode 100644 util/walk_test.go diff --git a/util/walk.go b/util/walk.go new file mode 100644 index 0000000..674ff4e --- /dev/null +++ b/util/walk.go @@ -0,0 +1,70 @@ +package util + +import ( + "os" + "path/filepath" + + "github.com/go-git/go-billy/v5" +) + +// walk recursively descends path, calling walkFn +// adapted from https://golang.org/src/path/filepath/path.go +func walk(fs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + names, err := readdirnames(fs, path) + err1 := walkFn(path, info, err) + // If err != nil, walk can't walk into this directory. + // err1 != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of err and err1 isn't nil, walk will return. + if err != nil || err1 != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore err and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return err1 + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := fs.Lstat(filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling fn for each file or directory in the tree, including root. +// +// All errors that arise visiting files and directories are filtered by fn: see the WalkFunc documentation for details. +// +// The files are walked in lexical order, which makes the output deterministic but requires Walk to read an entire directory into memory before proceeding to walk that directory. +// +// Walk does not follow symbolic links. +// +// adapted from https://github.com/golang/go/blob/3b770f2ccb1fa6fecc22ea822a19447b10b70c5c/src/path/filepath/path.go#L500 +func Walk(fs billy.Filesystem, root string, walkFn filepath.WalkFunc) error { + info, err := fs.Lstat(root) + + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(fs, root, info, walkFn) + } + if err == filepath.SkipDir { + return nil + } + return err +} diff --git a/util/walk_test.go b/util/walk_test.go new file mode 100644 index 0000000..ae4c390 --- /dev/null +++ b/util/walk_test.go @@ -0,0 +1,194 @@ +package util_test + +import ( + "errors" + "fmt" + "io/fs" + "path/filepath" + "reflect" + "testing" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-billy/v5/util" + + . "gopkg.in/check.v1" +) + +type WalkSuite struct{} + +func TestWalk(t *testing.T) { TestingT(t) } + +var _ = Suite(&WalkSuite{}) + +func (s *WalkSuite) TestWalkCanSkipTopDirectory(c *C) { + filesystem := memfs.New() + c.Assert(util.Walk(filesystem, "/root/that/does/not/exist", func(path string, info fs.FileInfo, err error) error { return filepath.SkipDir }), IsNil) +} + +func (s *WalkSuite) TestWalkReturnsAnErrorWhenRootDoesNotExist(c *C) { + filesystem := memfs.New() + c.Assert(util.Walk(filesystem, "/root/that/does/not/exist", func(path string, info fs.FileInfo, err error) error { return err }), NotNil) +} + +func (s *WalkSuite) TestWalkOnPlainFile(c *C) { + filesystem := memfs.New() + createFile(c, filesystem, "./README.md") + discoveredPaths := []string{} + c.Assert(util.Walk(filesystem, "./README.md", func(path string, info fs.FileInfo, err error) error { + discoveredPaths = append(discoveredPaths, path) + return nil + }), IsNil) + c.Assert(discoveredPaths, DeepEquals, []string{"./README.md"}) +} + +func (s *WalkSuite) TestWalkOnExistingFolder(c *C) { + filesystem := memfs.New() + createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") + createFile(c, filesystem, "path/to/some/file") + discoveredPaths := []string{} + c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + discoveredPaths = append(discoveredPaths, path) + return nil + }), IsNil) + c.Assert(discoveredPaths, Contains, "path") + c.Assert(discoveredPaths, Contains, "path/to") + c.Assert(discoveredPaths, Contains, "path/to/some") + c.Assert(discoveredPaths, Contains, "path/to/some/file") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder/that") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder/that/contain") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder/that/contain/file") +} + +func (s *WalkSuite) TestWalkCanSkipFolder(c *C) { + filesystem := memfs.New() + createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") + createFile(c, filesystem, "path/to/some/file") + discoveredPaths := []string{} + c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + discoveredPaths = append(discoveredPaths, path) + if path == "path/to/some/subfolder" { + return filepath.SkipDir + } + return nil + }), IsNil) + c.Assert(discoveredPaths, Contains, "path") + c.Assert(discoveredPaths, Contains, "path/to") + c.Assert(discoveredPaths, Contains, "path/to/some") + c.Assert(discoveredPaths, Contains, "path/to/some/file") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that/contain") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that/contain/file") +} + +func (s *WalkSuite) TestWalkStopsOnError(c *C) { + filesystem := memfs.New() + createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") + createFile(c, filesystem, "path/to/some/file") + discoveredPaths := []string{} + c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + discoveredPaths = append(discoveredPaths, path) + if path == "path/to/some/subfolder" { + return errors.New("uncaught error") + } + return nil + }), NotNil) + c.Assert(discoveredPaths, Contains, "path") + c.Assert(discoveredPaths, Contains, "path/to") + c.Assert(discoveredPaths, Contains, "path/to/some") + c.Assert(discoveredPaths, Contains, "path/to/some/file") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that/contain") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that/contain/file") +} + +func (s *WalkSuite) TestWalkForwardsStatErrors(c *C) { + memFilesystem := memfs.New() + filesystem := &fnFs{ + Filesystem: memFilesystem, + lstat: func(path string) (fs.FileInfo, error) { + if path == "path/to/some/subfolder" { + return nil, errors.New("uncaught error") + } + return memFilesystem.Lstat(path) + }, + } + + createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") + createFile(c, filesystem, "path/to/some/file") + discoveredPaths := []string{} + c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + discoveredPaths = append(discoveredPaths, path) + if path == "path/to/some/subfolder" { + c.Assert(err, NotNil) + } + return err + }), NotNil) + c.Assert(discoveredPaths, Contains, "path") + c.Assert(discoveredPaths, Contains, "path/to") + c.Assert(discoveredPaths, Contains, "path/to/some") + c.Assert(discoveredPaths, Contains, "path/to/some/file") + c.Assert(discoveredPaths, Contains, "path/to/some/subfolder") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that/contain") + c.Assert(discoveredPaths, NotContain, "path/to/some/subfolder/that/contain/file") +} + +func createFile(c *C, filesystem billy.Filesystem, path string) { + fd, err := filesystem.Create(path) + c.Assert(err, IsNil) + if err != nil { + fd.Close() + } +} + +type fnFs struct { + billy.Filesystem + lstat func(path string) (fs.FileInfo, error) +} + +func (f *fnFs) Lstat(path string) (fs.FileInfo, error) { + if f.lstat != nil { + return f.lstat(path) + } + return nil, errors.New("not implemented") +} + +type containsChecker struct { + *CheckerInfo +} + +func (checker *containsChecker) Check(params []interface{}, names []string) (result bool, err string) { + defer func() { + if v := recover(); v != nil { + result = false + err = fmt.Sprint(v) + } + }() + + value := reflect.ValueOf(params[0]) + result = false + err = fmt.Sprintf("%v does not contain %v", params[0], params[1]) + switch value.Kind() { + case reflect.Array, reflect.Slice: + for i := 0; i < value.Len(); i++ { + r := reflect.DeepEqual(value.Index(i).Interface(), params[1]) + if r { + result = true + err = "" + } + } + default: + return false, "obtained value type is not iterable" + } + return +} + +var Contains Checker = &containsChecker{ + &CheckerInfo{Name: "Contains", Params: []string{"obtained", "expected"}}, +} + +var NotContain Checker = Not(Contains) From e0768be422ff616fc042d1d62bfa65962f716ad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Thu, 3 Jun 2021 11:19:37 +0200 Subject: [PATCH 2/3] utils: Walk, minor style changes --- util/walk.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/util/walk.go b/util/walk.go index 674ff4e..1531bca 100644 --- a/util/walk.go +++ b/util/walk.go @@ -46,25 +46,27 @@ func walk(fs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.Wa return nil } -// Walk walks the file tree rooted at root, calling fn for each file or directory in the tree, including root. +// Walk walks the file tree rooted at root, calling fn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by fn: see the WalkFunc documentation for +// details. // -// All errors that arise visiting files and directories are filtered by fn: see the WalkFunc documentation for details. -// -// The files are walked in lexical order, which makes the output deterministic but requires Walk to read an entire directory into memory before proceeding to walk that directory. -// -// Walk does not follow symbolic links. -// -// adapted from https://github.com/golang/go/blob/3b770f2ccb1fa6fecc22ea822a19447b10b70c5c/src/path/filepath/path.go#L500 +// The files are walked in lexical order, which makes the output deterministic +// but requires Walk to read an entire directory into memory before proceeding +// to walk that directory. Walk does not follow symbolic links. +// +// Function adapted from https://github.com/golang/go/blob/3b770f2ccb1fa6fecc22ea822a19447b10b70c5c/src/path/filepath/path.go#L500 func Walk(fs billy.Filesystem, root string, walkFn filepath.WalkFunc) error { info, err := fs.Lstat(root) - if err != nil { err = walkFn(root, nil, err) } else { err = walk(fs, root, info, walkFn) } + if err == filepath.SkipDir { return nil } + return err } From 213e20d29a0def89306ec9b5d732b4ba1ceca099 Mon Sep 17 00:00:00 2001 From: Thibault Jamet Date: Thu, 3 Jun 2021 19:51:59 +0200 Subject: [PATCH 3/3] utils: Walk, use os.FileInfo fs.FileInfo has been introduced in go 1.16.5: https://github.com/golang/go/commit/d4da735091986868015369e01c63794af9cc9b84#diff-46ea5b4a04796237678b36cd31d68f74209c90d604123f15dcc4bcb605dbc136 go-billy supports the 2 latest versions The latest version of go is currently 1.16, hence use the previous instance in os instead --- util/walk_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/util/walk_test.go b/util/walk_test.go index ae4c390..c92cb63 100644 --- a/util/walk_test.go +++ b/util/walk_test.go @@ -3,7 +3,7 @@ package util_test import ( "errors" "fmt" - "io/fs" + "os" "path/filepath" "reflect" "testing" @@ -23,19 +23,19 @@ var _ = Suite(&WalkSuite{}) func (s *WalkSuite) TestWalkCanSkipTopDirectory(c *C) { filesystem := memfs.New() - c.Assert(util.Walk(filesystem, "/root/that/does/not/exist", func(path string, info fs.FileInfo, err error) error { return filepath.SkipDir }), IsNil) + c.Assert(util.Walk(filesystem, "/root/that/does/not/exist", func(path string, info os.FileInfo, err error) error { return filepath.SkipDir }), IsNil) } func (s *WalkSuite) TestWalkReturnsAnErrorWhenRootDoesNotExist(c *C) { filesystem := memfs.New() - c.Assert(util.Walk(filesystem, "/root/that/does/not/exist", func(path string, info fs.FileInfo, err error) error { return err }), NotNil) + c.Assert(util.Walk(filesystem, "/root/that/does/not/exist", func(path string, info os.FileInfo, err error) error { return err }), NotNil) } func (s *WalkSuite) TestWalkOnPlainFile(c *C) { filesystem := memfs.New() createFile(c, filesystem, "./README.md") discoveredPaths := []string{} - c.Assert(util.Walk(filesystem, "./README.md", func(path string, info fs.FileInfo, err error) error { + c.Assert(util.Walk(filesystem, "./README.md", func(path string, info os.FileInfo, err error) error { discoveredPaths = append(discoveredPaths, path) return nil }), IsNil) @@ -47,7 +47,7 @@ func (s *WalkSuite) TestWalkOnExistingFolder(c *C) { createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") createFile(c, filesystem, "path/to/some/file") discoveredPaths := []string{} - c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + c.Assert(util.Walk(filesystem, "path", func(path string, info os.FileInfo, err error) error { discoveredPaths = append(discoveredPaths, path) return nil }), IsNil) @@ -66,7 +66,7 @@ func (s *WalkSuite) TestWalkCanSkipFolder(c *C) { createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") createFile(c, filesystem, "path/to/some/file") discoveredPaths := []string{} - c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + c.Assert(util.Walk(filesystem, "path", func(path string, info os.FileInfo, err error) error { discoveredPaths = append(discoveredPaths, path) if path == "path/to/some/subfolder" { return filepath.SkipDir @@ -88,7 +88,7 @@ func (s *WalkSuite) TestWalkStopsOnError(c *C) { createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") createFile(c, filesystem, "path/to/some/file") discoveredPaths := []string{} - c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + c.Assert(util.Walk(filesystem, "path", func(path string, info os.FileInfo, err error) error { discoveredPaths = append(discoveredPaths, path) if path == "path/to/some/subfolder" { return errors.New("uncaught error") @@ -109,7 +109,7 @@ func (s *WalkSuite) TestWalkForwardsStatErrors(c *C) { memFilesystem := memfs.New() filesystem := &fnFs{ Filesystem: memFilesystem, - lstat: func(path string) (fs.FileInfo, error) { + lstat: func(path string) (os.FileInfo, error) { if path == "path/to/some/subfolder" { return nil, errors.New("uncaught error") } @@ -120,7 +120,7 @@ func (s *WalkSuite) TestWalkForwardsStatErrors(c *C) { createFile(c, filesystem, "path/to/some/subfolder/that/contain/file") createFile(c, filesystem, "path/to/some/file") discoveredPaths := []string{} - c.Assert(util.Walk(filesystem, "path", func(path string, info fs.FileInfo, err error) error { + c.Assert(util.Walk(filesystem, "path", func(path string, info os.FileInfo, err error) error { discoveredPaths = append(discoveredPaths, path) if path == "path/to/some/subfolder" { c.Assert(err, NotNil) @@ -147,10 +147,10 @@ func createFile(c *C, filesystem billy.Filesystem, path string) { type fnFs struct { billy.Filesystem - lstat func(path string) (fs.FileInfo, error) + lstat func(path string) (os.FileInfo, error) } -func (f *fnFs) Lstat(path string) (fs.FileInfo, error) { +func (f *fnFs) Lstat(path string) (os.FileInfo, error) { if f.lstat != nil { return f.lstat(path) }