reformatted

viant · Aug 22, 2019 · 52e7a5a · 52e7a5a
1 parent c8e4234
commit 52e7a5a
Show file tree

Hide file tree

Showing 21 changed files with 96 additions and 78 deletions.
diff --git a/README.md b/README.md
@@ -48,7 +48,7 @@ Delete(ctx context.Context, URL string, options ...Option) error
 * **Batch uploader:**
 
 ```go
-type Upload func(ctx context.Context, relativePath string, info os.FileInfo, reader io.Reader) error
+type Upload func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error
 
 Uploader(ctx context.Context, URL string, options ...Option) (Upload, io.Closer, error)
 ```
@@ -265,22 +265,21 @@ func main() {
 
 ## Options
 
-* **[Walker Matcher](option/matcher.go).**
+* **[Matcher](option/matcher.go).**
 
-Filters resources with Copy or Walk operations
-
-* **[List Matcher](option/matcher.go).**
-
-Filter resource with List operation
+Filters resources
 
 * **[Page](option/page.go)**
 
 Page paginates list result by offset and limit.
 
 * **[Modifier](option/modifier.go)**
 
+
 * **[Timeout](option/timeout.go)**
 
+Provider specific timeout.
+
 * **[BasicAuth](option/cred.go)** 
 
 Provides user/password auth.
@@ -301,6 +300,7 @@ Check out [storage manager](#storage-managers) for additional options.
 - [Tar](tar/README.md)
 - [Zip](zip/README.md)
 - [GCP - GS](https://github.com/viant/afsc/gs)
+- [AWS - S3](https://github.com/viant/afsc/s3)
 
 ## Testing service
 

diff --git a/asset/load.go b/asset/load.go
@@ -2,6 +2,7 @@ package asset
 
 import (
 	"context"
+	"fmt"
 	"github.com/viant/afs/storage"
 	"github.com/viant/afs/url"
 	"github.com/viant/afs/walker"
@@ -19,8 +20,9 @@ func Load(manager storage.Manager, URL string) (map[string]*Resource, error) {
 		managerWalker = walker.New(manager)
 	}
 	var result = make(map[string]*Resource)
-	err := managerWalker.Walk(context.Background(), URL, func(ctx context.Context, baseURL string, relativePath string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) {
-		key := path.Join(relativePath, info.Name())
+	err := managerWalker.Walk(context.Background(), URL, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) {
+		fmt.Printf("%v %v %v\n", parent, info.Name(), info.Mode())
+		key := path.Join(parent, info.Name())
 		var data []byte
 		if !info.IsDir() {
 			data, err = ioutil.ReadAll(reader)

diff --git a/base/manager.go b/base/manager.go
@@ -32,14 +32,6 @@ func (m *Manager) List(ctx context.Context, URL string, options ...storage.Optio
 	if err != nil {
 		return nil, err
 	}
-	var matcher option.ListMatcher
-	options, _ = option.Assign(options, &matcher)
-	if matcher == nil {
-		matcher = func(info os.FileInfo) bool {
-			return true
-		}
-	}
-
 	files, err := storager.List(ctx, URLPath, options)
 	if err != nil {
 		return nil, err
@@ -59,9 +51,6 @@ func (m *Manager) List(ctx context.Context, URL string, options ...storage.Optio
 
 	objects[0] = object.New(url.Join(baseURL, URLPath), files[0], nil)
 	for i := 1; i < len(files); i++ {
-		if !matcher(files[i]) {
-			continue
-		}
 		fileURL := url.Join(baseURL, path.Join(URLPath, files[i].Name()))
 		objects[i] = object.New(fileURL, files[i], nil)
 	}
@@ -192,7 +181,7 @@ func (m *Manager) Scheme() string {
 }
 
 //New creates base storager base Manager
-func New(manager storage.Manager, scheme string, provider func(ctx context.Context, baseURL string, options ...storage.Option) (storage.Storager, error), options ...storage.Option) *Manager {
+func New(manager storage.Manager, scheme string, provider func(ctx context.Context, baseURL string, options ...storage.Option) (storage.Storager, error), options []storage.Option) *Manager {
 	return &Manager{
 		Manager:   manager,
 		scheme:    scheme,

diff --git a/copy.go b/copy.go
@@ -27,22 +27,24 @@ func (s *service) copy(ctx context.Context, sourceURL, destURL string, srcOption
 	walker storage.Walker, uploader storage.BatchUploader) error {
 	destURL = s.updateDestURL(sourceURL, destURL)
 	object, err := s.Object(ctx, sourceURL, *srcOptions...)
+
+	destOpts := *destOptions
 	if err == nil && object.IsDir() {
-		err = s.Create(ctx, destURL, object.Mode(), object.IsDir(), *destOptions...)
+		err = s.Create(ctx, destURL, object.Mode(), object.IsDir(), destOpts...)
 	}
 	if err != nil {
 		return err
 	}
 
-	upload, closer, err := uploader.Uploader(ctx, destURL, *destOptions...)
+	upload, closer, err := uploader.Uploader(ctx, destURL, destOpts...)
 	if err != nil {
 		return err
 	}
 	defer func() {
 		_ = closer.Close()
 	}()
-	return walker.Walk(ctx, sourceURL, func(ctx context.Context, baseURL string, relativePath string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) {
-		err = upload(ctx, relativePath, info, reader)
+	return walker.Walk(ctx, sourceURL, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) {
+		err = upload(ctx, parent, info, reader)
 		return err == nil, err
 	}, *srcOptions...)
 
@@ -55,7 +57,7 @@ func (s *service) Copy(ctx context.Context, sourceURL, destURL string, options .
 	destOptions := option.NewDest()
 	var walker storage.Walker
 	var uploader storage.BatchUploader
-	var matcher option.WalkerMatcher
+	var matcher option.Matcher
 	_, _ = option.Assign(options, &sourceOptions, &destOptions, &matcher, &walker, &uploader)
 	if matcher != nil {
 		*sourceOptions = append(*sourceOptions, matcher)

diff --git a/mem/list.go b/mem/list.go
@@ -10,10 +10,15 @@ import (
 //List list directory or returns a file info
 func (s *storager) List(ctx context.Context, location string, options ...storage.Option) ([]os.FileInfo, error) {
 	page := &option.Page{}
-	_, _ = option.Assign(options, &page)
+	var matcher option.Matcher
+	_, _ = option.Assign(options, &page, &matcher)
 	root := s.Root
 	object, err := root.Lookup(location, 0)
-
+	if matcher == nil {
+		matcher = func(parent string, info os.FileInfo) bool {
+			return true
+		}
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -26,6 +31,9 @@ func (s *storager) List(ctx context.Context, location string, options ...storage
 		var result = make([]os.FileInfo, len(objects))
 
 		for i := range objects {
+			if !matcher(location, objects[i]) {
+				continue
+			}
 			page.Increment()
 			if page.ShallSkip() {
 				continue
@@ -37,6 +45,9 @@ func (s *storager) List(ctx context.Context, location string, options ...storage
 		}
 		return result, nil
 	}
+	if !matcher(location, object) {
+		return []os.FileInfo{}, nil
+	}
 	return []os.FileInfo{object}, nil
 }
 

diff --git a/mem/manager.go b/mem/manager.go
@@ -57,7 +57,7 @@ func New(options ...storage.Option) storage.Manager {
 
 func newManager(options ...storage.Option) *manager {
 	result := &manager{}
-	baseMgr := base.New(result, Scheme, result.provider, options...)
+	baseMgr := base.New(result, Scheme, result.provider, options)
 	result.Manager = baseMgr
 	return result
 }
diff --git a/move.go b/move.go
@@ -16,8 +16,7 @@ func (s *service) Move(ctx context.Context, sourceURL, destURL string, options .
 
 	sourceOptions := option.NewSource()
 	destOptions := option.NewDest()
-	var matcher option.WalkerMatcher
-	_, _ = option.Assign(options, &sourceOptions, &destOptions, &matcher)
+	_, _ = option.Assign(options, &sourceOptions, &destOptions)
 	if sourceScheme == destScheme {
 		if manager, err := s.manager(ctx, sourceURL, *sourceOptions...); err == nil {
 			if mover, ok := manager.(storage.Mover); ok {

diff --git a/option/crc.go b/option/crc.go
@@ -3,9 +3,10 @@ package option
 import (
 	"encoding/base64"
 	"fmt"
+	"hash/crc32"
 )
 
-//Crc represents crc value
+//Crc represents crc hash
 type Crc struct {
 	Hash uint32
 }
@@ -26,3 +27,10 @@ func (c *Crc) Decode(encoded string) error {
 	c.Hash = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3])
 	return nil
 }
+
+//NewCrc returns a crc hash for supplied data
+func NewCrc(data []byte) *Crc {
+	crc32Hash := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	_, _ = crc32Hash.Write(data)
+	return &Crc{Hash: crc32Hash.Sum32()}
+}
diff --git a/option/matcher.go b/option/matcher.go
@@ -2,8 +2,5 @@ package option
 
 import "os"
 
-//WalkerMatcher represent on skip assign, if return true skip processing
-type WalkerMatcher func(baseURL, relativePath string, info os.FileInfo) bool
-
-//ListMatcher reprsents a list matcher
-type ListMatcher func(info os.FileInfo) bool
+//Matcher represents a matcher
+type Matcher func(parent string, info os.FileInfo) bool
diff --git a/option/matcher/filepath.go b/option/matcher/filepath.go
@@ -7,9 +7,9 @@ import (
 )
 
 //FilepathMatcher returns filepath based filepath matcher
-func FilepathMatcher(pattern string) func(baseURL, relativePath string, info os.FileInfo) bool {
-	return func(baseURL, relativePath string, info os.FileInfo) bool {
-		name := path.Join(relativePath, info.Name())
+func FilepathMatcher(pattern string) func(baseURL, parent string, info os.FileInfo) bool {
+	return func(baseURL, parent string, info os.FileInfo) bool {
+		name := path.Join(parent, info.Name())
 		hasMatch, _ := filepath.Match(pattern, name)
 		return hasMatch
 	}

diff --git a/option/md5.go b/option/md5.go
@@ -1,6 +1,9 @@
 package option
 
-import "encoding/base64"
+import (
+	"crypto/md5"
+	"encoding/base64"
+)
 
 //Md5 represents md5 value
 type Md5 struct {
@@ -17,3 +20,10 @@ func (m *Md5) Decode(encoded string) (err error) {
 	m.Hash, err = base64.StdEncoding.DecodeString(encoded)
 	return err
 }
+
+//NewMd5 returns a MD5 hash for supplied data
+func NewMd5(data []byte) *Md5 {
+	hash := md5.New()
+	_, _ = hash.Write(data)
+	return &Md5{Hash: hash.Sum(nil)}
+}
diff --git a/storage/options.go b/storage/options.go
@@ -2,3 +2,8 @@ package storage
 
 //Options represents options
 type Options []Option
+
+//NewOptions returns new options
+func NewOptions(options []Option, extraOptions ...Option) []Option {
+	return append(options, extraOptions...)
+}
diff --git a/storage/uploader.go b/storage/uploader.go
@@ -6,7 +6,7 @@ import (
 	"os"
 )
 
-type Upload func(ctx context.Context, relativePath string, info os.FileInfo, reader io.Reader) error
+type Upload func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error
 
 //Uploader represents an uploader
 type Uploader interface {

diff --git a/storage/walker.go b/storage/walker.go
@@ -7,7 +7,7 @@ import (
 )
 
 //OnVisit represents on location visit handler
-type OnVisit func(ctx context.Context, baseURL string, relativePath string, info os.FileInfo, reader io.Reader) (toContinue bool, err error)
+type OnVisit func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error)
 
 //Walker represents abstract storage walker
 type Walker interface {

diff --git a/tar/uploader.go b/tar/uploader.go
@@ -31,15 +31,15 @@ func (u *uploader) Uploader(ctx context.Context, URL string, options ...storage.
 		buffer = new(bytes.Buffer)
 	}
 	writer := newWriter(ctx, buffer, URL, uploader)
-	return func(ctx context.Context, relativePath string, info os.FileInfo, reader io.Reader) error {
+	return func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error {
 		link := ""
 		var options []storage.Option
 		if fileInfo, ok := info.(*file.Info); ok {
 			link = fileInfo.Linkname
 			options = make([]storage.Option, 0)
 			options = append(options, fileInfo.Link)
 		}
-		filename := path.Join(relativePath, info.Name())
+		filename := path.Join(parent, info.Name())
 		info = file.NewInfo(filename, info.Size(), info.Mode(), info.ModTime(), info.IsDir(), options...)
 		header, err := tar.FileInfoHeader(info, link)
 		if err != nil {

diff --git a/tar/walker_test.go b/tar/walker_test.go
@@ -43,9 +43,9 @@ func TestWalker_Walk(t *testing.T) {
 		walker := NewWalker(afs.New())
 		ctx := context.Background()
 		actuals := make(map[string]*asset.Resource)
-		err := walker.Walk(ctx, useCase.location, func(ctx context.Context, baseURL string, relativePath string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) {
+		err := walker.Walk(ctx, useCase.location, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) {
 
-			resourceLocation := path.Join(relativePath, info.Name())
+			resourceLocation := path.Join(parent, info.Name())
 			linkName := ""
 			if rawInfo, ok := info.(*file.Info); ok {
 				linkName = rawInfo.Linkname
@@ -57,7 +57,7 @@ func TestWalker_Walk(t *testing.T) {
 					return false, err
 				}
 			}
-			actuals[resourceLocation] = asset.New(relativePath, info.Mode(), info.IsDir(), linkName, data)
+			actuals[resourceLocation] = asset.New(parent, info.Mode(), info.IsDir(), linkName, data)
 			return true, nil
 		})
 		assert.Nil(t, err, useCase.description)

diff --git a/uploader.go b/uploader.go
@@ -2,6 +2,7 @@ package afs
 
 import (
 	"context"
+	"fmt"
 	"github.com/viant/afs/file"
 	"github.com/viant/afs/storage"
 	"github.com/viant/afs/url"
@@ -21,11 +22,17 @@ func (s *service) Uploader(ctx context.Context, URL string, options ...storage.O
 	if ok {
 		return batchUploader.Uploader(ctx, URL, options...)
 	}
-	handler := func(ctx context.Context, relativePath string, info os.FileInfo, reader io.Reader) error {
-		location := path.Join(relativePath, info.Name())
+	handler := func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error {
+		location := path.Join(parent, info.Name())
 		URL := url.Join(URL, location)
+		if info.Mode()&os.ModeSymlink > 0 {
+			if rawInfo, ok := info.(*file.Info); ok && rawInfo.Linkname != "" {
+				fmt.Printf("is link %v\n", rawInfo)
+				options = append(options, rawInfo.Link)
+			}
+		}
 		if info.IsDir() {
-			return manager.Create(ctx, URL, info.Mode(), info.IsDir(), options)
+			return manager.Create(ctx, URL, info.Mode(), info.IsDir(), options...)
 		}
 		return manager.Upload(ctx, URL, info.Mode(), reader, options...)
 	}