reformatted

README.md

@@ -11,7 +11,9 @@ Please refer to [`CHANGELOG.md`](CHANGELOG.md) if you encounter breaking changes
 - [Motivation](#motivation)
 - [Introduction](#introduction)
 - [Usage](#usage)
+- [Matchers](#matchers)
 - [Options](#options)
+- [Storage Implementations](#storage-implementations)
 - [Testing mode](#testing-mode)
 - [Storage Manager](#storage-managers)
 - [GoCover](#gocover)
@@ -262,19 +264,118 @@ func main() {
 	}
 }
 ```
+## Matchers
 
-## Options
+To filter source content you can use [Matcher](option/matcher.go) option. 
+The following have been implemented.
 
-* **[Matcher](option/matcher.go).**
 
-Filters resources
+**[Basic Matcher](matcher/basic.go)**
 
-* **[Page](option/page.go)**
+```go
+func main() {
+    matcher, err := NewBasic("/data", ".avro", nil)
+    service := afs.New()
+    ctx := context.Background()
+    err := service.Copy(ctx, "/tmp/data", "s3://mybucket/data/", matcher.Match)
+    if err != nil {
+        log.Fatal(err)
+    }
+}
+```
 
-Page paginates list result by offset and limit.
+**[Filepath matcher](matcher/filepath.go)**
 
-* **[Modifier](option/modifier.go)**
+OS style filepath match, with the following terms:
+- '*'         matches any sequence of non-Separator characters
+- '?'         matches any single non-Separator character
+- '[' [ '^' ] { character-range } ']'
 
+```go
+
+func main() {
+    matcher := matcher.Filepath("*.avro")
+    service := afs.New()
+    ctx := context.Background()
+    err := service.Copy(ctx, "/tmp/data", "gs://mybucket/data/", matcher)
+    if err != nil {
+        log.Fatal(err)
+    }
+}	
+
+
+```
+
+**[Ignore Matcher](matcher/ignore.go)**
+
+Ignore matcher represents matcher that matches file that are not in the ignore rules.
+The syntax of ignore borrows heavily from that of .gitignore; see https://git-scm.com/docs/gitignore or man gitignore for a full reference.
+
+
+```go
+func mian(){
+	ignoreMatcher, err := matcher.NewIgnore([]string{"*.txt", ".ignore"})
+  	//or matcher.NewIgnore(option.NewLocation(".cloudignore"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	service := afs.New()
+	ctx := context.Background()
+	objects, err := service.List(ctx, "/tmp/folder", ignoreMatcher.Match)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, object := range objects {
+		fmt.Printf("%v %v\n", object.Name(), object.URL())
+		if object.IsDir() {
+			continue
+		}
+	}
+}	
+```
+
+## Content modified
+
+To modify resource content on the fly you can use [Modified](option/modifier.go) option.
+
+```go
+
+func main() {
+
+	modifier: func(info os.FileInfo, reader io.ReadCloser) (closer io.ReadCloser, e error) {
+        if strings.HasSuffix(info.Name() ,".info") {
+            data, err := ioutil.ReadAll(reader)
+            if err != nil {
+                return nil, err
+            }
+            _ = reader.Close()
+            expanded := strings.Replace(string(data), "$os.User", os.Getenv("USER"), 1)
+            reader = ioutil.NopCloser(strings.NewReader(expanded))
+        }
+        return reader, nil
+    }
+	service := afs.New()
+	reader ,err := service.DownloadWithURL(ctx, "s3://mybucket/meta.info", modifier)
+	if err != nil {
+	    log.Fatal(err)	
+	}
+	defer reader.Close()
+    content, err := ioutil.ReadAll(reader)
+    if err != nil {
+        log.Fatal(err)	
+    }
+	fmt.Printf("content: %s\n", content)
+
+}
+```
+
+
+## Options
+
+
+* **[Page](option/page.go)**
+
+To control number and position of listed resources you can yse page option.
 
 * **[Timeout](option/timeout.go)**
 
@@ -288,6 +389,26 @@ Provides user/password auth.
 
 Groups options by source or destination options. This options work with Copy or Move operations.
 
+```go
+
+func main() {
+    service := afs.New()
+
+    secretPath :=  path.Join(os.Getenv("HOME"), ".secret","gcp.json")
+    jwtConfig, err := gs.NewJwtConfig(option.NewLocation(secretPath))
+    if err != nil {
+    	log.Fatal(err)
+    }
+    sourceOptions := option.NewSource(jwtConfig)
+    authConfig, err := s3.NewAuthConfig(option.NewLocation("aws.json"))
+    if err != nil {
+        log.Fatal(err)
+    }
+    destOptions := option.NewDest(authConfig)
+	err = service.Copy(ctx, "gs://mybucket/data", "s3://mybucket/data",  sourceOptions, destOptions)
+}
+
+```
 
 Check out [storage manager](#storage-managers) for additional options. 
 
@@ -310,16 +431,19 @@ In addition you can use error options to test exception handling.
 
 - **DownloadError**
 ```go
+func mian() {
 	service := afs.NewFaker()
 	ctx := context.Background()
 	err := service.Upload(ctx, "gs://myBucket/folder/asset.txt", 0, strings.NewReader("some data"), option.NewUploadError(io.EOF))
 	if err != nil {
 		log.Fatalf("expect upload error: %v", err)
 	}
+}
 ```
 
 - **ReaderError**
 ```go
+func mian() {
     service := afs.NewFaker()
 	ctx := context.Background()
 	err := service.Upload(ctx, "gs://myBucket/folder/asset.txt", 0, strings.NewReader("some data"), option.NewDownloadError(io.EOF))
@@ -330,16 +454,19 @@ In addition you can use error options to test exception handling.
 	if err != nil {
 		log.Fatalf("expect download error: %v", err)
 	}
+}
 ```
 
 - **UploadError**
 ```go
+func mian() {
     service := afs.NewFaker()
     ctx := context.Background()
     err := service.Upload(ctx, "gs://myBucket/folder/asset.txt", 0, strings.NewReader("some data"), option.NewUploadError(io.EOF))
     if err != nil {
         log.Fatalf("expect upload error: %v", err)
     }
+}
 ```
 
 
@@ -396,13 +523,11 @@ func Test_XXX(t *testing.T) {
 
 ```
 
-
 ## GoCover
 
 [![GoCover](https://gocover.io/github.com/viant/afs)](https://gocover.io/github.com/viant/afs)
 
-	
-	
+
 <a name="License"></a>
 ## License
 

base/manager.go

@@ -91,6 +91,9 @@ func (m *Manager) Download(ctx context.Context, object storage.Object, options .
 //DownloadWithURL downloads content
 func (m *Manager) DownloadWithURL(ctx context.Context, URL string, options ...storage.Option) (io.ReadCloser, error) {
 	baseURL, URLPath := url.Base(URL, m.scheme)
+
+	var modifier option.Modifier
+	option.Assign(options, &modifier)
 	storager, err := m.Storager(ctx, baseURL, options...)
 	if err != nil {
 		return nil, err

copy.go

@@ -58,7 +58,7 @@ func (s *service) Copy(ctx context.Context, sourceURL, destURL string, options .
 	var walker storage.Walker
 	var uploader storage.BatchUploader
 	var matcher option.Matcher
-	_, _ = option.Assign(options, &sourceOptions, &destOptions, &matcher, &walker, &uploader)
+	option.Assign(options, &sourceOptions, &destOptions, &matcher, &walker, &uploader)
 	if matcher != nil {
 		*sourceOptions = append(*sourceOptions, matcher)
 	}

example_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/viant/afs"
 	"github.com/viant/afs/asset"
 	"github.com/viant/afs/file"
+	"github.com/viant/afs/matcher"
 	"github.com/viant/afs/option"
 	"github.com/viant/afs/scp"
 	"io"
@@ -166,3 +167,23 @@ func Example_ReaderError() {
 		log.Fatalf("expect download error: %v", err)
 	}
 }
+
+func Example_IgnoreMatcher() {
+	ignoreMatcher, err := matcher.NewIgnore([]string{"*.txt", ".ignore"})
+	if err != nil {
+		log.Fatal(err)
+	}
+	service := afs.New()
+	ctx := context.Background()
+	objects, err := service.List(ctx, "/tmp/folder", ignoreMatcher)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, object := range objects {
+		fmt.Printf("%v %v\n", object.Name(), object.URL())
+		if object.IsDir() {
+			continue
+		}
+	}
+
+}

file/info.go

@@ -51,7 +51,7 @@ func (i *Info) Sys() interface{} {
 //NewInfo returns a ew file Info
 func NewInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, isDir bool, options ...storage.Option) os.FileInfo {
 	link := &object.Link{}
-	_, _ = option.Assign(options, &link)
+	option.Assign(options, &link)
 	if link.Source == nil && len(options) == 1 {
 		link.Source = options[0]
 	}

file/list.go

@@ -20,7 +20,7 @@ func List(ctx context.Context, URL string, options ...storage.Option) ([]storage
 	}
 	var matcher option.Matcher
 	page := option.Page{}
-	_, _ = option.Assign(options, &matcher, &page)
+	option.Assign(options, &matcher, &page)
 	if matcher == nil {
 		matcher = func(parent string, info os.FileInfo) bool {
 			return true

file/list_test.go

@@ -4,27 +4,51 @@ import (
 	"context"
 	"fmt"
 	"github.com/stretchr/testify/assert"
+	"github.com/viant/afs/matcher"
+	"github.com/viant/afs/option"
+	"github.com/viant/afs/storage"
 	"io/ioutil"
 	"os"
 	"path"
 	"testing"
+	"time"
 )
 
 func TestManager_List(t *testing.T) {
 
 	baseDir := os.TempDir()
 	fileManager := New()
 
+	ignoreMatcher, _ := matcher.NewIgnore([]string{"*.txt", ".ignore"})
+
 	var useCases = []struct {
 		description  string
 		location     string
 		baseLocation string
 		assets       map[string]string
+		matcher      option.Matcher
 	}{
 		{
 			description:  "location download",
-			location:     path.Join(baseDir, "file_list/"),
-			baseLocation: path.Join(baseDir, "file_list"),
+			location:     path.Join(baseDir, "file_list_001/"),
+			baseLocation: path.Join(baseDir, "file_list_001"),
+			assets: map[string]string{
+				"file1.txt": "abc",
+				"file2.txt": "abc",
+			},
+		},
+		{
+			description:  "location download with filter",
+			location:     path.Join(baseDir, "file_list_002/"),
+			baseLocation: path.Join(baseDir, "file_list_002/"),
+			assets: map[string]string{
+				"file1.txt": "abc1",
+				"file2.txt": "abc2",
+				"asset.csv": "abc3",
+				"asset.tsv": "abc4",
+				".ignore":   "abc4",
+			},
+			matcher: ignoreMatcher.Match,
 		},
 	}
 
@@ -36,12 +60,16 @@ func TestManager_List(t *testing.T) {
 		_ = fileManager.Create(ctx, useCase.baseLocation, 0744, true)
 
 		for name, content := range useCase.assets {
-			filename := path.Join(baseDir, name)
+			filename := path.Join(useCase.baseLocation, name)
 			_ = ioutil.WriteFile(filename, []byte(content), 0744)
 		}
 
 		actuals := map[string]string{}
-		objects, err := manager.List(ctx, useCase.location)
+		var options = make([]storage.Option, 0)
+		if useCase.matcher != nil {
+			options = append(options, useCase.matcher)
+		}
+		objects, err := manager.List(ctx, useCase.location, options...)
 		assert.Nil(t, err, useCase.description)
 		for _, object := range objects {
 			content := ""
@@ -56,6 +84,12 @@ func TestManager_List(t *testing.T) {
 		}
 
 		for name := range useCase.assets {
+			if useCase.matcher != nil {
+				info := NewInfo(name, 0, 0644, time.Now(), false)
+				if !useCase.matcher("", info) {
+					continue
+				}
+			}
 			_, ok := actuals[name]
 			assert.True(t, ok, fmt.Sprintf(useCase.description+"  %v, %v", name, actuals))
 		}

file/upload.go

@@ -32,7 +32,7 @@ func Upload(ctx context.Context, URL string, mode os.FileMode, reader io.Reader,
 		return errors.Wrap(err, "unable to create parent for "+filePath)
 	}
 	link := &object.Link{}
-	_, _ = option.Assign(options, &link)
+	option.Assign(options, &link)
 	if link.Linkname != "" {
 		return os.Symlink(filePath, link.Linkname)
 	}

http/client.go

@@ -20,7 +20,7 @@ func (s *manager) getClient(baseURL string, options ...storage.Option) (*http.Cl
 		options = append(s.options, options...)
 	}
 	var clientProvider ClientProvider
-	_, _ = option.Assign(options, &clientProvider)
+	option.Assign(options, &clientProvider)
 	if clientProvider == nil {
 		if s.client == nil {
 			s.client = http.DefaultClient

http/create.go

@@ -13,7 +13,7 @@ import (
 //Create send post request
 func (s *manager) Create(ctx context.Context, URL string, mode os.FileMode, isDir bool, options ...storage.Option) error {
 	var reader io.Reader
-	_, _ = option.Assign(options, &reader)
+	option.Assign(options, &reader)
 	request, err := http.NewRequest(http.MethodPost, URL, reader)
 	if err != nil {
 		return err