diff --git a/Makefile b/Makefile index 32781e349..f20ee9656 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ SHELL := /bin/bash +PACKAGES = credential endpoint -.PHONY: all check format vet lint build test generate tidy integration_test +.PHONY: all check format vet lint build test generate tidy integration_test $(PACKAGES) help: @echo "Please use \`make \` where is one of" @@ -33,6 +34,11 @@ build: tidy generate format check @go build -tags tools ./... @echo "ok" +build-all: $(PACKAGES) + +$(PACKAGES): + pushd $@ && make build && popd + test: @echo "run test" @go test -race -coverprofile=coverage.txt -covermode=atomic -v ./... diff --git a/credential/CHANGELOG.md b/credential/CHANGELOG.md new file mode 100644 index 000000000..9c713a11e --- /dev/null +++ b/credential/CHANGELOG.md @@ -0,0 +1,13 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [Unreleased] + +## v0.1.0 - 2021-09-27 + +- Migrate credential from go-storage to separate repo. +- Add basic support diff --git a/credential/Makefile b/credential/Makefile new file mode 100644 index 000000000..641b5145e --- /dev/null +++ b/credential/Makefile @@ -0,0 +1,36 @@ +SHELL := /bin/bash + +.PHONY: all check format vet lint build test generate tidy integration_test + +help: + @echo "Please use \`make \` where is one of" + @echo " check to do static check" + @echo " build to create bin directory and build" + @echo " test to run test" + +check: vet + +format: + @echo "go fmt" + @go fmt ./... + @echo "ok" + +vet: + @echo "go vet" + @go vet ./... + @echo "ok" + +build: tidy check + @echo "build go-credential" + @go build ./... + @echo "ok" + +test: + @echo "run test" + @go test -race -coverprofile=coverage.txt -covermode=atomic -v ./... + @go tool cover -html="coverage.txt" -o "coverage.html" + @echo "ok" + +tidy: + @go mod tidy + @go mod verify diff --git a/credential/README.md b/credential/README.md new file mode 100644 index 000000000..704ac4fa7 --- /dev/null +++ b/credential/README.md @@ -0,0 +1,49 @@ +# credential + +Both human and machine-readable credential format. + +## Format + +``` +:+ +``` + +For example: + +- hmac: `hmac:access_key:secret_key` +- apikey: `apikey:apikey` +- file: `file:/path/to/config/file` +- basic: `basic:user:password` + +## Quick Start + +```go +cred, err := credential.Parse("hmac:access_key:secret_key) +if err != nil { + log.Fatal("parse: ", err) +} + +switch cred.Protocol() { +case ProtocolHmac: + ak, sk := cred.Hmac() + log.Println("access_key: ", ak) + log.Println("secret_key: ", sk) +case ProtocolAPIKey: + apikey := cred.APIKey() + log.Println("apikey: ", apikey) +case ProtocolFile: + path := cred.File() + log.Println("path: ", path) +case ProtocolEnv: + log.Println("use env value") +case ProtocolBase64: + content := cred.Base64() + log.Println("base64: ", content) +case ProtocolBasic: + user, password := cred.Basic() + log.Println("user: ", user) + log.Println("password: ", password) +default: + panic("unsupported protocol") +} +``` diff --git a/pkg/credential/credential.go b/credential/credential.go similarity index 61% rename from pkg/credential/credential.go rename to credential/credential.go index bd5f418ba..393fd8d66 100644 --- a/pkg/credential/credential.go +++ b/credential/credential.go @@ -30,33 +30,37 @@ const ( // Storage service like gcs will take token files as input, we provide base64 protocol so that user // can pass token binary data directly. ProtocolBase64 = "base64" + // ProtocolBasic will hold user and password credential. + // + // value = [user, password] + ProtocolBasic = "basic" ) -// Provider will provide credential protocol and values. -type Provider struct { +// Credential will provide credential protocol and values. +type Credential struct { protocol string args []string } // Protocol provides current credential's protocol. -func (p Provider) Protocol() string { +func (p Credential) Protocol() string { return p.protocol } // Value provides current credential's value in string array. -func (p Provider) Value() []string { +func (p Credential) Value() []string { return p.args } // Value provides current credential's value in string array. -func (p Provider) String() string { +func (p Credential) String() string { if len(p.args) == 0 { return p.protocol } return p.protocol + ":" + strings.Join(p.args, ":") } -func (p Provider) Hmac() (accessKey, secretKey string) { +func (p Credential) Hmac() (accessKey, secretKey string) { if p.protocol != ProtocolHmac { panic(Error{ Op: "hmac", @@ -68,7 +72,7 @@ func (p Provider) Hmac() (accessKey, secretKey string) { return p.args[0], p.args[1] } -func (p Provider) APIKey() (apiKey string) { +func (p Credential) APIKey() (apiKey string) { if p.protocol != ProtocolAPIKey { panic(Error{ Op: "api_key", @@ -80,7 +84,7 @@ func (p Provider) APIKey() (apiKey string) { return p.args[0] } -func (p Provider) File() (path string) { +func (p Credential) File() (path string) { if p.protocol != ProtocolFile { panic(Error{ Op: "file", @@ -92,7 +96,7 @@ func (p Provider) File() (path string) { return p.args[0] } -func (p Provider) Base64() (value string) { +func (p Credential) Base64() (value string) { if p.protocol != ProtocolBase64 { panic(Error{ Op: "base64", @@ -104,8 +108,20 @@ func (p Provider) Base64() (value string) { return p.args[0] } -// Parse will parse config string to create a credential Provider. -func Parse(cfg string) (Provider, error) { +func (p Credential) Basic() (user, password string) { + if p.protocol != ProtocolBasic { + panic(Error{ + Op: "basic", + Err: ErrInvalidValue, + Protocol: p.protocol, + Values: p.args, + }) + } + return p.args[0], p.args[1] +} + +// Parse will parse config string to create a credential Credential. +func Parse(cfg string) (Credential, error) { s := strings.Split(cfg, ":") switch s[0] { @@ -119,32 +135,39 @@ func Parse(cfg string) (Provider, error) { return NewEnv(), nil case ProtocolBase64: return NewBase64(s[1]), nil + case ProtocolBasic: + return NewBasic(s[1], s[2]), nil default: - return Provider{}, &Error{"parse", ErrUnsupportedProtocol, s[0], nil} + return Credential{}, &Error{"parse", ErrUnsupportedProtocol, s[0], nil} } } // NewHmac create a hmac provider. -func NewHmac(accessKey, secretKey string) Provider { - return Provider{ProtocolHmac, []string{accessKey, secretKey}} +func NewHmac(accessKey, secretKey string) Credential { + return Credential{ProtocolHmac, []string{accessKey, secretKey}} } // NewAPIKey create a api key provider. -func NewAPIKey(apiKey string) Provider { - return Provider{ProtocolAPIKey, []string{apiKey}} +func NewAPIKey(apiKey string) Credential { + return Credential{ProtocolAPIKey, []string{apiKey}} } // NewFile create a file provider. -func NewFile(filePath string) Provider { - return Provider{ProtocolFile, []string{filePath}} +func NewFile(filePath string) Credential { + return Credential{ProtocolFile, []string{filePath}} } // NewEnv create a env provider. -func NewEnv() Provider { - return Provider{ProtocolEnv, nil} +func NewEnv() Credential { + return Credential{ProtocolEnv, nil} } // NewBase64 create a base64 provider. -func NewBase64(value string) Provider { - return Provider{ProtocolBase64, []string{value}} +func NewBase64(value string) Credential { + return Credential{ProtocolBase64, []string{value}} +} + +// NewBasic create a basic provider. +func NewBasic(user, password string) Credential { + return Credential{ProtocolBasic, []string{user, password}} } diff --git a/credential/credential_test.go b/credential/credential_test.go new file mode 100644 index 000000000..71c20e45a --- /dev/null +++ b/credential/credential_test.go @@ -0,0 +1,115 @@ +package credential + +import ( + "errors" + "log" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func TestProvider(t *testing.T) { + protocol := uuid.New().String() + args := []string{uuid.New().String(), uuid.New().String()} + + p := Credential{protocol: protocol, args: args} + + assert.Equal(t, protocol, p.Protocol()) + assert.EqualValues(t, args, p.Value()) +} + +func TestParse(t *testing.T) { + cases := []struct { + name string + cfg string + value Credential + err error + }{ + { + "hmac", + "hmac:ak:sk", + Credential{protocol: ProtocolHmac, args: []string{"ak", "sk"}}, + nil, + }, + { + "api key", + "apikey:key", + Credential{protocol: ProtocolAPIKey, args: []string{"key"}}, + nil, + }, + { + "file", + "file:/path/to/file", + Credential{protocol: ProtocolFile, args: []string{"/path/to/file"}}, + nil, + }, + { + "env", + "env", + Credential{protocol: ProtocolEnv}, + nil, + }, + { + "base64", + "base64:aGVsbG8sd29ybGQhCg==", + Credential{protocol: ProtocolBase64, args: []string{"aGVsbG8sd29ybGQhCg=="}}, + nil, + }, + { + "basic", + "basic:user:password", + Credential{protocol: ProtocolBasic, args: []string{"user", "password"}}, + nil, + }, + { + "not supported protocol", + "notsupported:ak:sk", + Credential{}, + ErrUnsupportedProtocol, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + p, err := Parse(tt.cfg) + if tt.err == nil { + assert.Nil(t, err) + } else { + assert.True(t, errors.Is(err, tt.err)) + } + assert.EqualValues(t, tt.value, p) + }) + } +} + +func ExampleParse() { + cred, err := Parse("hmac:access_key:secret_key") + if err != nil { + log.Fatal("parse: ", err) + } + + switch cred.Protocol() { + case ProtocolHmac: + ak, sk := cred.Hmac() + log.Println("access_key: ", ak) + log.Println("secret_key: ", sk) + case ProtocolAPIKey: + apikey := cred.APIKey() + log.Println("apikey: ", apikey) + case ProtocolFile: + path := cred.File() + log.Println("path: ", path) + case ProtocolEnv: + log.Println("use env value") + case ProtocolBase64: + content := cred.Base64() + log.Println("base64: ", content) + case ProtocolBasic: + user, password := cred.Basic() + log.Println("user: ", user) + log.Println("password: ", password) + default: + panic("unsupported protocol") + } +} diff --git a/credential/docs/rfcs/3-add-protocol-basic.md b/credential/docs/rfcs/3-add-protocol-basic.md new file mode 100644 index 000000000..d8cf3935f --- /dev/null +++ b/credential/docs/rfcs/3-add-protocol-basic.md @@ -0,0 +1,33 @@ +- Author: npofsi +- Start Date: 2021-08-02 +- RFC PR: https://github.com/beyondstorage/go-credential/pull/3 +- Tracking Issue: [beyondstorage/go-credential#1](https://github.com/beyondstorage/go-credential/issues/1) + +# RFC-3: Add protocol basic + + +## Background + +Some services don't support other credentials, but user or password. + + +## Proposal + +Add protocol `basic`. Like `hmac`, `basic` have two parameters, corresponding to user and password of an account. + +For example, go-service-ftp need a account to sign in, like + +`ftp://xxx?credential=basic:user:password` + +## Rationale + +- Account is the only certification to some platform. +- Account is a basic method to identify quests. + +## Compatibility + +Will just add a choose to use protocol `basic`. + +## Implementation + +Just need to parse `basic` like `hmac`. \ No newline at end of file diff --git a/pkg/credential/error.go b/credential/error.go similarity index 100% rename from pkg/credential/error.go rename to credential/error.go diff --git a/credential/go.mod b/credential/go.mod new file mode 100644 index 000000000..3d718bdf5 --- /dev/null +++ b/credential/go.mod @@ -0,0 +1,8 @@ +module go.beyondstorage.io/credential + +go 1.15 + +require ( + github.com/google/uuid v1.3.0 + github.com/stretchr/testify v1.7.0 +) diff --git a/credential/go.sum b/credential/go.sum new file mode 100644 index 000000000..facefbb7b --- /dev/null +++ b/credential/go.sum @@ -0,0 +1,13 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/endpoint/CHANGELOG.md b/endpoint/CHANGELOG.md new file mode 100644 index 000000000..f6de5a7a1 --- /dev/null +++ b/endpoint/CHANGELOG.md @@ -0,0 +1,33 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [Unreleased] + +## [v1.1.0] - 2021-07-28 + +### Added + +- Implementing RFC-8: Add TCP Protocol (#11) +- RFC-8: Add TCP protocol (#8) + +## [v1.0.1] - 2021-06-15 + +### Fixed + +- Fix protocol file that contains ":" (#4) +- Fix HTTP url generated incorrectly (#5) + +## v1.0.0 - 2021-06-09 + +### Added + +- Add Parse support +- Add http/https/file protocol support + +[Unreleased]: https://github.com/beyondstorage/go-endpoint/compare/v1.1.0...HEAD +[v1.1.0]: https://github.com/beyondstorage/go-endpoint/compare/v1.0.1...v1.1.0 +[v1.0.1]: https://github.com/beyondstorage/go-endpoint/compare/v1.0.0...v1.0.1 diff --git a/endpoint/Makefile b/endpoint/Makefile new file mode 100644 index 000000000..8ab611b70 --- /dev/null +++ b/endpoint/Makefile @@ -0,0 +1,36 @@ +SHELL := /bin/bash + +.PHONY: all check format vet lint build test generate tidy integration_test + +help: + @echo "Please use \`make \` where is one of" + @echo " check to do static check" + @echo " build to create bin directory and build" + @echo " test to run test" + +check: vet + +format: + @echo "go fmt" + @go fmt ./... + @echo "ok" + +vet: + @echo "go vet" + @go vet ./... + @echo "ok" + +build: tidy check + @echo "build storage" + @go build -tags tools ./... + @echo "ok" + +test: + @echo "run test" + @go test -race -coverprofile=coverage.txt -covermode=atomic -v ./... + @go tool cover -html="coverage.txt" -o "coverage.html" + @echo "ok" + +tidy: + @go mod tidy + @go mod verify diff --git a/endpoint/README.md b/endpoint/README.md new file mode 100644 index 000000000..81cd4e452 --- /dev/null +++ b/endpoint/README.md @@ -0,0 +1,42 @@ +# endpoint + +Both human and machine readable endpoint format. + +## Format + +``` +:+ +``` + +For example: + +- File: `file:/var/cache/data` +- HTTP: `http:example.com:80` +- HTTPS: `https:example.com:443` + +## Quick Start + +```go +ep, err := endpoint.Parse("https:example.com") +if err != nil { + log.Fatal("parse: ", err) +} + +switch ep.Protocol() { +case ProtocolHTTP: + url, host, port := ep.HTTP() + log.Println("url: ", url) + log.Println("host: ", host) + log.Println("port: ", port) +case ProtocolHTTPS: + url, host, port := ep.HTTPS() + log.Println("url: ", url) + log.Println("host: ", host) + log.Println("port: ", port) +case ProtocolFile: + path := ep.File() + log.Println("path: ", path) +default: + panic("unsupported protocol") +} +``` diff --git a/endpoint/doc.go b/endpoint/doc.go new file mode 100644 index 000000000..6b1f4e8f0 --- /dev/null +++ b/endpoint/doc.go @@ -0,0 +1,4 @@ +/* +Package endpoint intends to provide a unified storage layer for Golang. +*/ +package endpoint diff --git a/endpoint/docs/rfcs/8-add-tcp-protocol.md b/endpoint/docs/rfcs/8-add-tcp-protocol.md new file mode 100644 index 000000000..20e0b0007 --- /dev/null +++ b/endpoint/docs/rfcs/8-add-tcp-protocol.md @@ -0,0 +1,36 @@ +- Author: bokket +- Start Date: 2021-07-17 +- RFC PR: [beyondstorage/go-endpoint#8](https://github.com/beyondstorage/go-endpoint/pull/8) +- Tracking Issue: [beyondstorage/go-endpoint/issues/9](https://github.com/beyondstorage/go-endpoint/issues/9) + +# RFC-8: Add TCP protocol + +Releated issue: [beyondstorage/go-endpoint/issues/7](https://github.com/beyondstorage/go-endpoint/issues/7) + +## Background + +hdfs usually use the `New(address string)` method to access a namenode node, the user will be the user running the code. If the address is an empty string, it will try to get the NameNode address from the Hadoop configuration file. + +## Proposal + +I suggest adding a tcp protocol to allow the user to specify the address. + +It likes `tcp::` + +- The `type` of `tcp` should be `String` and is a `const` +- The `value` of `endpoint` should be parsed into `ProtocolTCP` and `args include :` + +## Rationale + +Now we don't have a pair operation on the `hdfs address` or tcp-like operation + +## Compatibility + +No compatibility issues at this time. + +## Implementation + +- Add protocol `tcp` +- Implement protocol tcp formatted (`func (p Endpoint) TCP() (addr, host string, port int)`) +- Implement protocol tcp parser (`func Parse(cfg string) (p Endpoint, err error)`) +- Implement protocol tcp object (`func NewTCP(host string,port int) Endpoint `) diff --git a/endpoint/endpoint.go b/endpoint/endpoint.go new file mode 100644 index 000000000..8b0c54264 --- /dev/null +++ b/endpoint/endpoint.go @@ -0,0 +1,187 @@ +package endpoint + +import ( + "fmt" + "strconv" + "strings" +) + +const ( + // ProtocolHTTP is the http endpoint protocol. + ProtocolHTTP = "http" + // ProtocolHTTPS is the https endpoint protocol. + ProtocolHTTPS = "https" + // ProtocolFile is the file endpoint protocol + ProtocolFile = "file" + // ProtocolTCP is the tcp endpoint protocol + ProtocolTCP = "tcp" +) + +// Parse will parse config string to create a endpoint Endpoint. +func Parse(cfg string) (p Endpoint, err error) { + s := strings.Split(cfg, ":") + + //delete headmost '//' + if len(s) > 1 && strings.HasPrefix(s[1], "//") { + s[1] = s[1][2:] + } + switch s[0] { + case ProtocolHTTP: + host, port, err := parseHostPort(s[1:]) + if err != nil || strings.HasPrefix(host, "/") { + return Endpoint{}, &Error{"parse", ErrInvalidValue, s[0], s[1:]} + } + if port == 0 { + port = 80 + } + return NewHTTP(host, port), nil + case ProtocolHTTPS: + host, port, err := parseHostPort(s[1:]) + if err != nil || strings.HasPrefix(host, "/") { + return Endpoint{}, &Error{"parse", ErrInvalidValue, s[0], s[1:]} + } + if port == 0 { + port = 443 + } + return NewHTTPS(host, port), nil + case ProtocolFile: + // Handle file paths that contains ":" (often happens on windows platform) + // + // See issue: https://github.com/beyondstorage/go-endpoint/issues/3 + path := strings.Join(s[1:], ":") + return NewFile(path), nil + case ProtocolTCP: + //See issue: https://github.com/beyondstorage/go-endpoint/issues/7 + host, port, err := parseHostPort(s[1:]) + if err != nil || strings.HasPrefix(host, "/") { + return Endpoint{}, &Error{"parse", ErrInvalidValue, s[0], s[1:]} + } + return NewTCP(host, port), nil + default: + return Endpoint{}, &Error{"parse", ErrUnsupportedProtocol, s[0], nil} + } +} + +type hostPort struct { + host string + port int +} + +func (hp hostPort) String() string { + return fmt.Sprintf("%s:%d", hp.host, hp.port) +} + +func parseHostPort(s []string) (host string, port int, err error) { + if len(s) == 1 { + return s[0], 0, nil + } + v, err := strconv.ParseInt(s[1], 10, 64) + if err != nil { + return "", 0, err + } + return s[0], int(v), nil +} + +type Endpoint struct { + protocol string + args interface{} +} + +func NewHTTP(host string, port int) Endpoint { + return Endpoint{ + protocol: ProtocolHTTP, + args: hostPort{host, port}, + } +} + +func NewHTTPS(host string, port int) Endpoint { + return Endpoint{ + protocol: ProtocolHTTPS, + args: hostPort{host, port}, + } +} + +func NewFile(path string) Endpoint { + return Endpoint{ + protocol: ProtocolFile, + args: path, + } +} + +func NewTCP(host string, port int) Endpoint { + return Endpoint{ + protocol: ProtocolTCP, + args: hostPort{host, port}, + } +} + +func (p Endpoint) Protocol() string { + return p.protocol +} + +func (p Endpoint) String() string { + if p.args == nil { + return p.protocol + } + return fmt.Sprintf("%s:%s", p.protocol, p.args) +} + +func (p Endpoint) HTTP() (url, host string, port int) { + if p.protocol != ProtocolHTTP { + panic(Error{ + Op: "http", + Err: ErrInvalidValue, + Protocol: p.protocol, + Values: p.args, + }) + } + + hp := p.args.(hostPort) + if hp.port == 80 { + return fmt.Sprintf("%s://%s", p.protocol, hp.host), hp.host, 80 + } + return fmt.Sprintf("%s://%s:%d", p.protocol, hp.host, hp.port), hp.host, hp.port +} + +func (p Endpoint) HTTPS() (url, host string, port int) { + if p.protocol != ProtocolHTTPS { + panic(Error{ + Op: "https", + Err: ErrInvalidValue, + Protocol: p.protocol, + Values: p.args, + }) + } + + hp := p.args.(hostPort) + if hp.port == 443 { + return fmt.Sprintf("%s://%s", p.protocol, hp.host), hp.host, 443 + } + return fmt.Sprintf("%s://%s:%d", p.protocol, hp.host, hp.port), hp.host, hp.port +} + +func (p Endpoint) File() (path string) { + if p.protocol != ProtocolFile { + panic(Error{ + Op: "file", + Err: ErrInvalidValue, + Protocol: p.protocol, + Values: p.args, + }) + } + + return p.args.(string) +} + +func (p Endpoint) TCP() (addr, host string, port int) { + if p.protocol != ProtocolTCP { + panic(Error{ + Op: "tcp", + Err: ErrInvalidValue, + Protocol: p.protocol, + Values: p.args, + }) + } + hp := p.args.(hostPort) + return fmt.Sprintf("%s:%d", hp.host, hp.port), hp.host, hp.port +} diff --git a/endpoint/endpoint_test.go b/endpoint/endpoint_test.go new file mode 100644 index 000000000..1601eb530 --- /dev/null +++ b/endpoint/endpoint_test.go @@ -0,0 +1,379 @@ +package endpoint + +import ( + "errors" + "log" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParse(t *testing.T) { + cases := []struct { + name string + cfg string + value Endpoint + err error + }{ + { + "invalid string", + "abcx", + Endpoint{}, + ErrUnsupportedProtocol, + }, + { + "normal http", + "http:example.com:80", + Endpoint{ProtocolHTTP, hostPort{"example.com", 80}}, + nil, + }, + { + "normal http with //", + "http://example.com:80", + Endpoint{ProtocolHTTP, hostPort{"example.com", 80}}, + nil, + }, + { + "normal http with multi /", + "http://////example.com:80", + Endpoint{}, + ErrInvalidValue, + }, + { + "normal http without port", + "http:example.com", + Endpoint{ProtocolHTTP, hostPort{"example.com", 80}}, + nil, + }, + { + "normal http without port, with //", + "http://example.com", + Endpoint{ProtocolHTTP, hostPort{"example.com", 80}}, + nil, + }, + { + "normal http without port, with multi /", + "http://///example.com", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in http", + "http:example.com:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in http, with //", + "http://example.com:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in http, with multi /", + "http://///example.com:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "normal https", + "https:example.com:443", + Endpoint{ProtocolHTTPS, hostPort{"example.com", 443}}, + nil, + }, + { + "normal https with //", + "https://example.com:443", + Endpoint{ProtocolHTTPS, hostPort{"example.com", 443}}, + nil, + }, + { + "normal https with multi /", + "https://///example.com:443", + Endpoint{}, + ErrInvalidValue, + }, + { + "normal https without port", + "https:example.com", + Endpoint{ProtocolHTTPS, hostPort{"example.com", 443}}, + nil, + }, + { + "normal https without port with //", + "https://example.com", + Endpoint{ProtocolHTTPS, hostPort{"example.com", 443}}, + nil, + }, + { + "normal https without port with multi /", + "https://///example.com", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in https", + "https:example.com:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in https with //", + "https://example.com:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in https with multi /", + "https://///example.com:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "not supported protocol", + "notsupported:abc.com", + Endpoint{}, + ErrUnsupportedProtocol, + }, + { + "not supported protocol with //", + "notsupported://abc.com", + Endpoint{}, + ErrUnsupportedProtocol, + }, + { + "normal file", + "file:/root/data", + Endpoint{ProtocolFile, "/root/data"}, + nil, + }, + { + "normal file with multi /", + "file:///root/data", + Endpoint{ProtocolFile, "/root/data"}, + nil, + }, + { + "files contains `:`", + "file:C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\TestStorage_Stat286526883\\001\\199446694", + Endpoint{ProtocolFile, "C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\TestStorage_Stat286526883\\001\\199446694"}, + nil, + }, + { + "files contains `:` with muti /", + "file:///C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\TestStorage_Stat286526883\\001\\199446694", + Endpoint{ProtocolFile, "/C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\TestStorage_Stat286526883\\001\\199446694"}, + nil, + }, + { + "normal tcp", + "tcp:127.0.0.1:8000", + Endpoint{ProtocolTCP, hostPort{"127.0.0.1", 8000}}, + nil, + }, + { + "normal tcp with //", + "tcp://127.0.0.1:8000", + Endpoint{ProtocolTCP, hostPort{"127.0.0.1", 8000}}, + nil, + }, + { + "normal tcp with multi /", + "tcp://///127.0.0.1:8000", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in tcp", + "tcp:127.0.0.1:xxx", + Endpoint{}, + ErrInvalidValue, + }, + { + "wrong port number in tcp with //", + "tcp://127.0.0.1:xxx", + Endpoint{}, + ErrInvalidValue, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + p, err := Parse(tt.cfg) + if tt.err == nil { + assert.Nil(t, err) + } else { + assert.True(t, errors.Is(err, tt.err)) + } + assert.EqualValues(t, tt.value, p) + }) + } +} + +func TestNewFile(t *testing.T) { + assert.Equal(t, Endpoint{ProtocolFile, "/example"}, NewFile("/example")) +} + +func TestNewHTTP(t *testing.T) { + assert.Equal(t, + Endpoint{ProtocolHTTP, hostPort{"example.com", 8080}}, + NewHTTP("example.com", 8080), + ) +} + +func TestNewHTTPS(t *testing.T) { + assert.Equal(t, + Endpoint{ProtocolHTTPS, hostPort{"example.com", 4433}}, + NewHTTPS("example.com", 4433), + ) +} + +func TestNewTCP(t *testing.T) { + assert.Equal(t, + Endpoint{ProtocolTCP, hostPort{"127.0.0.1", 8000}}, + NewTCP("127.0.0.1", 8000), + ) +} + +func TestEndpoint_Protocol(t *testing.T) { + ep := NewFile("/test") + + assert.Equal(t, ProtocolFile, ep.Protocol()) +} + +func TestEndpoint_Protocol2(t *testing.T) { + ep := NewTCP("127.0.0.1", 8000) + + assert.Equal(t, ProtocolTCP, ep.Protocol()) +} + +func TestEndpoint_String(t *testing.T) { + cases := []struct { + name string + value Endpoint + expected string + }{ + { + "file", + Endpoint{ProtocolFile, "/test"}, + "file:/test", + }, + { + "http without port", + Endpoint{ProtocolHTTP, hostPort{"example.com", 80}}, + "http:example.com:80", + }, + { + "http with port", + Endpoint{ProtocolHTTP, hostPort{"example.com", 8080}}, + "http:example.com:8080", + }, + { + "https without port", + Endpoint{ProtocolHTTPS, hostPort{"example.com", 443}}, + "https:example.com:443", + }, + { + "https with port", + Endpoint{ProtocolHTTPS, hostPort{"example.com", 4433}}, + "https:example.com:4433", + }, + { + "tcp with port", + Endpoint{ProtocolTCP, hostPort{"127.0.0.1", 8000}}, + "tcp:127.0.0.1:8000", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.value.String()) + }) + } +} + +func TestEndpoint(t *testing.T) { + p := NewFile("/test") + + assert.Panics(t, func() { + p.HTTP() + }) + assert.Panics(t, func() { + p.HTTPS() + }) + + assert.Panics(t, func() { + p.TCP() + }) + + assert.Equal(t, "/test", p.File()) +} + +func TestEndpoint_HTTP(t *testing.T) { + p := NewHTTP("example.com", 80) + + url, host, port := p.HTTP() + assert.Equal(t, "http://example.com", url) + assert.Equal(t, "example.com", host) + assert.Equal(t, 80, port) + + p = NewHTTP("example.com", 8080) + url, host, port = p.HTTP() + assert.Equal(t, "http://example.com:8080", url) + assert.Equal(t, "example.com", host) + assert.Equal(t, 8080, port) +} + +func TestEndpoint_HTTPS(t *testing.T) { + p := NewHTTPS("example.com", 443) + + url, host, port := p.HTTPS() + assert.Equal(t, "https://example.com", url) + assert.Equal(t, "example.com", host) + assert.Equal(t, 443, port) + + p = NewHTTPS("example.com", 4433) + url, host, port = p.HTTPS() + assert.Equal(t, "https://example.com:4433", url) + assert.Equal(t, "example.com", host) + assert.Equal(t, 4433, port) +} + +func TestEndpoint_TCP(t *testing.T) { + p := NewTCP("127.0.0.1", 8000) + + addr, host, port := p.TCP() + assert.Equal(t, "127.0.0.1:8000", addr) + assert.Equal(t, "127.0.0.1", host) + assert.Equal(t, 8000, port) +} + +func ExampleParse() { + ep, err := Parse("http:example.com") + if err != nil { + log.Fatal(err) + } + + switch ep.Protocol() { + case ProtocolHTTP: + url, host, port := ep.HTTP() + log.Println("url: ", url) + log.Println("host: ", host) + log.Println("port: ", port) + case ProtocolHTTPS: + url, host, port := ep.HTTPS() + log.Println("url: ", url) + log.Println("host: ", host) + log.Println("port: ", port) + case ProtocolFile: + path := ep.File() + log.Println("path: ", path) + case ProtocolTCP: + addr, host, port := ep.TCP() + log.Println("addr:", addr) + log.Println("host:", host) + log.Println("port", port) + default: + panic("unsupported protocol") + } +} diff --git a/pkg/endpoint/error.go b/endpoint/error.go similarity index 76% rename from pkg/endpoint/error.go rename to endpoint/error.go index 36f40a3bf..c983b2e61 100644 --- a/pkg/endpoint/error.go +++ b/endpoint/error.go @@ -7,24 +7,18 @@ import ( var ( // ErrUnsupportedProtocol will return if protocol is unsupported. - // - // Deprecated: Moved to github.com/beyondstorage/go-endpoint ErrUnsupportedProtocol = errors.New("unsupported protocol") // ErrInvalidValue means value is invalid. - // - // Deprecated: Moved to github.com/beyondstorage/go-endpoint ErrInvalidValue = errors.New("invalid value") ) // Error represents error related to endpoint. -// -// Deprecated: Moved to github.com/beyondstorage/go-endpoint type Error struct { Op string Err error Protocol string - Values []string + Values interface{} } func (e *Error) Error() string { diff --git a/endpoint/go.mod b/endpoint/go.mod new file mode 100644 index 000000000..105627017 --- /dev/null +++ b/endpoint/go.mod @@ -0,0 +1,5 @@ +module go.beyondstorage.io/endpoint + +go 1.15 + +require github.com/stretchr/testify v1.7.0 diff --git a/endpoint/go.sum b/endpoint/go.sum new file mode 100644 index 000000000..acb88a48f --- /dev/null +++ b/endpoint/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.mod b/go.mod index a4b9beb40..8c45f65be 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/Xuanwo/templateutils v0.1.0 github.com/dave/dst v0.26.2 github.com/golang/mock v1.6.0 - github.com/google/uuid v1.3.0 github.com/kevinburke/go-bindata v3.22.0+incompatible github.com/pelletier/go-toml v1.9.4 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index f82647bf6..621844135 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= diff --git a/pkg/credential/credential_test.go b/pkg/credential/credential_test.go deleted file mode 100644 index 5baf9350e..000000000 --- a/pkg/credential/credential_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package credential - -import ( - "errors" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" -) - -func TestProvider(t *testing.T) { - protocol := uuid.New().String() - args := []string{uuid.New().String(), uuid.New().String()} - - p := Provider{protocol: protocol, args: args} - - assert.Equal(t, protocol, p.Protocol()) - assert.EqualValues(t, args, p.Value()) -} - -func TestParse(t *testing.T) { - cases := []struct { - name string - cfg string - value Provider - err error - }{ - { - "hmac", - "hmac:ak:sk", - Provider{protocol: ProtocolHmac, args: []string{"ak", "sk"}}, - nil, - }, - { - "api key", - "apikey:key", - Provider{protocol: ProtocolAPIKey, args: []string{"key"}}, - nil, - }, - { - "file", - "file:/path/to/file", - Provider{protocol: ProtocolFile, args: []string{"/path/to/file"}}, - nil, - }, - { - "env", - "env", - Provider{protocol: ProtocolEnv}, - nil, - }, - { - "base64", - "base64:aGVsbG8sd29ybGQhCg==", - Provider{protocol: ProtocolBase64, args: []string{"aGVsbG8sd29ybGQhCg=="}}, - nil, - }, - { - "not supported protocol", - "notsupported:ak:sk", - Provider{}, - ErrUnsupportedProtocol, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - p, err := Parse(tt.cfg) - if tt.err == nil { - assert.Nil(t, err) - } else { - assert.True(t, errors.Is(err, tt.err)) - } - assert.EqualValues(t, tt.value, p) - }) - } -} diff --git a/pkg/endpoint/endpoint.go b/pkg/endpoint/endpoint.go deleted file mode 100644 index 5b5551bef..000000000 --- a/pkg/endpoint/endpoint.go +++ /dev/null @@ -1,102 +0,0 @@ -package endpoint - -import ( - "fmt" - "strconv" - "strings" -) - -const ( - // ProtocolHTTPS is the https credential protocol. - // - // Deprecated: Moved to github.com/beyondstorage/go-endpoint - ProtocolHTTPS = "https" - // ProtocolHTTP is the http credential protocol. - // - // Deprecated: Moved to github.com/beyondstorage/go-endpoint - ProtocolHTTP = "http" -) - -// Value is the required info to connect a service. -// -// Deprecated: Moved to github.com/beyondstorage/go-endpoint -type Value struct { - Protocol string - Host string - Port int -} - -// String will compose all info into a valid URL. -func (v Value) String() string { - if defaultPort[v.Protocol] == v.Port { - return fmt.Sprintf("%s://%s", v.Protocol, v.Host) - } - return fmt.Sprintf("%s://%s:%d", v.Protocol, v.Host, v.Port) -} - -// Parse will parse config string to create a endpoint Provider. -// -// Deprecated: Moved to github.com/beyondstorage/go-endpoint -func Parse(cfg string) (p Value, err error) { - s := strings.Split(cfg, ":") - if len(s) < 2 { - return Value{}, &Error{"parse", ErrInvalidValue, s[0], nil} - } - - defer func() { - if err != nil { - err = &Error{"parse", err, s[0], s[1:]} - } - }() - - var port int - if len(s) >= 3 { - xport, err := strconv.ParseInt(s[2], 10, 64) - if err != nil { - return Value{}, err - } - port = int(xport) - } - - switch s[0] { - case ProtocolHTTPS: - if port == 0 { - port = defaultPort[ProtocolHTTPS] - } - return NewHTTPS(s[1], port), nil - case ProtocolHTTP: - if port == 0 { - port = defaultPort[ProtocolHTTP] - } - return NewHTTP(s[1], port), nil - default: - return Value{}, ErrUnsupportedProtocol - } -} - -// NewHTTPS will create a static endpoint from parsed URL. -// -// Deprecated: Moved to github.com/beyondstorage/go-endpoint -func NewHTTPS(host string, port int) Value { - return Value{ - Protocol: ProtocolHTTPS, - Host: host, - Port: port, - } -} - -// NewHTTP will create a static endpoint from parsed URL. -// -// Deprecated: Moved to github.com/beyondstorage/go-endpoint -func NewHTTP(host string, port int) Value { - return Value{ - Protocol: ProtocolHTTP, - Host: host, - Port: port, - } -} - -var defaultPort = map[string]int{ - ProtocolHTTP: 80, - ProtocolHTTPS: 443, -} diff --git a/pkg/endpoint/endpoint_test.go b/pkg/endpoint/endpoint_test.go deleted file mode 100644 index 69cf223ec..000000000 --- a/pkg/endpoint/endpoint_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package endpoint - -import ( - "errors" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestValue_String(t *testing.T) { - t.Run("standard port", func(t *testing.T) { - v := &Value{ - Protocol: "http", - Host: "example.com", - Port: 80, - } - - assert.Equal(t, "http://example.com", v.String()) - }) - t.Run("non-standard port", func(t *testing.T) { - v := &Value{ - Protocol: "http", - Host: "example.com", - Port: 8080, - } - - assert.Equal(t, "http://example.com:8080", v.String()) - }) -} - -func TestParse(t *testing.T) { - cases := []struct { - name string - cfg string - value Value - err error - }{ - { - "invalid string", - "abcx", - Value{}, - ErrInvalidValue, - }, - { - "normal http", - "http:example.com:80", - NewHTTP("example.com", 80), - nil, - }, - { - "normal http without port", - "http:example.com", - NewHTTP("example.com", 80), - nil, - }, - { - "wrong port number in http", - "http:example.com:xxx", - Value{}, - strconv.ErrSyntax, - }, - { - "normal https", - "https:example.com:443", - NewHTTPS("example.com", 443), - nil, - }, - { - "normal https without port", - "https:example.com", - NewHTTPS("example.com", 443), - nil, - }, - { - "wrong port number in https", - "https:example.com:xxx", - Value{}, - strconv.ErrSyntax, - }, - { - "not supported protocol", - "notsupported:abc.com", - Value{}, - ErrUnsupportedProtocol, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - p, err := Parse(tt.cfg) - if tt.err == nil { - assert.Nil(t, err) - } else { - assert.True(t, errors.Is(err, tt.err)) - } - assert.EqualValues(t, tt.value, p) - }) - } -} diff --git a/pkg/iowrap/pipe.go b/pkg/iowrap/pipe.go deleted file mode 100644 index b13e1de83..000000000 --- a/pkg/iowrap/pipe.go +++ /dev/null @@ -1,171 +0,0 @@ -// Package iowrap's Pipe Inspired by following project: -// - golang stdlib: io.Pipe and bytes.Buffer -// - https://github.com/djherbis/nio -package iowrap - -import ( - "io" - "sync" -) - -// Pipe creates a synchronous in-memory pipe. -// It can be used to connect code expecting an io.Reader -// with code expecting an io.Writer. -// -// NOTES: -// - PipeReader and PipeWriter is not thread safe -// - Internal buffer will never be grow up, so write could be block while no space to write. -func Pipe() (r *PipeReader, w *PipeWriter) { - bp := &bufpipe{ - buf: make([]byte, 64*1024), // Set buffer to 64k - length: 64 * 1024, - } - bp.rwait.L = &bp.l - bp.wwait.L = &bp.l - - return &PipeReader{bp}, &PipeWriter{bp} -} - -type bufpipe struct { - buf []byte - length int // Buffer's length, which will not changed. - size int // Valid content size. - offset int // Offset of consumed data. - - // rwait and wwait will reuse the global lock. - l sync.Mutex - wwait sync.Cond - rwait sync.Cond - - werr error //nolint:structcheck - rerr error //nolint:structcheck -} - -// Available space to write data. -func (p *bufpipe) gap() int { - return p.length - p.size -} - -// All data have been consumed. -func (p *bufpipe) empty() bool { - return p.offset >= p.size -} - -// Only size and offset need to be reset. -func (p *bufpipe) reset() { - p.size = 0 - p.offset = 0 -} - -type PipeReader struct { - *bufpipe -} - -func (r *PipeReader) Read(p []byte) (n int, err error) { - // Lock here to prevent concurrent read/write on buffer. - r.l.Lock() - // Send signal to wwait to allow next write. - defer r.wwait.Signal() - defer r.l.Unlock() - - for r.empty() { - // Buffer is empty, reset to recover space. - r.reset() - - if r.rerr != nil { - return 0, io.ErrClosedPipe - } - - if r.werr != nil { - return 0, r.werr - } - - // Buffer has consumed, allow next write. - r.wwait.Signal() - // Wait for read. - r.rwait.Wait() - } - - n = copy(p, r.buf[r.offset:r.size]) - r.offset += n - return n, nil -} - -func (r *PipeReader) Close() error { - r.CloseWithError(nil) - - return nil -} - -func (r *PipeReader) CloseWithError(err error) { - if err == nil { - err = io.ErrClosedPipe - } - - r.l.Lock() - defer r.l.Unlock() - if r.rerr == nil { - r.rerr = err - r.rwait.Signal() - r.wwait.Signal() - } -} - -type PipeWriter struct { - *bufpipe -} - -func (w *PipeWriter) Write(p []byte) (n int, err error) { - var m int - - // Lock here to prevent concurrent read/write on buffer. - w.l.Lock() - // Send signal to rwait to allow next read. - defer w.rwait.Signal() - defer w.l.Unlock() - - l := len(p) - - for towrite := l; towrite > 0; towrite = l - n { - for w.gap() == 0 { - if w.rerr != nil { - return 0, w.rerr - } - - if w.werr != nil { - return 0, io.ErrClosedPipe - } - - // Buffer is full, allow next read. - w.rwait.Signal() - // Wait for write. - w.wwait.Wait() - } - - m = copy(w.buf, p[n:]) - w.size += m - n += m - } - - return -} - -func (w *PipeWriter) Close() error { - w.CloseWithError(nil) - - return nil -} - -func (w *PipeWriter) CloseWithError(err error) { - if err == nil { - err = io.EOF - } - - w.l.Lock() - defer w.l.Unlock() - if w.werr == nil { - w.werr = err - w.rwait.Signal() - w.wwait.Signal() - } -} diff --git a/pkg/iowrap/pipe_bench_test.go b/pkg/iowrap/pipe_bench_test.go deleted file mode 100644 index 487c43bd1..000000000 --- a/pkg/iowrap/pipe_bench_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package iowrap - -import ( - "io" - "io/ioutil" - "testing" - - "go.beyondstorage.io/v5/pkg/randbytes" -) - -func BenchmarkStdPipe(b *testing.B) { - cases := []struct { - name string - size int - }{ - {"4k", 4 * 1024}, - {"64k", 64 * 1024}, - {"4m", 4 * 1024 * 1024}, - } - - for _, v := range cases { - b.Run(v.name, func(b *testing.B) { - content := make([]byte, v.size) - _, err := randbytes.NewRand().Read(content) - if err != nil { - b.Error(err) - } - - r, w := io.Pipe() - - go func() { - _, _ = io.Copy(ioutil.Discard, r) - }() - - b.SetBytes(int64(v.size)) - b.StartTimer() - for i := 0; i < b.N; i++ { - _, _ = w.Write(content) - } - b.StopTimer() - }) - } -} - -func BenchmarkIowrapPipe(b *testing.B) { - cases := []struct { - name string - size int - }{ - {"4k", 4 * 1024}, - {"64k", 64 * 1024}, - {"4m", 4 * 1024 * 1024}, - } - - for _, v := range cases { - b.Run(v.name, func(b *testing.B) { - content := make([]byte, v.size) - _, err := randbytes.NewRand().Read(content) - if err != nil { - b.Error(err) - } - - r, w := Pipe() - - go func() { - _, _ = io.Copy(ioutil.Discard, r) - }() - - b.SetBytes(int64(v.size)) - b.StartTimer() - for i := 0; i < b.N; i++ { - _, _ = w.Write(content) - } - b.StopTimer() - }) - } -} diff --git a/pkg/iowrap/pipe_test.go b/pkg/iowrap/pipe_test.go deleted file mode 100644 index 64d29275c..000000000 --- a/pkg/iowrap/pipe_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package iowrap - -import ( - "github.com/stretchr/testify/assert" - "go.beyondstorage.io/v5/pkg/randbytes" - "io" - "io/ioutil" - "testing" -) - -func TestPipe(t *testing.T) { - cases := []struct { - name string - size int - }{ - {"1B", 1}, - {"4k", 4 * 1024}, - {"16m", 16 * 1024 * 1024}, - } - - for _, v := range cases { - t.Run(v.name, func(t *testing.T) { - expected := make([]byte, v.size) - _, err := randbytes.NewRand().Read(expected) - if err != nil { - t.Error(err) - } - - r, w := Pipe() - io.Pipe() - - go func() { - defer w.Close() - - _, _ = w.Write(expected) - }() - - actual, err := ioutil.ReadAll(r) - if err != nil { - t.Error(err) - } - assert.EqualValues(t, expected, actual) - }) - } -}