Skip to content
This repository was archived by the owner on Oct 22, 2021. It is now read-only.

Implement commitAppend #18

Merged
merged 2 commits into from
Apr 29, 2021
Merged

Implement commitAppend #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ea96b21
Implement commitAppend
Apr 27, 2021
ded58ec
add append related metadata
Apr 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 105 additions & 21 deletions generated.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.14
require (
github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible
github.com/aos-dev/go-integration-test/v3 v3.0.0
github.com/aos-dev/go-storage/v3 v3.6.0
github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please provide append related metadata in obejct.

github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/google/uuid v1.2.0
github.com/satori/go.uuid v1.2.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9
github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M=
github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q=
github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY=
github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM=
github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk=
github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA=
github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw=
github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA=
github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0=
github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws=
github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4=
github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=
Expand Down
5 changes: 4 additions & 1 deletion service.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@ optional = ["offset", "io_callback", "size"]
[namespace.storage.op.write]
optional = ["content_md5", "content_type", "io_callback", "storage_class", "server_side_encryption", "server_side_data_encryption", "server_side_encryption_key_id"]

[namespace.storage.op.create_append]
optional = ["content_type", "server_side_encryption", "server_side_encryption_key_id", "server_side_data_encryption", "storage_class"]

[namespace.storage.op.write_append]
optional = ["server_side_encryption", "server_side_data_encryption", "server_side_encryption_key_id"]
optional = ["content_md5", "io_callback"]

[pairs.default_service_pairs]
type = "DefaultServicePairs"
Expand Down
52 changes: 42 additions & 10 deletions storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ import (
. "github.com/aos-dev/go-storage/v3/types"
)

func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) {
return
}

func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
o = s.newObject(false)
o.Mode = ModeRead
Expand All @@ -23,11 +27,36 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
}

func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) {
rp := s.getAbsPath(path)

options := make([]oss.Option, 0)
options = append(options, oss.ContentLength(0))
if opt.HasContentType {
options = append(options, oss.ContentType(opt.ContentType))
}
if opt.HasStorageClass {
options = append(options, oss.StorageClass(oss.StorageClassType(opt.StorageClass)))
}
if opt.HasServerSideEncryption {
options = append(options, oss.ServerSideEncryption(opt.ServerSideEncryption))
}
if opt.HasServerSideDataEncryption {
options = append(options, oss.ServerSideDataEncryption(opt.ServerSideDataEncryption))
}
if opt.HasServerSideEncryptionKeyID {
options = append(options, oss.ServerSideEncryptionKeyID(opt.ServerSideEncryptionKeyID))
}

offset, err := s.bucket.AppendObject(rp, nil, 0, options...)
if err != nil {
return
}

o = s.newObject(true)
o.Mode = ModeRead | ModeAppend
o.ID = s.getAbsPath(path)
o.ID = rp
o.Path = path
o.SetAppendOffset(0)
o.SetAppendOffset(offset)
return o, nil
}

Expand Down Expand Up @@ -242,8 +271,17 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6
}

func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) {
if !o.Mode.IsAppend() {
err = fmt.Errorf("object not appendable")
return
}

rp := o.GetID()

if opt.HasIoCallback {
r = iowrap.CallbackReader(r, opt.IoCallback)
}

offset, ok := o.GetAppendOffset()
if !ok {
err = fmt.Errorf("append offset is not set")
Expand All @@ -252,14 +290,8 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size

options := make([]oss.Option, 0)
options = append(options, oss.ContentLength(size))
if opt.HasServerSideEncryption {
options = append(options, oss.ServerSideEncryption(opt.ServerSideEncryption))
}
if opt.HasServerSideDataEncryption {
options = append(options, oss.ServerSideDataEncryption(opt.ServerSideDataEncryption))
}
if opt.HasServerSideEncryptionKeyID {
options = append(options, oss.ServerSideEncryptionKeyID(opt.ServerSideEncryptionKeyID))
if opt.HasContentMd5 {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should pass the content_type the first time of the AppendObject call.

As we added the content_type in Object, but that value is not stored by service. If the user tries to Stat an object, he/she will found those metadata is missing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

content-type has set in create_append.

options = append(options, oss.ContentMD5(opt.ContentMd5))
}

offset, err = s.bucket.AppendObject(rp, r, offset, options...)
Expand Down