Skip to content

Commit

Permalink
[b2] use HEAD instead of Range 0-1 to stat files
Browse files Browse the repository at this point in the history
Fixes #71
  • Loading branch information
kurin committed Mar 27, 2020
1 parent ade0c1f commit 3ad36f1
Show file tree
Hide file tree
Showing 7 changed files with 36 additions and 13 deletions.
3 changes: 2 additions & 1 deletion b2/b2.go
Original file line number Diff line number Diff line change
Expand Up @@ -613,8 +613,9 @@ func (discard) Write(p []byte) (int, error) {
}

func (b *Bucket) getObject(ctx context.Context, name string) (*Object, error) {
fr, err := b.b.downloadFileByName(ctx, name, 0, 1)
fr, err := b.b.downloadFileByName(ctx, name, 0, 0, true)
if err != nil {
fmt.Printf("%v: %T\n", err, err)
return nil, err
}
io.Copy(discard{}, fr)
Expand Down
2 changes: 1 addition & 1 deletion b2/b2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func (t *testBucket) listUnfinishedLargeFiles(ctx context.Context, count int, co
return nil, "", fmt.Errorf("testBucket.listUnfinishedLargeFiles(ctx, %d, %q): not implemented", count, cont)
}

func (t *testBucket) downloadFileByName(_ context.Context, name string, offset, size int64) (b2FileReaderInterface, error) {
func (t *testBucket) downloadFileByName(_ context.Context, name string, offset, size int64, _ bool) (b2FileReaderInterface, error) {
gmux.Lock()
defer gmux.Unlock()
f := t.files[name]
Expand Down
6 changes: 3 additions & 3 deletions b2/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ type beBucketInterface interface {
listFileNames(context.Context, int, string, string, string) ([]beFileInterface, string, error)
listFileVersions(context.Context, int, string, string, string, string) ([]beFileInterface, string, string, error)
listUnfinishedLargeFiles(context.Context, int, string) ([]beFileInterface, string, error)
downloadFileByName(context.Context, string, int64, int64) (beFileReaderInterface, error)
downloadFileByName(context.Context, string, int64, int64, bool) (beFileReaderInterface, error)
hideFile(context.Context, string) (beFileInterface, error)
getDownloadAuthorization(context.Context, string, time.Duration, string) (string, error)
baseURL() string
Expand Down Expand Up @@ -427,11 +427,11 @@ func (b *beBucket) listUnfinishedLargeFiles(ctx context.Context, count int, cont
return files, cont, nil
}

func (b *beBucket) downloadFileByName(ctx context.Context, name string, offset, size int64) (beFileReaderInterface, error) {
func (b *beBucket) downloadFileByName(ctx context.Context, name string, offset, size int64, header bool) (beFileReaderInterface, error) {
var reader beFileReaderInterface
f := func() error {
g := func() error {
fr, err := b.b2bucket.downloadFileByName(ctx, name, offset, size)
fr, err := b.b2bucket.downloadFileByName(ctx, name, offset, size, header)
if err != nil {
return err
}
Expand Down
6 changes: 3 additions & 3 deletions b2/baseline.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ type b2BucketInterface interface {
listFileNames(context.Context, int, string, string, string) ([]b2FileInterface, string, error)
listFileVersions(context.Context, int, string, string, string, string) ([]b2FileInterface, string, string, error)
listUnfinishedLargeFiles(context.Context, int, string) ([]b2FileInterface, string, error)
downloadFileByName(context.Context, string, int64, int64) (b2FileReaderInterface, error)
downloadFileByName(context.Context, string, int64, int64, bool) (b2FileReaderInterface, error)
hideFile(context.Context, string) (b2FileInterface, error)
getDownloadAuthorization(context.Context, string, time.Duration, string) (string, error)
baseURL() string
Expand Down Expand Up @@ -367,8 +367,8 @@ func (b *b2Bucket) listUnfinishedLargeFiles(ctx context.Context, count int, cont
return files, cont, nil
}

func (b *b2Bucket) downloadFileByName(ctx context.Context, name string, offset, size int64) (b2FileReaderInterface, error) {
fr, err := b.b.DownloadFileByName(ctx, name, offset, size)
func (b *b2Bucket) downloadFileByName(ctx context.Context, name string, offset, size int64, header bool) (b2FileReaderInterface, error) {
fr, err := b.b.DownloadFileByName(ctx, name, offset, size, header)
if err != nil {
code, _ := base.Code(err)
switch code {
Expand Down
18 changes: 18 additions & 0 deletions b2/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -919,6 +919,24 @@ func TestDeleteWithoutName(t *testing.T) {
}
}

func TestZeroByteObject(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()

bucket, done := startLiveTest(ctx, t)
defer done()

_, _, err := writeFile(ctx, bucket, smallFileName, 0, 0)
if err != nil {
t.Fatal(err)
}

if err := bucket.Object(smallFileName).Delete(ctx); err != nil {
t.Fatal(err)
}
}

func TestListUnfinishedLargeFiles(t *testing.T) {
ctx := context.Background()
bucket, done := startLiveTest(ctx, t)
Expand Down
2 changes: 1 addition & 1 deletion b2/reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func (r *Reader) thread() {
}
var b backoff
redo:
fr, err := r.o.b.b.downloadFileByName(r.ctx, r.name, offset, size)
fr, err := r.o.b.b.downloadFileByName(r.ctx, r.name, offset, size, false)
if err == errNoMoreContent {
// this read generated a 416 so we are entirely past the end of the object
r.readOffEnd = true
Expand Down
12 changes: 8 additions & 4 deletions base/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,10 @@ func logRequest(req *http.Request, args []byte) {
hstr := strings.Join(headers, ";")
method := req.Header.Get("X-Blazer-Method")
if args != nil {
blog.V(2).Infof(">> %s uri: %v headers: {%s} args: (%s)", method, req.URL, hstr, string(args))
blog.V(2).Infof(">> %s %v: %v headers: {%s} args: (%s)", method, req.Method, req.URL, hstr, string(args))
return
}
blog.V(2).Infof(">> %s uri: %v {%s} (no args)", method, req.URL, hstr)
blog.V(2).Infof(">> %s %v: %v {%s} (no args)", method, req.Method, req.URL, hstr)
}

var authRegexp = regexp.MustCompile(`"authorizationToken": ".[^"]*"`)
Expand Down Expand Up @@ -1115,9 +1115,13 @@ func mkRange(offset, size int64) string {
}

// DownloadFileByName wraps b2_download_file_by_name.
func (b *Bucket) DownloadFileByName(ctx context.Context, name string, offset, size int64) (*FileReader, error) {
func (b *Bucket) DownloadFileByName(ctx context.Context, name string, offset, size int64, header bool) (*FileReader, error) {
uri := fmt.Sprintf("%s/file/%s/%s", b.b2.downloadURI, b.Name, escape(name))
req, err := http.NewRequest("GET", uri, nil)
method := "GET"
if header {
method = "HEAD"
}
req, err := http.NewRequest(method, uri, nil)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 3ad36f1

Please sign in to comment.