Skip to content

Commit

Permalink
Fix downloadChunked for larger files
Browse files Browse the repository at this point in the history
closes #298
  • Loading branch information
corny committed Sep 4, 2023
1 parent 14b47a0 commit 86c4fb8
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 13 deletions.
26 changes: 14 additions & 12 deletions client.go
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,6 @@ func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *For
} else {
// we have length information, let's download by chunks!
c.downloadChunked(ctx, req, w, format)

}

return r, contentLength, nil
Expand Down Expand Up @@ -402,19 +401,21 @@ func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.P
currentChunk := atomic.Uint32{}
for i := 0; i < maxRoutines; i++ {
go func() {
i := int(currentChunk.Add(1)) - 1
if i > len(chunks) {
// no more chunks
return
}
for {
chunkIndex := int(currentChunk.Add(1)) - 1
if chunkIndex >= len(chunks) {
// no more chunks
return
}

chunk := &chunks[i]
err := c.downloadChunk(req.Clone(cancelCtx), chunk)
close(chunk.data)
chunk := &chunks[chunkIndex]
err := c.downloadChunk(req.Clone(cancelCtx), chunk)
close(chunk.data)

if err != nil {
abort(err)
return
if err != nil {
abort(err)
return
}
}
}()
}
Expand All @@ -424,6 +425,7 @@ func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.P
for i := 0; i < len(chunks); i++ {
select {
case <-cancelCtx.Done():
abort(context.Canceled)
return
case data := <-chunks[i].data:
_, err := io.Copy(w, bytes.NewBuffer(data))
Expand Down
11 changes: 10 additions & 1 deletion client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,15 @@ func TestGetVideoWithManifestURL(t *testing.T) {
func TestGetStream(t *testing.T) {
assert, require := assert.New(t), require.New(t)

expectedSize := 988479

// Create testclient to enforce re-using of routines
testClient := Client{
Debug: true,
MaxRoutines: 10,
ChunkSize: int64(expectedSize) / 11,
}

// Download should not last longer than a minute.
// Otherwise we assume Youtube is throtteling us.
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
Expand All @@ -141,7 +150,7 @@ func TestGetStream(t *testing.T) {

reader, size, err := testClient.GetStreamContext(ctx, video, &video.Formats[0])
require.NoError(err)
assert.EqualValues(988479, size)
assert.EqualValues(expectedSize, size)

data, err := io.ReadAll(reader)
require.NoError(err)
Expand Down

0 comments on commit 86c4fb8

Please sign in to comment.