Skip to content

Commit

Permalink
fix: simplify the code for downloadInChunks (#2323)
Browse files Browse the repository at this point in the history
* fix: simplify the code for downloadInChunks

* cleanup p-limit function
  • Loading branch information
ddelgrosso1 authored Oct 11, 2023
1 parent a7d09c1 commit 6519929
Showing 1 changed file with 19 additions and 31 deletions.
50 changes: 19 additions & 31 deletions src/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -651,42 +651,30 @@ export class TransferManager {
let chunkEnd = start + chunkSize - 1;
chunkEnd = chunkEnd > size ? size : chunkEnd;
promises.push(
limit(() =>
file
.download({
start: chunkStart,
end: chunkEnd,
[GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_SHARDED,
})
.then(resp => {
return fileToWrite.write(resp[0], 0, resp[0].length, chunkStart);
})
)
limit(async () => {
const resp = await file.download({
start: chunkStart,
end: chunkEnd,
[GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_SHARDED,
});
return fileToWrite.write(resp[0], 0, resp[0].length, chunkStart);
})
);

start += chunkSize;
}

return new Promise((resolve, reject) => {
let results: DownloadResponse;
Promise.all(promises)
.then(data => {
results = data.map(result => result.buffer) as DownloadResponse;
if (options.validation === 'crc32c') {
return CRC32C.fromFile(filePath);
}
return;
})
.then(() => {
resolve(results);
})
.catch(e => {
reject(e);
})
.finally(() => {
fileToWrite.close();
});
});
let results: DownloadResponse;
try {
const data = await Promise.all(promises);
results = data.map(result => result.buffer) as DownloadResponse;
if (options.validation === 'crc32c') {
await CRC32C.fromFile(filePath);
}
return results;
} finally {
fileToWrite.close();
}
}

/**
Expand Down

0 comments on commit 6519929

Please sign in to comment.