Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: initiateNewMultipartUpload in TypeScript #1189

Merged
merged 7 commits into from
Jul 25, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion src/internal/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import {
uriResourceEscape,
} from './helper.ts'
import { request } from './request.ts'
import { drainResponse, readAsString } from './response.ts'
import { drainResponse, readAsBuffer, readAsString } from './response.ts'
import type { Region } from './s3-endpoints.ts'
import { getS3Endpoint } from './s3-endpoints.ts'
import type {
Expand All @@ -49,6 +49,7 @@ import type {
} from './type.ts'
import type { UploadedPart } from './xml-parser.ts'
import * as xmlParsers from './xml-parser.ts'
import { parseInitiateMultipart } from './xml-parser.ts'

// will be replaced by bundler.
const Package = { version: process.env.MINIO_JS_PACKAGE_VERSION || 'development' }
Expand Down Expand Up @@ -862,6 +863,29 @@ export class TypedClient {
await this.makeRequestAsyncOmit({ method, bucketName, objectName, headers, query }, '', [200, 204])
}

// Calls implemented below are related to multipart.

/**
* Initiate a new multipart upload.
* @internal
*/
async initiateNewMultipartUpload(bucketName: string, objectName: string, headers: RequestHeaders): Promise<string> {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`)
}
if (!isObject(headers)) {
throw new errors.InvalidObjectNameError('contentType should be of type "object"')
}
const method = 'POST'
const query = 'uploads'
const res = await this.makeRequestAsync({ method, bucketName, objectName, query, headers })
const body = await readAsBuffer(res)
return parseInitiateMultipart(body.toString())
}

/**
* Internal Method to abort a multipart upload request in case of any errors.
*
Expand Down
14 changes: 14 additions & 0 deletions src/internal/xml-parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,17 @@ export function parseListBucket(xml: string) {
}
return result
}

export function parseInitiateMultipart(xml: string): string {
let xmlobj = parseXml(xml)

if (!xmlobj.InitiateMultipartUploadResult) {
throw new errors.InvalidXMLError('Missing tag: "InitiateMultipartUploadResult"')
}
xmlobj = xmlobj.InitiateMultipartUploadResult

if (xmlobj.UploadId) {
return xmlobj.UploadId
}
throw new errors.InvalidXMLError('Missing tag: "UploadId"')
}
41 changes: 8 additions & 33 deletions src/minio.js
Original file line number Diff line number Diff line change
Expand Up @@ -1381,33 +1381,6 @@ export class Client extends TypedClient {
})
}

// Calls implemented below are related to multipart.

// Initiate a new multipart upload.
initiateNewMultipartUpload(bucketName, objectName, metaData, cb) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`)
}
if (!isObject(metaData)) {
throw new errors.InvalidObjectNameError('contentType should be of type "object"')
}
var method = 'POST'
let headers = Object.assign({}, metaData)
var query = 'uploads'
this.makeRequest({ method, bucketName, objectName, query, headers }, '', [200], '', true, (e, response) => {
if (e) {
return cb(e)
}
var transformer = transformers.getInitiateMultipartTransformer()
pipesetup(response, transformer)
.on('error', (e) => cb(e))
.on('data', (uploadId) => cb(null, uploadId))
})
}

// Complete the multipart upload. After all the parts are uploaded issuing
// this call will aggregate the parts on the server into a single object.
completeMultipartUpload(bucketName, objectName, uploadId, etags, cb) {
Expand Down Expand Up @@ -2638,12 +2611,14 @@ export class Client extends TypedClient {

const newUploadHeaders = destObjConfig.getHeaders()

me.initiateNewMultipartUpload(destObjConfig.Bucket, destObjConfig.Object, newUploadHeaders, (err, uploadId) => {
if (err) {
return cb(err, null)
}
performUploadParts(uploadId)
})
me.initiateNewMultipartUpload(destObjConfig.Bucket, destObjConfig.Object, newUploadHeaders).then(
(uploadId) => {
performUploadParts(uploadId)
},
(err) => {
cb(err, null)
},
)
})
.catch((error) => {
cb(error, null)
Expand Down
19 changes: 9 additions & 10 deletions src/object-uploader.js
Original file line number Diff line number Diff line change
Expand Up @@ -125,16 +125,15 @@ export class ObjectUploader extends Transform {

// If no upload ID exists, initiate a new one.
if (!id) {
this.client.initiateNewMultipartUpload(this.bucketName, this.objectName, this.metaData, (err, id) => {
if (err) {
return callback(err)
}

this.id = id

// We are now ready to accept new chunks — this will flush the buffered chunk.
this.emit('ready')
})
this.client.initiateNewMultipartUpload(this.bucketName, this.objectName, this.metaData).then(
(id) => {
this.id = id

// We are now ready to accept new chunks — this will flush the buffered chunk.
this.emit('ready')
},
(err) => callback(err),
)

return
}
Expand Down
5 changes: 0 additions & 5 deletions src/transformers.js
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,6 @@ export function getListMultipartTransformer() {
return getConcater(xmlParsers.parseListMultipart)
}

// Parses initMultipartUpload response.
export function getInitiateMultipartTransformer() {
return getConcater(xmlParsers.parseInitiateMultipart)
}

// Parses listObjects response.
export function getListObjectsTransformer() {
return getConcater(xmlParsers.parseListObjects)
Expand Down
15 changes: 0 additions & 15 deletions src/xml-parsers.js
Original file line number Diff line number Diff line change
Expand Up @@ -167,21 +167,6 @@ export function parseBucketNotification(xml) {
return result
}

// parse XML response when a new multipart upload is initiated
export function parseInitiateMultipart(xml) {
var xmlobj = parseXml(xml)

if (!xmlobj.InitiateMultipartUploadResult) {
throw new errors.InvalidXMLError('Missing tag: "InitiateMultipartUploadResult"')
}
xmlobj = xmlobj.InitiateMultipartUploadResult

if (xmlobj.UploadId) {
return xmlobj.UploadId
}
throw new errors.InvalidXMLError('Missing tag: "UploadId"')
}

// parse XML response when a multipart upload is completed
export function parseCompleteMultipart(xml) {
var xmlobj = parseXml(xml).CompleteMultipartUploadResult
Expand Down
8 changes: 2 additions & 6 deletions tests/functional/functional-tests.js
Original file line number Diff line number Diff line change
Expand Up @@ -829,9 +829,7 @@ describe('functional tests', function () {
describe('listIncompleteUploads removeIncompleteUpload', () => {
step(
`initiateNewMultipartUpload(bucketName, objectName, metaData, cb)_bucketName:${bucketName}, objectName:${_65mbObjectName}, metaData:${metaData}`,
(done) => {
client.initiateNewMultipartUpload(bucketName, _65mbObjectName, metaData, done)
},
() => client.initiateNewMultipartUpload(bucketName, _65mbObjectName, metaData),
)
step(
`listIncompleteUploads(bucketName, prefix, recursive)_bucketName:${bucketName}, prefix:${_65mbObjectName}, recursive: true_`,
Expand Down Expand Up @@ -4282,9 +4280,7 @@ describe('functional tests', function () {

step(
`initiateNewMultipartUpload(bucketName, objectName, metaData, cb)_bucketName:${spBucketName}, objectName:${spObjWithPrefix}, metaData:${metaData}`,
(done) => {
client.initiateNewMultipartUpload(spBucketName, spObjWithPrefix, metaData, done)
},
() => client.initiateNewMultipartUpload(spBucketName, spObjWithPrefix, metaData),
)

step(
Expand Down
Loading