Skip to content

feat: Support multi-part uploads in JS API #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
b5c80d9
Add tests for splitFileExtension
lucaas May 26, 2023
16c2d94
Merge remote-tracking branch 'origin/main' into backlog/multipart-upl…
lucaas May 29, 2023
cfe44d4
refactor: Move session.createComponent to Uploader
lucaas May 29, 2023
ba74ec8
Add getChunkSize util
lucaas May 29, 2023
f6e6c30
feat: Multipart uploads
lucaas May 30, 2023
31a5da0
Add docstrings
lucaas May 30, 2023
71ced51
Merge remote-tracking branch 'origin/main' into backlog/multipart-upl…
lucaas Jun 2, 2023
e3e9a3e
Add new backOff util
lucaas Jun 2, 2023
1bcb31a
Async-ify uploadNextChunk, use backOff util
lucaas Jun 2, 2023
f77b79a
Promisify methods, floor progress callback, format
lucaas Jun 7, 2023
71c99fa
More docstrings
lucaas Jun 7, 2023
36fa25f
Add tests for uploader
lucaas Jun 7, 2023
bb1e88f
Add tests for aborting, callbacks
lucaas Jun 8, 2023
cfe31ef
Use backOff for single-part uploads as well
lucaas Jun 8, 2023
6a49bc6
Update to snake_case API
lucaas Jun 8, 2023
c26e5fe
Add debug variables for investigating upload params
lucaas Jun 8, 2023
b7c47ea
Fix tests
lucaas Jun 8, 2023
4a4759f
Don't lint yarn sources
lucaas Jun 8, 2023
b76e737
Do not re-attempt aborted requests
lucaas Jun 8, 2023
b97331a
Update fixture to new format
lucaas Jun 9, 2023
3c909b4
Merge branch 'main' into backlog/multipart-upload-in-studio/support-m…
lucaas Jun 13, 2023
dcdbcf4
Remove underscore number seperator, only supported in recent browsers
lucaas Jun 13, 2023
cb1d7fd
Merge branch 'backlog/multipart-upload-in-studio/support-multipart-up…
lucaas Jun 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ coverage/
build/
dist/
.eggs/
.yarn/
renovate.json
167 changes: 19 additions & 148 deletions source/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,8 @@ import {
ServerValidationError,
ServerError,
AbortError,
CreateComponentError,
} from "./error.js";
import { SERVER_LOCATION_ID } from "./constant.js";

import normalizeString from "./util/normalize_string.js";
import type {
ActionResponse,
CallOptions,
Expand All @@ -37,31 +34,12 @@ import type {
UpdateResponse,
} from "./types.js";
import { convertToIsoString } from "./util/convert_to_iso_string.js";
import { Uploader } from "./uploader.js";

const logger = loglevel.getLogger("ftrack_api");

const ENCODE_DATETIME_FORMAT = "YYYY-MM-DDTHH:mm:ss";

/**
* Create component from *file* and add to server location.
*
* @param {fileName} The name of the file.
* @return {array} Array with [basename, extension] from filename.
*/
function splitFileExtension(fileName: string) {
let basename = fileName || "";
let extension =
fileName.slice((Math.max(0, fileName.lastIndexOf(".")) || Infinity) + 1) ||
"";

if (extension.length) {
extension = `.${extension}`;
basename = fileName.slice(0, -1 * extension.length) || "";
}

return [basename, extension];
}

/**
* ftrack API session
* @class Session
Expand Down Expand Up @@ -927,136 +905,29 @@ export class Session {
* @return {Promise} Promise resolved with the response when creating
* Component and ComponentLocation.
*/
createComponent<T extends Data = Data>(
async createComponent(
file: Blob,
options: CreateComponentOptions = {}
): Promise<
[CreateResponse<T>, CreateResponse<T>, GetUploadMetadataResponse]
readonly [CreateResponse, CreateResponse, GetUploadMetadataResponse]
> {
const componentName = options.name ?? (file as File).name;

let normalizedFileName;
if (componentName) {
normalizedFileName = normalizeString(componentName);
}

if (!normalizedFileName) {
throw new CreateComponentError("Component name is missing.");
}
const fileNameParts = splitFileExtension(normalizedFileName);
const defaultProgress = (progress: number) => progress;
const defaultAbort = () => {};

if (options.xhr) {
logger.warn(
"[session.createComponent] options.xhr is deprecated, use options.signal for aborting uploads."
);
}

const data = options.data || {};
const onProgress = options.onProgress || defaultProgress;
const xhr = options.xhr || new XMLHttpRequest();
const onAborted = options.onAborted || defaultAbort;

const fileType = data.file_type || fileNameParts[1];
const fileName = data.name || fileNameParts[0];
const fileSize = data.size || file.size;
const componentId = data.id || uuidV4();
const componentLocationId = uuidV4();
let url: string;
let headers: Record<string, string> = {};

const handleAbortSignal = () => {
xhr.abort();
options.signal?.removeEventListener("abort", handleAbortSignal);
};
options.signal?.addEventListener("abort", handleAbortSignal);

const updateOnProgressCallback = (
oEvent: ProgressEvent<XMLHttpRequestEventTarget>
) => {
let progress = 0;

if (oEvent.lengthComputable) {
progress = Math.floor((oEvent.loaded / oEvent.total) * 100);
}

onProgress(progress);
};

logger.debug("Registering component and fetching upload metadata.");

const component = Object.assign(data, {
id: componentId,
name: fileName,
file_type: fileType,
size: fileSize,
});
const componentLocation = {
id: componentLocationId,
component_id: componentId,
resource_identifier: componentId,
location_id: SERVER_LOCATION_ID,
};

const componentAndLocationPromise = this.call<
[CreateResponse<T>, CreateResponse<T>, GetUploadMetadataResponse]
>([
operation.create("FileComponent", component),
operation.create("ComponentLocation", componentLocation),
{
action: "get_upload_metadata",
file_name: `${fileName}${fileType}`,
file_size: fileSize,
component_id: componentId,
},
]).then((response) => {
url = response[2].url;
headers = response[2].headers;
return response;
});

return componentAndLocationPromise.then(() => {
logger.debug(`Uploading file to: ${url}`);

return new Promise((resolve, reject) => {
// wait until file is uploaded
xhr.upload.addEventListener("progress", updateOnProgressCallback);
xhr.open("PUT", url, true);
xhr.onabort = () => {
onAborted();
this.delete("FileComponent", [componentId]).then(() => {
reject(
new CreateComponentError(
"Upload aborted by client",
"UPLOAD_ABORTED"
)
);
});
};
return new Promise((resolve, reject) => {
const uploader = new Uploader(this, file, {
...options,
onError(error) {
reject(error);
},
onComplete() {
// TODO: Deprecate createComponent response.
resolve([
uploader.createComponentResponse!,
uploader.createComponentLocationResponse!,
uploader.uploadMetadata!,
]);
},
});

for (const key in headers) {
if (headers.hasOwnProperty(key) && key !== "Content-Length") {
xhr.setRequestHeader(key, headers[key]);
}
}
xhr.onload = () => {
if (xhr.status >= 400) {
reject(
new CreateComponentError(`Failed to upload file: ${xhr.status}`)
);
}
resolve(xhr.response);
};
xhr.onerror = () => {
this.delete("FileComponent", [componentId]).then(() => {
reject(
new CreateComponentError(`Failed to upload file: ${xhr.status}`)
);
});
};
xhr.send(file);
}).then(() => componentAndLocationPromise);
uploader.start();
});
}
}
19 changes: 17 additions & 2 deletions source/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,27 @@ export interface DelayedJobResponse {
export interface EncodeMediaResponse {
job_id: string;
}
export interface GetUploadMetadataResponse {

export interface SinglePartGetUploadMetadataResponse {
url: string;
component_id?: string;
component_id: string;
headers: Data;
}

export interface MultiPartUploadPart {
signed_url: string;
part_number: number;
}
export interface MultiPartGetUploadMetadataResponse {
component_id: string;
urls: MultiPartUploadPart[];
upload_id: string;
}

export type GetUploadMetadataResponse =
| SinglePartGetUploadMetadataResponse
| MultiPartGetUploadMetadataResponse;

export interface SendReviewSessionInviteResponse {
sent: true;
}
Expand Down
Loading