Skip to content

Commit 3b3cdf9

Browse files
EmptyInfinityEmptyInfinity
authored
Fix upload limit (#64)
1 parent 1eca4d7 commit 3b3cdf9

File tree

9 files changed

+174
-67
lines changed

9 files changed

+174
-67
lines changed

‎CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
## 2.1.0 (May 06, 2022)
2+
* Now method `uploadAttachment` from `AttachmentProcessor` automatically sets the `contentType`
3+
* Fixed retrying logic for method `uploadAttachment`
4+
* Removed validation of the file size (`REQUEST_MAX_BODY_LENGTH` env) because it not works with streams
5+
16
## 2.0.2 (April 13, 2022)
27
* Fix dependencies
38

‎dist/spec-integration/attachmentProcessor.spec.js

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,20 @@ const fs_1 = __importDefault(require("fs"));
77
const path_1 = __importDefault(require("path"));
88
const chai_1 = require("chai");
99
const common_1 = require("./common");
10-
const AttachmentProcessor_1 = require("../src/attachment/AttachmentProcessor");
1110
(0, common_1.setEnvs)();
11+
// eslint-disable-next-line import/first
12+
const AttachmentProcessor_1 = require("../src/attachment/AttachmentProcessor");
1213
describe('AttachmentProcessor', () => {
1314
describe('uploadAttachment', () => {
1415
it('uploadAttachment (/samples/sample.json)', async () => {
1516
const file = fs_1.default.createReadStream(path_1.default.join(__dirname, './samples/sample.json'));
16-
const res = await new AttachmentProcessor_1.AttachmentProcessor().uploadAttachment(file, 'application/octet-stream');
17-
(0, chai_1.expect)(res.data.contentType).to.be.equal('application/octet-stream');
17+
const { data } = await new AttachmentProcessor_1.AttachmentProcessor().uploadAttachment(file);
18+
(0, chai_1.expect)(data.contentType).to.be.equal('application/json');
1819
});
1920
it('uploadAttachment (/samples/image.png)', async () => {
2021
const file = fs_1.default.createReadStream(path_1.default.join(__dirname, './samples/image.png'));
21-
const res = await new AttachmentProcessor_1.AttachmentProcessor().uploadAttachment(file, 'image/png');
22-
(0, chai_1.expect)(res.data.contentType).to.be.equal('image/png');
22+
const { data } = await new AttachmentProcessor_1.AttachmentProcessor().uploadAttachment(file);
23+
(0, chai_1.expect)(data.contentType).to.be.equal('image/png');
2324
});
2425
});
2526
});

‎dist/spec-integration/common.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,8 @@ const setEnvs = () => {
77
if ((0, fs_1.existsSync)('.env')) {
88
(0, dotenv_1.config)();
99
}
10+
else {
11+
throw new Error('.env doesn\'t exist!');
12+
}
1013
};
1114
exports.setEnvs = setEnvs;

‎dist/src/attachment/AttachmentProcessor.js

Lines changed: 64 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,35 @@
11
"use strict";
2+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3+
if (k2 === undefined) k2 = k;
4+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
5+
}) : (function(o, m, k, k2) {
6+
if (k2 === undefined) k2 = k;
7+
o[k2] = m[k];
8+
}));
9+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
10+
Object.defineProperty(o, "default", { enumerable: true, value: v });
11+
}) : function(o, v) {
12+
o["default"] = v;
13+
});
14+
var __importStar = (this && this.__importStar) || function (mod) {
15+
if (mod && mod.__esModule) return mod;
16+
var result = {};
17+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
18+
__setModuleDefault(result, mod);
19+
return result;
20+
};
221
var __importDefault = (this && this.__importDefault) || function (mod) {
322
return (mod && mod.__esModule) ? mod : { "default": mod };
423
};
524
Object.defineProperty(exports, "__esModule", { value: true });
625
exports.AttachmentProcessor = exports.MAESTER_OBJECT_ID_ENDPOINT = exports.DEFAULT_STORAGE_TYPE = exports.STORAGE_TYPE_PARAMETER = void 0;
726
/* eslint-disable class-methods-use-this */
8-
const axios_1 = __importDefault(require("axios"));
27+
const axios_1 = __importStar(require("axios"));
928
const url_1 = require("url");
1029
const dist_1 = require("@elastic.io/maester-client/dist");
30+
const form_data_1 = __importDefault(require("form-data"));
31+
const logger_1 = require("../logger/logger");
32+
const logger = (0, logger_1.getLogger)();
1133
exports.STORAGE_TYPE_PARAMETER = 'storage_type';
1234
exports.DEFAULT_STORAGE_TYPE = 'steward';
1335
exports.MAESTER_OBJECT_ID_ENDPOINT = '/objects/';
@@ -16,7 +38,7 @@ const maesterCreds = { jwtSecret: ELASTICIO_OBJECT_STORAGE_TOKEN, uri: ELASTICIO
1638
const REQUEST_TIMEOUT = process.env.REQUEST_TIMEOUT ? parseInt(process.env.REQUEST_TIMEOUT, 10) : 10000; // 10s
1739
const REQUEST_MAX_RETRY = process.env.REQUEST_MAX_RETRY ? parseInt(process.env.REQUEST_MAX_RETRY, 10) : 7; // 10s
1840
const REQUEST_RETRY_DELAY = process.env.REQUEST_RETRY_DELAY ? parseInt(process.env.REQUEST_RETRY_DELAY, 10) : 7000; // 7s
19-
const REQUEST_MAX_BODY_LENGTH = process.env.REQUEST_MAX_BODY_LENGTH ? parseInt(process.env.REQUEST_MAX_BODY_LENGTH, 10) : 104857600; // 100MB
41+
const axiosCriticalErrors = []; // errors that couldn't be retried
2042
class AttachmentProcessor {
2143
async getAttachment(url, responseType) {
2244
const storageType = AttachmentProcessor.getStorageTypeByUrl(url);
@@ -34,24 +56,9 @@ class AttachmentProcessor {
3456
default: throw new Error(`Storage type "${storageType}" is not supported`);
3557
}
3658
}
37-
async uploadAttachment(body, contentType) {
38-
const ax = axios_1.default.create();
39-
AttachmentProcessor.addRetryCountInterceptorToAxios(ax);
40-
const url = `${ELASTICIO_OBJECT_STORAGE_URI}${exports.MAESTER_OBJECT_ID_ENDPOINT}`;
41-
const axConfig = {
42-
url,
43-
data: body,
44-
method: 'post',
45-
headers: {
46-
Authorization: `Bearer ${ELASTICIO_OBJECT_STORAGE_TOKEN}`,
47-
'Content-Type': contentType,
48-
},
49-
timeout: REQUEST_TIMEOUT,
50-
retry: REQUEST_MAX_RETRY,
51-
delay: REQUEST_RETRY_DELAY,
52-
maxBodyLength: REQUEST_MAX_BODY_LENGTH,
53-
};
54-
return ax(axConfig);
59+
async uploadAttachment(body) {
60+
logger.debug('Start uploading attachment');
61+
return axiosUploadAttachment(body);
5562
}
5663
static async getStewardAttachment(axConfig) {
5764
const ax = axios_1.default.create();
@@ -94,3 +101,40 @@ class AttachmentProcessor {
94101
}
95102
}
96103
exports.AttachmentProcessor = AttachmentProcessor;
104+
// uploads attachment to "Maester" and applies request-retry logic
105+
const axiosUploadAttachment = async (body, currentRetryCount = 0) => {
106+
var _a;
107+
const data = new form_data_1.default();
108+
data.append('file', body);
109+
const config = {
110+
method: 'post',
111+
url: `${ELASTICIO_OBJECT_STORAGE_URI}${exports.MAESTER_OBJECT_ID_ENDPOINT}`,
112+
headers: {
113+
Authorization: `Bearer ${ELASTICIO_OBJECT_STORAGE_TOKEN}`,
114+
...data.getHeaders()
115+
},
116+
maxRedirects: 0,
117+
data
118+
};
119+
try {
120+
const resp = await (0, axios_1.default)(config);
121+
return resp;
122+
}
123+
catch (error) {
124+
logger.error(`Error occurred: ${((_a = error.response) === null || _a === void 0 ? void 0 : _a.data) || error.message}`);
125+
if (error instanceof axios_1.AxiosError) {
126+
const errorCouldNotBeRetried = axiosCriticalErrors.includes(error.code);
127+
if (errorCouldNotBeRetried)
128+
throw error;
129+
}
130+
if (currentRetryCount + 1 <= REQUEST_MAX_RETRY) {
131+
logger.debug(`Start retrying #${currentRetryCount + 1}`);
132+
await sleep(REQUEST_RETRY_DELAY);
133+
return axiosUploadAttachment(body, currentRetryCount + 1);
134+
}
135+
throw error;
136+
}
137+
};
138+
const sleep = async (ms) => new Promise((resolve) => {
139+
setTimeout(resolve, ms);
140+
});

‎package-lock.json

Lines changed: 34 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@elastic.io/component-commons-library",
3-
"version": "2.0.2",
3+
"version": "2.1.0",
44
"description": "Library for most common component development cases",
55
"author": {
66
"name": "elastic.io GmbH",
@@ -22,9 +22,9 @@
2222
"lint": "eslint --ext .ts --quiet --fix",
2323
"pretest": "rm -rf dist && eslint --ext .ts --quiet --fix && find src spec spec-integration -name \"*.js\" -type f -delete",
2424
"posttest": "tsc",
25-
"test": "nyc mocha --require ts-node/register \"spec/**/*spec.ts\" --timeout 10000",
25+
"test": "nyc mocha --require ts-node/register \"spec/**/*spec.ts\" --timeout 50000",
2626
"integration-test": "npm run pretest && nyc mocha --require ts-node/register \"spec-integration/**/*spec.ts\" --timeout 500000",
27-
"build": "rm -rf dist tsc"
27+
"build": "rm -rf dist && tsc"
2828
},
2929
"engines": {
3030
"node": ">=12"
@@ -34,11 +34,12 @@
3434
"@elastic.io/maester-client": "3.4.3",
3535
"@elastic.io/ntlm-client": "1.0.0",
3636
"async": "3.2.3",
37-
"axios": "0.24.0",
37+
"axios": "0.27.2",
3838
"bunyan": "1.8.14",
3939
"bunyan-format": "0.2.1",
4040
"bunyan-serializers": "0.0.2",
4141
"elasticio-node": "0.0.9",
42+
"form-data": "4.0.0",
4243
"remove-leading-slash": "1.0.1",
4344
"remove-trailing-slash": "0.1.1",
4445
"request": "2.88.2"
@@ -67,4 +68,4 @@
6768
},
6869
"repository": "elasticio/component-commons-library",
6970
"license": "Apache-2.0"
70-
}
71+
}

‎spec-integration/attachmentProcessor.spec.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,23 @@ import fs from 'fs';
22
import path from 'path';
33
import { expect } from 'chai';
44
import { setEnvs } from './common';
5-
import { AttachmentProcessor } from '../src/attachment/AttachmentProcessor';
65

76
setEnvs();
87

8+
// eslint-disable-next-line import/first
9+
import { AttachmentProcessor } from '../src/attachment/AttachmentProcessor';
10+
911
describe('AttachmentProcessor', () => {
1012
describe('uploadAttachment', () => {
1113
it('uploadAttachment (/samples/sample.json)', async () => {
1214
const file = fs.createReadStream(path.join(__dirname, './samples/sample.json'));
13-
const res = await new AttachmentProcessor().uploadAttachment(file, 'application/octet-stream');
14-
expect(res.data.contentType).to.be.equal('application/octet-stream');
15+
const { data } = await new AttachmentProcessor().uploadAttachment(file);
16+
expect(data.contentType).to.be.equal('application/json');
1517
});
1618
it('uploadAttachment (/samples/image.png)', async () => {
1719
const file = fs.createReadStream(path.join(__dirname, './samples/image.png'));
18-
const res = await new AttachmentProcessor().uploadAttachment(file, 'image/png');
19-
expect(res.data.contentType).to.be.equal('image/png');
20+
const { data } = await new AttachmentProcessor().uploadAttachment(file);
21+
expect(data.contentType).to.be.equal('image/png');
2022
});
2123
});
2224
});

‎spec-integration/common.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,7 @@ import { config } from 'dotenv';
44
export const setEnvs = () => {
55
if (existsSync('.env')) {
66
config();
7+
} else {
8+
throw new Error('.env doesn\'t exist!');
79
}
810
};

0 commit comments

Comments
 (0)