-
Notifications
You must be signed in to change notification settings - Fork 0
/
handler.js
45 lines (40 loc) · 1.06 KB
/
handler.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
const aws = require("aws-sdk");
const axios = require("axios");
const s3Client = new aws.S3({
region: process.env.BUCKET_REGION,
credentials: {
accessKeyId: process.env.BUCKET_ACCESS_KEY,
secretAccessKey: process.env.BUCKET_SECRET_KEY
}
});
exports.handler = async ({ scrapeUrl, timeout = 5000 }) => {
try {
const res = await axios(scrapeUrl, { method: "GET", timeout });
if (res.data) {
console.log("Data scraped successfully.");
await saveData({ data: JSON.stringify(res.data) });
}
} catch (error) {
console.error("Unable to scrape data.");
throw error;
}
}
/**
* Save and handle the JSON dataset appropriately.
* (S3 File or Database etc)
*/
const saveData = async ({ data }) => {
try {
const params = {
Bucket: process.env.BUCKET_NAME,
Key: "data.json",
Body: data,
ContentType: "application/json"
};
await s3Client.putObject(params).promise();
console.log("Data uploaded successfully.");
} catch (error) {
console.error("Unable to upload data.");
throw error;
}
}