Skip to content

Commit 31b2664

Browse files
guyromellemagayanoguyromellemagayano
committed
fix: additional fixes for caching features
1 parent 8225a01 commit 31b2664

File tree

3 files changed

+107
-74
lines changed

3 files changed

+107
-74
lines changed

‎src/gatsby-node.js‎

Lines changed: 81 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { ACCESS_CONTROL_ALLOW_CREDENTIALS, ACCESS_CONTROL_ALLOW_HEADERS, CORS_ORIGIN, REQUEST_TIMEOUT, REQUEST_URL_SLUG } from "./constants";
44
import Auth from "./utils/auth";
5-
import { convertObjectToString } from "./utils/convertValues";
5+
import { convertObjectToString, convertStringToLowercase } from "./utils/convertValues";
66
import { logger } from "./utils/logger";
77
import { Optimizely } from "./utils/optimizely";
88

@@ -20,7 +20,7 @@ const handleCreateNodeFromData = async (item, nodeType, helpers, endpoint, log)
2020

2121
const nodeMetadata = {
2222
...item,
23-
id: createNodeId(`${nodeType}-${item?.id || item?.contentLink?.id || item?.name || null}`),
23+
id: createNodeId(`${nodeType}-${endpoint}`),
2424
parent: null,
2525
children: [],
2626
internal: {
@@ -34,12 +34,12 @@ const handleCreateNodeFromData = async (item, nodeType, helpers, endpoint, log)
3434

3535
await createNode(node)
3636
.then(() => {
37-
log.http(`[CREATE NODE] ${endpoint} - ${createNodeId(`${nodeType}-${item.id || item.name}`)} (OK)`);
37+
log.http(`[NODE] ${endpoint} - ${createNodeId(`${nodeType}-${endpoint}`)} (OK)`);
3838

3939
return Promise.resolve(node);
4040
})
4141
.catch((err) => {
42-
log.error(`[CREATE NODE] ${endpoint} - ${createNodeId(`${nodeType}-${item.id || item.name}`)} (FAIL)`);
42+
log.error(`[NODE] ${endpoint} - ${createNodeId(`${nodeType}-${endpoint}`)} (FAIL)`);
4343

4444
return Promise.reject(err);
4545
});
@@ -108,7 +108,7 @@ exports.onPreInit = () => console.info("@epicdesignlabs/gatsby-source-optimizely
108108
* @param {Object} pluginOptions
109109
* @returns {Promise<void>} Node creation promise
110110
*/
111-
exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }, pluginOptions) => {
111+
exports.sourceNodes = async ({ actions, cache, createNodeId, createContentDigest }, pluginOptions) => {
112112
// Prepare plugin options
113113
const {
114114
auth: { site_url = null, username = null, password = null, grant_type = "password", client_id = "Default", headers = {} },
@@ -127,46 +127,84 @@ exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }, plu
127127
createNodeId
128128
});
129129

130-
// Authenticate with the Optimizely/Episerver
131-
const auth = new Auth({ site_url, username, password, grant_type, client_id, log, response_type, request_timeout });
132-
133-
const authData = await auth.post();
134-
135-
// Display the auth data
136-
log.info(`(OK) [AUTH] ${convertObjectToString(authData)}`);
137-
138-
// Create a new Optimizely instance
139-
const optimizely = new Optimizely({
140-
site_url,
141-
response_type,
142-
headers: Object.assign(headers, {
143-
"Authorization": `Bearer ${authData.access_token}`,
144-
"Access-Control-Allow-Headers": ACCESS_CONTROL_ALLOW_HEADERS,
145-
"Access-Control-Allow-Credentials": ACCESS_CONTROL_ALLOW_CREDENTIALS,
146-
"Access-Control-Allow-Origin": CORS_ORIGIN
147-
}),
148-
log,
149-
request_timeout
150-
});
130+
// Store the response from the Optimizely/Episerver API in the cache
131+
const cacheKey = "gatsby-source-optimizely-api-data-key";
132+
const nodes = [];
133+
134+
let sourceData = await cache.get(cacheKey);
135+
136+
// If the data is not cached, fetch it from the Optimizely/Episerver API
137+
if (!sourceData) {
138+
// Authenticate with the Optimizely/Episerver
139+
const auth = new Auth({ site_url, username, password, grant_type, client_id, log, response_type, request_timeout });
140+
141+
const authData = await auth.post();
142+
143+
// Display the auth data
144+
authData?.access_token ? log.info(`[AUTH] ${authData.access_token} - OK`) : log.error(`[AUTH] ${authData.access_token} - FAIL`);
145+
146+
// Create a new Optimizely instance
147+
const optimizely = new Optimizely({
148+
site_url,
149+
response_type,
150+
headers: Object.assign(headers, {
151+
"Authorization": `Bearer ${authData.access_token}`,
152+
"Access-Control-Allow-Headers": ACCESS_CONTROL_ALLOW_HEADERS,
153+
"Access-Control-Allow-Credentials": ACCESS_CONTROL_ALLOW_CREDENTIALS,
154+
"Access-Control-Allow-Origin": CORS_ORIGIN
155+
}),
156+
log,
157+
request_timeout
158+
});
159+
160+
for (const [nodeName, endpoint] of Object.entries(endpoints)) {
161+
await optimizely
162+
.get(endpoint)
163+
.then(async (res) => {
164+
nodes.push({
165+
nodeName,
166+
data: res
167+
});
168+
169+
// Create nodes for each item in the response
170+
return res && Array.isArray(res) && res.length > 0
171+
? await Promise.allSettled(res.map(async (datum) => Promise.resolve(await handleCreateNodeFromData(datum, nodeName, helpers, convertStringToLowercase(datum.contentLink.url), log))))
172+
: Promise.resolve(await handleCreateNodeFromData(res, nodeName, helpers, endpoint, log));
173+
})
174+
.catch((err) => {
175+
log.error(`[GET] ${site_url + REQUEST_URL_SLUG + endpoint} (${err.status + " " + err.statusText})`);
151176

152-
for (const [nodeName, endpoint] of Object.entries(endpoints)) {
153-
try {
154-
const res = await optimizely.get(endpoint);
155-
156-
if (res && Array.isArray(res) && res?.length > 0) {
157-
await Promise.allSettled(
158-
res.map(async (datum) => {
159-
await handleCreateNodeFromData(datum, nodeName, helpers, site_url + REQUEST_URL_SLUG + endpoint, log);
160-
})
161-
);
162-
} else {
163-
await handleCreateNodeFromData(res, nodeName, helpers, site_url + REQUEST_URL_SLUG + endpoint, log);
164-
}
165-
} catch (error) {
166-
console.log(error);
167-
168-
log.error(`An error occurred while fetching ${endpoint} endpoint data`, error.message);
177+
return Promise.reject(err);
178+
});
169179
}
180+
181+
// Store the response from the Optimizely/Episerver API in the cache
182+
sourceData = nodes.length > 0 ? nodes : sourceData;
183+
184+
// Cache the data
185+
await cache
186+
.set(cacheKey, sourceData)
187+
.then(() => {
188+
// If the data is cached, display a success message
189+
log.info(`[CACHE] ${cacheKey} (OK) - ${sourceData?.length || 0} items`);
190+
191+
return Promise.resolve();
192+
})
193+
.catch((err) => {
194+
// If the data is not cached, display an error message
195+
log.error(`[CACHE] ${cacheKey} (FAIL)`);
196+
197+
return Promise.reject(err);
198+
});
199+
} else {
200+
// Create nodes for each item in the cached response
201+
sourceData?.map(({ nodeName = "", data = [] }) =>
202+
data?.map(async (item) =>
203+
item && Array.isArray(item) && item.length > 0
204+
? await Promise.allSettled(item.map(async (datum) => Promise.resolve(await handleCreateNodeFromData(datum, nodeName, helpers, convertStringToLowercase(datum.contentLink.url), log))))
205+
: Promise.resolve(await handleCreateNodeFromData(item, nodeName, helpers, convertStringToLowercase(item.contentLink.url), log))
206+
)
207+
) || null;
170208
}
171209

172210
log.info("@epicdesignlabs/gatsby-source-optimizely task processing complete!");

‎src/utils/optimizely.js‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ export class Optimizely {
2828
});
2929

3030
// Run request
31-
const response = sleep(REQUEST_THROTTLE_INTERVAL).then(async () => await request.run(method, path, body));
31+
const response = sleep(REQUEST_THROTTLE_INTERVAL)
32+
.then(async () => await request.run(method, path, body))
33+
.catch((err) => err);
3234

3335
// Handle expanded keys and values
3436
const handleExpandedKeyValues = async (values, label) => {
@@ -38,7 +40,9 @@ export class Optimizely {
3840
values.map(async (value) => {
3941
if (value && Object.prototype.hasOwnProperty.call(value, "contentLink") && Object.keys(value)?.length > 0) {
4042
if (typeof value.contentLink.id === "number") {
41-
const response = sleep(REQUEST_THROTTLE_INTERVAL).then(async () => await request.run("get", CONTENT_ENDPOINT + value.contentLink.id + "?expand=*", body));
43+
const response = sleep(REQUEST_THROTTLE_INTERVAL)
44+
.then(async () => await request.run("get", CONTENT_ENDPOINT + value.contentLink.id + "?expand=*", body))
45+
.catch((err) => err);
4246

4347
promises.push(response);
4448
} else {
@@ -54,7 +58,9 @@ export class Optimizely {
5458
let promises = [];
5559

5660
if (values && Object.prototype.hasOwnProperty.call(values, "id") && Object.keys(values)?.length > 0) {
57-
const response = sleep(REQUEST_THROTTLE_INTERVAL).then(async () => await request.run("get", CONTENT_ENDPOINT + values.id + "?expand=*", body));
61+
const response = sleep(REQUEST_THROTTLE_INTERVAL)
62+
.then(async () => await request.run("get", CONTENT_ENDPOINT + values.id + "?expand=*", body))
63+
.catch((err) => err);
5864

5965
promises.push(response);
6066
} else {

‎src/utils/request.js‎

Lines changed: 17 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"use strict";
22

33
import axios from "axios";
4-
import { REQUEST_ACCEPT_HEADER, REQUEST_MAX_COUNT, REQUEST_THROTTLE_INTERVAL, REQUEST_URL_SLUG } from "../constants";
4+
import { REQUEST_ACCEPT_HEADER, REQUEST_URL_SLUG } from "../constants";
55
import { convertStringToUppercase } from "./convertValues";
66

77
export class Request {
@@ -11,7 +11,6 @@ export class Request {
1111
this.response_type = response_type;
1212
this.log = log;
1313
this.request_timeout = request_timeout;
14-
this.pending_requests = 0;
1514
}
1615

1716
/**
@@ -36,50 +35,40 @@ export class Request {
3635

3736
// Use `axios` interceptors for all HTTP methods (GET, POST, PUT, DELETE, etc.)
3837
RequestAxiosInstance.interceptors.request.use(
39-
(config) =>
40-
new Promise((resolve) => {
41-
this.log.warn(`[${method.toUpperCase()}] ${this.hostname + path} - Request sent`);
42-
43-
let requestInterval = setInterval(() => {
44-
if (this.pending_requests < REQUEST_MAX_COUNT) {
45-
this.pending_requests++;
46-
clearInterval(requestInterval);
47-
resolve(config);
48-
}
49-
}, REQUEST_THROTTLE_INTERVAL);
50-
})
38+
(config) => {
39+
this.log.warn(`[${method.toUpperCase()}] ${this.hostname + REQUEST_URL_SLUG + path} - Request sent`);
40+
return config;
41+
},
42+
(res) => {
43+
this.log.info(`[${method.toUpperCase()}] ${this.hostname + REQUEST_URL_SLUG + path} - Response received`);
44+
return Promise.resolve(res);
45+
},
46+
(err) => {
47+
this.log.error(`[${method.toUpperCase()}] ${this.hostname + REQUEST_URL_SLUG + path} - Request failed`);
48+
return Promise.reject(err);
49+
}
5150
);
5251

5352
// Use `axios` interceptors for all HTTP methods (GET, POST, PUT, DELETE, etc.)
5453
RequestAxiosInstance.interceptors.response.use(
5554
(res) => {
5655
// Send log message when endpoint request is successful
57-
this.log.info(
58-
`[${convertStringToUppercase(method)}] ${this.hostname + path} (${res.status + " " + res.statusText}) - ${
59-
res && Array.isArray(res) && res.length > 0 ? res.length : res && Object.prototype.toString.call(res) === "[object Object]" && Object.keys(res).length > 0 ? Object.keys(res).length : 0
60-
} items found`
61-
);
62-
63-
// Decrement pending requests
64-
this.pending_requests = Math.max(0, this.pending_requests - 1);
56+
this.log.info(`[${convertStringToUppercase(method)}] ${this.hostname + REQUEST_URL_SLUG + path} (${res.status + " " + res.statusText})`);
6557

6658
return Promise.resolve(res);
6759
},
6860
(err) => {
6961
if (err.response) {
7062
// Send log message when error response is received
71-
this.log.error(`[${convertStringToUppercase(method)}] ${this.hostname + path} (${err.response.status + " " + err.response.statusText}). Restarting request...`);
63+
this.log.error(`[${convertStringToUppercase(method)}] ${this.hostname + REQUEST_URL_SLUG + path} - Restarting request...`);
7264
} else if (err.request) {
7365
// Send log message when error request is received
74-
this.log.error(`[${convertStringToUppercase(method)}] ${this.hostname + path} (${err.request.status + " " + err.request.statusText}). Restarting request...`);
66+
this.log.error(`[${convertStringToUppercase(method)}] ${this.hostname + REQUEST_URL_SLUG + path} - Restarting request...`);
7567
} else {
7668
// Send log message when error is thrown
77-
this.log.error(`[${convertStringToUppercase(method)}] ${this.hostname + path} - ${err.message}`);
69+
this.log.error(`[${convertStringToUppercase(method)}] ${this.hostname + REQUEST_URL_SLUG + path} - ${err.message}`);
7870
}
7971

80-
// Decrement pending requests
81-
this.pending_requests = Math.max(0, this.pending_requests - 1);
82-
8372
return Promise.reject(err);
8473
}
8574
);

0 commit comments

Comments
 (0)