Skip to content

Commit 45fc1b4

Browse files
guyromellemagayanoguyromellemagayano
committed
fix: fix rate limit issues within the plugin
1 parent 270e6e7 commit 45fc1b4

File tree

7 files changed

+220
-231
lines changed

7 files changed

+220
-231
lines changed

‎README.md‎

Lines changed: 85 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@ This unofficial source plugin makes Optimizely/Episerver API data available in G
1818
## Features
1919

2020
- Support for multiple `optimizely/episerver` API versions
21-
- Log level options for `optimizely/episerver` API endpoint requests: `info`, `debug`, `warn`, `error`
22-
- Support for additional headers
21+
- Log level options for `optimizely/episerver` API endpoint requests: `error`, `warn`, `info`, `http`, `verbose`, `debug`, `silly`
22+
- Support for multiple, additional custom **headers**
2323
- Support for custom request timeout in all `optimizely/episerver` API requests
24+
- Support for data caching on subsequent `gatsby` source plugin runs
25+
- Add support for `expanded` data on content blocks _(`images`, `dynamicStyles`, `items`, `form` object data are currently supported with more to come in the future)_
2426

2527
## Installation and Setup
2628

@@ -56,7 +58,11 @@ module.exports = {
5658
client_id: process.env.OPTMIZELY_API_CLIENT_ID, // The client ID of the Optimizely/Episerver API user. Default is "Default"
5759
},
5860
endpoints: {
59-
OptimizelySites: "/v2.0/site",
61+
OptimizelyAboutUsDesignersPageContentChildren: "/v2.0/content/14675/children?expand=*",
62+
OptimizelyAboutUsPageContentChildren: "/v2.0/content/14110/children?expand=*",
63+
OptimizelyBedAccessoriesHeadboardsPageContentChildren: "/v2.0/content/14129/children?expand=*",
64+
OptimizelyBedAccessoriesLegsPageContentChildren: "/v2.0/content/14131/children?expand=*",
65+
6066
},
6167
},
6268
},
@@ -76,12 +82,12 @@ options: {
7682

7783
endpoints: {
7884
// Single endpoint
79-
OptimizelySites: "/v2.0/site",
85+
OptimizelyAboutUsDesignersPageContentChildren: "/v2.0/content/14675/children?expand=*",
8086

8187
// Multiple endpoints
82-
OptimizelyContent: "/v2.0/content/994?expand=*",
83-
OptimizelyContent: "/v2.0/content/994/children?expand=*",
84-
OptimizelySearchContent: "/v2.0/search/content?expand=*",
88+
OptimizelyAboutUsPageContentChildren: "/v2.0/content/14110/children?expand=*",
89+
OptimizelyBedAccessoriesHeadboardsPageContentChildren: "/v2.0/content/14129/children?expand=*",
90+
OptimizelyBedAccessoriesLegsPageContentChildren: "/v2.0/content/14131/children?expand=*",
8591
}
8692
}
8793
```
@@ -127,13 +133,13 @@ options: {
127133

128134
Set a custom request timeout for the Optimizely/Episerver API requests (in milliseconds).
129135

130-
**Default:** `10000`.
136+
**Default:** `30000`.
131137

132138
```javascript
133139
options: {
134140
// ...
135141

136-
request_timeout: 10000;
142+
request_timeout: 30000;
137143
}
138144
```
139145

@@ -146,7 +152,7 @@ options: {
146152
// ...
147153

148154
endpoints: {
149-
OptimizelyContent: "/v2.0/content/994?expand=*";
155+
OptimizelyAboutUsDesignersPageContentChildren: "/v2.0/content/14675/children?expand=*",
150156
}
151157
}
152158
```
@@ -155,23 +161,31 @@ you can query the data as follows:
155161

156162
```graphql
157163
{
158-
allOptimizelyContent(limit: 1) {
164+
allOptimizelyAboutUsDesignersPageContentChildren(filter: { status: { eq: "Published" } }) {
159165
edges {
160166
node {
167+
id
168+
name
169+
contentLink {
170+
id
171+
url
172+
}
173+
contentType
174+
language {
175+
displayName
176+
link
177+
name
178+
}
179+
status
161180
contentBlocks {
181+
displayOption
162182
contentLink {
163-
id
164183
expanded {
165-
anchor1
166-
subHeading
167-
name
168-
altText1
169-
autoCrop
170184
body
171185
column1Body
172186
column1PrimaryCTA {
173-
target
174187
text
188+
target
175189
title
176190
url
177191
}
@@ -193,93 +207,86 @@ you can query the data as follows:
193207
title
194208
url
195209
}
210+
contentLink {
211+
id
212+
}
196213
contentType
197214
disableImageZoom
198-
displayFilter
199-
215+
dynamicStyles {
216+
contentLink {
217+
id
218+
}
219+
themeColor
220+
}
200221
eyeBrow
201-
fourColumnDisplay
202-
fullBleed
203222
heading
204223
headingH1
205-
height
206-
image {
207-
url
208-
guidValue
209-
id
210-
workId
211-
}
212-
image1 {
213-
guidValue
214-
id
215-
url
216-
workId
217-
}
218-
imageRatio
219-
images {
220-
displayOption
221-
}
222224
items {
223-
displayOption
225+
contentLink {
226+
id
227+
}
228+
body
229+
contentType
230+
eyeBrow
231+
heading
232+
image {
233+
id
234+
url
235+
expanded {
236+
contentLink {
237+
id
238+
url
239+
}
240+
contentType
241+
height
242+
name
243+
status
244+
url
245+
width
246+
size
247+
}
248+
}
249+
link {
250+
target
251+
text
252+
title
253+
url
254+
}
255+
name
256+
status
257+
parentLink {
258+
id
259+
url
260+
}
224261
}
225-
layout
226-
link {
227-
target
228-
text
229-
title
230-
url
231-
}
232-
logo {
233-
guidValue
234-
id
235-
url
236-
workId
237-
}
238-
logoAltText
239-
maxCount
240-
mediaTextColor
241-
noAutoPlay
242-
orientation
243-
parentPage {
244-
guidValue
245-
id
246-
url
247-
workId
262+
language {
263+
displayName
264+
name
248265
}
266+
layout
249267
primaryCTA {
250-
target
251268
text
252-
title
253269
url
270+
title
271+
target
254272
}
255-
recurse
256273
secondaryCTA {
257274
target
258275
text
259276
title
260277
url
261278
}
279+
status
262280
style
263-
textAlign
264281
textPosition
265282
textPosition2
266-
themeColor
267-
useEpiSort
268-
usePrimaryLinkStyle
269-
video1 {
270-
guidValue
283+
image1 {
271284
id
272285
url
273-
workId
274286
}
275287
}
276-
guidValue
277-
workId
278288
}
279-
displayOption
280289
}
281-
metaDescription
282-
metaTitle
283290
}
284291
}
285292
}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
"gatsby-source-filesystem": "^4.21.1",
4141
"micro": "^9.4.1",
4242
"qs": "^6.11.0",
43+
"sleep-promise": "^9.1.0",
4344
"winston": "^3.8.1"
4445
},
4546
"devDependencies": {

‎src/constants/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ export const IS_PROD = process.env.NODE_ENV === "production";
66
export const REQUEST_URL_SLUG = "/api/episerver";
77
export const REQUEST_ACCEPT_HEADER = "application/json";
88
export const REQUEST_TIMEOUT = 30000;
9+
export const REQUEST_THROTTLE_INTERVAL = 1500;
10+
export const REQUEST_MAX_COUNT = 5;
911

1012
// Auth
1113
export const AUTH_REQUEST_CONTENT_TYPE_HEADER = "application/x-www-form-urlencoded";

‎src/gatsby-node.js‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
1-
/* eslint-disable prettier/prettier */
21
"use strict";
32

43
import { ACCESS_CONTROL_ALLOW_CREDENTIALS, ACCESS_CONTROL_ALLOW_HEADERS, CORS_ORIGIN, REQUEST_TIMEOUT, REQUEST_URL_SLUG } from "./constants";
54
import Auth from "./utils/auth";
65
import { convertObjectToString } from "./utils/convertValues";
76
import { logger } from "./utils/logger";
8-
import Optimizely from "./utils/optimizely";
7+
import { Optimizely } from "./utils/optimizely";
98

109
/**
1110
* @description Create a node from the data
@@ -17,29 +16,33 @@ import Optimizely from "./utils/optimizely";
1716
* @returns {Promise<void>} Node creation promise
1817
*/
1918
const handleCreateNodeFromData = async (item, nodeType, helpers, endpoint, log) => {
19+
const { createNode, createNodeId, createContentDigest } = helpers;
20+
2021
const nodeMetadata = {
2122
...item,
22-
id: helpers.createNodeId(`${nodeType}-${item?.id || item?.name}`),
23+
id: createNodeId(`${nodeType}-${item?.id || item?.contentLink?.id || item?.name || null}`),
2324
parent: null,
2425
children: [],
2526
internal: {
2627
type: nodeType,
2728
content: convertObjectToString(item),
28-
contentDigest: helpers.createContentDigest(item)
29+
contentDigest: createContentDigest(item)
2930
}
3031
};
3132

3233
const node = Object.assign({}, item, nodeMetadata);
3334

34-
try {
35-
await helpers.createNode(node);
35+
await createNode(node)
36+
.then(() => {
37+
log.http(`[CREATE NODE] ${endpoint} - ${createNodeId(`${nodeType}-${item.id || item.name}`)} (OK)`);
3638

37-
log.warn(`(OK) [CREATE NODE] ${endpoint} - ${helpers.createNodeId(`${nodeType}-${item.id || item.name}`)}`);
39+
return Promise.resolve(node);
40+
})
41+
.catch((err) => {
42+
log.error(`[CREATE NODE] ${endpoint} - ${createNodeId(`${nodeType}-${item.id || item.name}`)} (FAIL)`);
3843

39-
return node;
40-
} catch (error) {
41-
log.error(`(FAIL) [CREATE NODE] ${endpoint} - ${helpers.createNodeId(`${nodeType}-${item.id || item.name}`)}`, error.message);
42-
}
44+
return Promise.reject(err);
45+
});
4346
};
4447

4548
/**
@@ -86,7 +89,7 @@ exports.pluginOptionsSchema = ({ Joi }) =>
8689
"object.required": "The `endpoints` object is required."
8790
})
8891
.description("The endpoints to create nodes for"),
89-
log_level: Joi.string().default("info").description("The log level to use"),
92+
log_level: Joi.string().default("debug").description("The log level to use"),
9093
response_type: Joi.string().default("json").description("The response type to use"),
9194
request_timeout: Joi.number().default(REQUEST_TIMEOUT).description("The request timeout to use in milliseconds")
9295
});
@@ -98,7 +101,7 @@ exports.pluginOptionsSchema = ({ Joi }) =>
98101
exports.onPreInit = () => console.info("@epicdesignlabs/gatsby-source-optimizely loaded successfully!");
99102

100103
/**
101-
* @description Source nodes from the Optimizely/Episerver API
104+
* @description Source and cache nodes from the Optimizely/Episerver API
102105
* @param {Object} actions
103106
* @param {Object} createNodeId
104107
* @param {Object} createContentDigest
@@ -110,7 +113,7 @@ exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }, plu
110113
const {
111114
auth: { site_url = null, username = null, password = null, grant_type = "password", client_id = "Default", headers = {} },
112115
endpoints = null,
113-
log_level = "info",
116+
log_level = "debug",
114117
response_type = "json",
115118
request_timeout = REQUEST_TIMEOUT
116119
} = pluginOptions;
@@ -149,6 +152,7 @@ exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }, plu
149152
for (const [nodeName, endpoint] of Object.entries(endpoints)) {
150153
try {
151154
const res = await optimizely.get(endpoint);
155+
152156
if (res && Array.isArray(res) && res?.length > 0) {
153157
await Promise.allSettled(
154158
res.map(async (datum) => {
@@ -159,6 +163,8 @@ exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }, plu
159163
await handleCreateNodeFromData(res, nodeName, helpers, site_url + REQUEST_URL_SLUG + endpoint, log);
160164
}
161165
} catch (error) {
166+
console.log(error);
167+
162168
log.error(`An error occurred while fetching ${endpoint} endpoint data`, error.message);
163169
}
164170
}

‎src/utils/convertValues.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ export const convertStringToObject = (e) => (typeof e === "string" ? JSON.parse(
9797
* @param {object} e
9898
* @returns {string} String
9999
*/
100-
export const convertObjectToString = (e) => (Object.prototype.toString.call(e) === "[object Object]" ? JSON.stringify(e, getCircularReplacer(), 2) : e);
100+
export const convertObjectToString = (e) => (Object.prototype.toString.call(e) === "[object Object]" ? JSON.stringify(e, getCircularReplacer()) : e);
101101

102102
/**
103103
* @description Convert number to string

0 commit comments

Comments
 (0)