Skip to content

Commit c9315d2

Browse files
piehpieh
committed
fix(gatsby-source-contentful): maintain back reference map between runs
1 parent b0c0913 commit c9315d2

File tree

4 files changed

+103
-27
lines changed

4 files changed

+103
-27
lines changed

‎packages/gatsby-source-contentful/src/__tests__/gatsby-node.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,9 @@ describe(`gatsby-node`, () => {
534534
Array [
535535
"contentful-content-types-testSpaceId-master",
536536
],
537+
Array [
538+
"contentful-foreign-reference-map-state-testSpaceId-master",
539+
],
537540
]
538541
`)
539542

@@ -546,6 +549,7 @@ describe(`gatsby-node`, () => {
546549
expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(`
547550
Array [
548551
"contentful-content-types-testSpaceId-master",
552+
"contentful-foreign-reference-map-state-testSpaceId-master",
549553
]
550554
`)
551555
expect(actions.createNode).toHaveBeenCalledTimes(32)

‎packages/gatsby-source-contentful/src/__tests__/normalize.js‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,17 @@ describe(`generic`, () => {
101101
entryList,
102102
})
103103

104-
const foreignReferenceMap = buildForeignReferenceMap({
104+
const foreignReferenceMapState = buildForeignReferenceMap({
105105
contentTypeItems,
106106
entryList,
107107
resolvable,
108108
defaultLocale,
109109
space,
110110
useNameForId: true,
111+
previousForeignReferenceMapState: undefined,
112+
deletedEntries: [],
111113
})
112-
const referenceKeys = Object.keys(foreignReferenceMap)
114+
const referenceKeys = Object.keys(foreignReferenceMapState.backLinks)
113115
const expectedReferenceKeys = [
114116
`2Y8LhXLnYAYqKCGEWG4EKI___Asset`,
115117
`3wtvPBbBjiMKqKKga8I2Cu___Asset`,
@@ -129,7 +131,7 @@ describe(`generic`, () => {
129131
expect(referenceKeys).toHaveLength(expectedReferenceKeys.length)
130132
expect(referenceKeys).toEqual(expect.arrayContaining(expectedReferenceKeys))
131133

132-
Object.keys(foreignReferenceMap).forEach(referenceId => {
134+
Object.keys(foreignReferenceMapState.backLinks).forEach(referenceId => {
133135
expect(resolvable).toContain(referenceId)
134136

135137
let expectedLength = 1
@@ -139,7 +141,9 @@ describe(`generic`, () => {
139141
if (referenceId === `7LAnCobuuWYSqks6wAwY2a___Entry`) {
140142
expectedLength = 3
141143
}
142-
expect(foreignReferenceMap[referenceId]).toHaveLength(expectedLength)
144+
expect(foreignReferenceMapState.backLinks[referenceId]).toHaveLength(
145+
expectedLength
146+
)
143147
})
144148
})
145149
})
@@ -156,22 +160,26 @@ describe(`Process contentful data (by name)`, () => {
156160
entryList,
157161
})
158162

159-
const foreignReferenceMap = buildForeignReferenceMap({
163+
const foreignReferenceMapState = buildForeignReferenceMap({
160164
contentTypeItems,
161165
entryList,
162166
resolvable,
163167
defaultLocale,
164168
space,
165169
useNameForId: true,
170+
previousForeignReferenceMapState: undefined,
171+
deletedEntries: [],
166172
})
167173

168-
expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe(
169-
`product___NODE`
170-
)
174+
expect(
175+
foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0]
176+
.name
177+
).toBe(`product___NODE`)
171178

172-
expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe(
173-
`brand___NODE`
174-
)
179+
expect(
180+
foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0]
181+
.name
182+
).toBe(`brand___NODE`)
175183
})
176184

177185
it(`creates nodes for each entry`, () => {
@@ -192,6 +200,8 @@ describe(`Process contentful data (by name)`, () => {
192200
defaultLocale,
193201
space,
194202
useNameForId: true,
203+
previousForeignReferenceMapState: undefined,
204+
deletedEntries: [],
195205
})
196206

197207
const createNode = jest.fn()
@@ -291,6 +301,8 @@ describe(`Process existing mutated nodes in warm build`, () => {
291301
defaultLocale,
292302
space,
293303
useNameForId: true,
304+
previousForeignReferenceMapState: undefined,
305+
deletedEntries: [],
294306
})
295307

296308
const createNode = jest.fn()
@@ -377,22 +389,26 @@ describe(`Process contentful data (by id)`, () => {
377389
assets: currentSyncData.assets,
378390
entryList,
379391
})
380-
const foreignReferenceMap = buildForeignReferenceMap({
392+
const foreignReferenceMapState = buildForeignReferenceMap({
381393
contentTypeItems,
382394
entryList,
383395
resolvable,
384396
defaultLocale,
385397
space,
386398
useNameForId: false,
399+
previousForeignReferenceMapState: undefined,
400+
deletedEntries: [],
387401
})
388402

389-
expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe(
390-
`2pqfxujwe8qsykum0u6w8m___NODE`
391-
)
403+
expect(
404+
foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0]
405+
.name
406+
).toBe(`2pqfxujwe8qsykum0u6w8m___NODE`)
392407

393-
expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe(
394-
`sfztzbsum8coewygeuyes___NODE`
395-
)
408+
expect(
409+
foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0]
410+
.name
411+
).toBe(`sfztzbsum8coewygeuyes___NODE`)
396412
})
397413

398414
it(`creates nodes for each entry`, () => {
@@ -411,6 +427,8 @@ describe(`Process contentful data (by id)`, () => {
411427
defaultLocale,
412428
space,
413429
useNameForId: false,
430+
previousForeignReferenceMapState: undefined,
431+
deletedEntries: [],
414432
})
415433

416434
const createNode = jest.fn()

‎packages/gatsby-source-contentful/src/normalize.js‎

Lines changed: 53 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,45 @@ export const buildResolvableSet = ({
101101
return resolvable
102102
}
103103

104+
function cleanupReferencesFromEntry(foreignReferenceMapState, entry) {
105+
const { links, backLinks } = foreignReferenceMapState
106+
const entryId = entry.sys.id
107+
108+
const entryLinks = links[entryId]
109+
if (entryLinks) {
110+
entryLinks.forEach(link => {
111+
const backLinksForLink = backLinks[link]
112+
if (backLinksForLink) {
113+
backLinks[link] = backLinksForLink.filter(({ id }) => id !== entryId)
114+
}
115+
})
116+
}
117+
118+
delete links[entryId]
119+
}
120+
104121
export const buildForeignReferenceMap = ({
105122
contentTypeItems,
106123
entryList,
107124
resolvable,
108125
defaultLocale,
109126
space,
110127
useNameForId,
128+
previousForeignReferenceMapState,
129+
deletedEntries,
111130
}) => {
112-
const foreignReferenceMap = {}
131+
const foreignReferenceMapState = previousForeignReferenceMapState || {
132+
links: {},
133+
backLinks: {},
134+
}
135+
136+
const { links, backLinks } = foreignReferenceMapState
137+
138+
for (const deletedEntry of deletedEntries) {
139+
// remove stored entries from entry that is being deleted
140+
cleanupReferencesFromEntry(foreignReferenceMapState, deletedEntry)
141+
}
142+
113143
contentTypeItems.forEach((contentTypeItem, i) => {
114144
// Establish identifier for content type
115145
// Use `name` if specified, otherwise, use internal id (usually a natural-language constant,
@@ -122,6 +152,9 @@ export const buildForeignReferenceMap = ({
122152
}
123153

124154
entryList[i].forEach(entryItem => {
155+
// clear links added in previous runs for given entry, as we will recreate them anyway
156+
cleanupReferencesFromEntry(foreignReferenceMapState, entryItem)
157+
125158
const entryItemFields = entryItem.fields
126159
Object.keys(entryItemFields).forEach(entryItemFieldKey => {
127160
if (entryItemFields[entryItemFieldKey]) {
@@ -143,15 +176,21 @@ export const buildForeignReferenceMap = ({
143176
return
144177
}
145178

146-
if (!foreignReferenceMap[key]) {
147-
foreignReferenceMap[key] = []
179+
if (!backLinks[key]) {
180+
backLinks[key] = []
148181
}
149-
foreignReferenceMap[key].push({
182+
backLinks[key].push({
150183
name: `${contentTypeItemId}___NODE`,
151184
id: entryItem.sys.id,
152185
spaceId: space.sys.id,
153186
type: entryItem.sys.type,
154187
})
188+
189+
if (!links[entryItem.sys.id]) {
190+
links[entryItem.sys.id] = []
191+
}
192+
193+
links[entryItem.sys.id].push(key)
155194
})
156195
}
157196
} else if (
@@ -166,22 +205,28 @@ export const buildForeignReferenceMap = ({
166205
return
167206
}
168207

169-
if (!foreignReferenceMap[key]) {
170-
foreignReferenceMap[key] = []
208+
if (!backLinks[key]) {
209+
backLinks[key] = []
171210
}
172-
foreignReferenceMap[key].push({
211+
backLinks[key].push({
173212
name: `${contentTypeItemId}___NODE`,
174213
id: entryItem.sys.id,
175214
spaceId: space.sys.id,
176215
type: entryItem.sys.type,
177216
})
217+
218+
if (!links[entryItem.sys.id]) {
219+
links[entryItem.sys.id] = []
220+
}
221+
222+
links[entryItem.sys.id].push(key)
178223
}
179224
}
180225
})
181226
})
182227
})
183228

184-
return foreignReferenceMap
229+
return foreignReferenceMapState
185230
}
186231

187232
function prepareTextNode(id, node, key, text) {

‎packages/gatsby-source-contentful/src/source-nodes.js‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ export async function sourceNodes(
117117

118118
const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}`
119119
const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}`
120+
const CACHE_FOREIGN_REFERENCE_MAP_STATE = `contentful-foreign-reference-map-state-${sourceId}`
120121

121122
/*
122123
* Subsequent calls of Contentfuls sync API return only changed data.
@@ -240,16 +241,24 @@ export async function sourceNodes(
240241
assets,
241242
})
242243

244+
const previousForeignReferenceMapState = await cache.get(
245+
CACHE_FOREIGN_REFERENCE_MAP_STATE
246+
)
243247
// Build foreign reference map before starting to insert any nodes
244-
const foreignReferenceMap = buildForeignReferenceMap({
248+
const foreignReferenceMapState = buildForeignReferenceMap({
245249
contentTypeItems,
246250
entryList,
247251
resolvable,
248252
defaultLocale,
249253
space,
250254
useNameForId: pluginConfig.get(`useNameForId`),
255+
previousForeignReferenceMapState,
256+
deletedEntries: currentSyncData?.deletedEntries,
251257
})
252258

259+
await cache.set(CACHE_FOREIGN_REFERENCE_MAP_STATE, foreignReferenceMapState)
260+
const foreignReferenceMap = foreignReferenceMapState.backLinks
261+
253262
reporter.verbose(`Resolving Contentful references`)
254263

255264
const newOrUpdatedEntries = new Set()

0 commit comments

Comments
 (0)