Skip to content

Commit 91187da

Browse files
KyleAMathewsgatsbybot
KyleAMathews
and
gatsbybot
authored
feat(gatsby-source-drupal): Add tracing for full/delta fetches and http requests (#33142)
* feat(gatsby-source-drupal): Add tracing for full/delta fetches and http requests * Track creating nodes * don't create a parent sourceNodes span (redundant) and standardize other names * Fix tests * Use @opentelemetry/semantic-conventions for names Co-authored-by: gatsbybot <mathews.kyle+gatsbybot@gmail.com>
1 parent d31645c commit 91187da

File tree

5 files changed

+104
-15
lines changed

5 files changed

+104
-15
lines changed

‎packages/gatsby-source-drupal/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@
88
},
99
"dependencies": {
1010
"@babel/runtime": "^7.15.4",
11+
"@opentelemetry/semantic-conventions": "0.24.0",
1112
"agentkeepalive": "^4.1.4",
1213
"bluebird": "^3.7.2",
1314
"body-parser": "^1.19.0",
1415
"fastq": "^1.11.1",
16+
"opentracing": "^0.14.4",
1517
"gatsby-source-filesystem": "^3.14.0-next.2",
1618
"got": "^11.8.2",
1719
"http2-wrapper": "^2.0.5",

‎packages/gatsby-source-drupal/src/__tests__/index.js‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,8 @@ describe(`gatsby-source-drupal`, () => {
553553
expect(reporter.activityTimer).toHaveBeenCalledTimes(1)
554554
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
555555
1,
556-
`Fetch all data from Drupal`
556+
`Fetch all data from Drupal`,
557+
{ parentSpan: undefined }
557558
)
558559

559560
expect(activity.start).toHaveBeenCalledTimes(1)
@@ -576,11 +577,13 @@ describe(`gatsby-source-drupal`, () => {
576577
expect(reporter.activityTimer).toHaveBeenCalledTimes(2)
577578
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
578579
1,
579-
`Fetch all data from Drupal`
580+
`Fetch all data from Drupal`,
581+
{ parentSpan: undefined }
580582
)
581583
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
582584
2,
583-
`Remote file download`
585+
`Remote file download`,
586+
{ parentSpan: undefined }
584587
)
585588

586589
expect(activity.start).toHaveBeenCalledTimes(2)
@@ -639,7 +642,8 @@ describe(`gatsby-source-drupal`, () => {
639642
expect(reporter.activityTimer).toHaveBeenCalledTimes(1)
640643
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
641644
1,
642-
`Fetch incremental changes from Drupal`
645+
`Fetch incremental changes from Drupal`,
646+
{ parentSpan: {} }
643647
)
644648

645649
expect(activity.start).toHaveBeenCalledTimes(1)

‎packages/gatsby-source-drupal/src/gatsby-node.js‎

Lines changed: 88 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,14 @@ const _ = require(`lodash`)
33
const urlJoin = require(`url-join`)
44
import HttpAgent from "agentkeepalive"
55
// const http2wrapper = require(`http2-wrapper`)
6+
const opentracing = require(`opentracing`)
7+
const { SemanticAttributes } = require(`@opentelemetry/semantic-conventions`)
68

79
const { HttpsAgent } = HttpAgent
810

911
const { setOptions, getOptions } = require(`./plugin-options`)
1012

11-
const {
12-
nodeFromData,
13-
downloadFile,
14-
isFileNode,
15-
createNodeIdWithVersion,
16-
} = require(`./normalize`)
13+
const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
1714
const {
1815
handleReferences,
1916
handleWebhookUpdate,
@@ -31,6 +28,20 @@ let apiRequestCount = 0
3128
let initialSourcing = true
3229
let globalReporter
3330
async function worker([url, options]) {
31+
const tracer = opentracing.globalTracer()
32+
const httpSpan = tracer.startSpan(`http.get`, {
33+
childOf: options.parentSpan,
34+
})
35+
const parsedUrl = new URL(url)
36+
httpSpan.setTag(SemanticAttributes.HTTP_URL, url)
37+
httpSpan.setTag(SemanticAttributes.HTTP_HOST, parsedUrl.host)
38+
httpSpan.setTag(
39+
SemanticAttributes.HTTP_SCHEME,
40+
parsedUrl.protocol.replace(/:$/, ``)
41+
)
42+
httpSpan.setTag(SemanticAttributes.HTTP_TARGET, parsedUrl.pathname)
43+
httpSpan.setTag(`plugin`, `gatsby-source-drupal`)
44+
3445
// Log out progress during the initial sourcing.
3546
if (initialSourcing) {
3647
apiRequestCount += 1
@@ -48,13 +59,25 @@ async function worker([url, options]) {
4859
}
4960
}
5061

51-
return got(url, {
62+
const response = await got(url, {
5263
agent,
5364
cache: false,
5465
// request: http2wrapper.auto,
5566
// http2: true,
5667
...options,
5768
})
69+
70+
httpSpan.setTag(SemanticAttributes.HTTP_STATUS_CODE, response.statusCode)
71+
httpSpan.setTag(SemanticAttributes.HTTP_METHOD, `GET`)
72+
httpSpan.setTag(SemanticAttributes.NET_PEER_IP, response.ip)
73+
httpSpan.setTag(
74+
SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH,
75+
response.rawBody?.length
76+
)
77+
78+
httpSpan.finish()
79+
80+
return response
5881
}
5982

6083
const requestQueue = require(`fastq`).promise(worker, 20)
@@ -97,6 +120,8 @@ exports.sourceNodes = async (
97120
},
98121
pluginOptions
99122
) => {
123+
const tracer = opentracing.globalTracer()
124+
100125
globalReporter = reporter
101126
const {
102127
baseUrl,
@@ -213,6 +238,12 @@ ${JSON.stringify(webhookBody, null, 4)}
213238
}
214239

215240
if (fastBuilds) {
241+
const fastBuildsSpan = tracer.startSpan(`sourceNodes.fetch`, {
242+
childOf: parentSpan,
243+
})
244+
fastBuildsSpan.setTag(`plugin`, `gatsby-source-drupal`)
245+
fastBuildsSpan.setTag(`sourceNodes.fetch.type`, `delta`)
246+
216247
const lastFetched =
217248
store.getState().status.plugins?.[`gatsby-source-drupal`]?.lastFetched ??
218249
0
@@ -222,7 +253,8 @@ ${JSON.stringify(webhookBody, null, 4)}
222253
)
223254

224255
const drupalFetchIncrementalActivity = reporter.activityTimer(
225-
`Fetch incremental changes from Drupal`
256+
`Fetch incremental changes from Drupal`,
257+
{ parentSpan: fastBuildsSpan }
226258
)
227259
let requireFullRebuild = false
228260

@@ -238,6 +270,7 @@ ${JSON.stringify(webhookBody, null, 4)}
238270
headers,
239271
searchParams: params,
240272
responseType: `json`,
273+
parentSpan: fastBuildsSpan,
241274
},
242275
])
243276

@@ -251,12 +284,31 @@ ${JSON.stringify(webhookBody, null, 4)}
251284
setPluginStatus({ lastFetched: res.body.timestamp })
252285
requireFullRebuild = true
253286
} else {
287+
const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, {
288+
childOf: fastBuildsSpan,
289+
})
290+
touchNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
291+
254292
// Touch nodes so they are not garbage collected by Gatsby.
293+
let touchCount = 0
255294
getNodes().forEach(node => {
256295
if (node.internal.owner === `gatsby-source-drupal`) {
296+
touchCount += 1
257297
touchNode(node)
258298
}
259299
})
300+
touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount)
301+
touchNodesSpan.finish()
302+
303+
const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, {
304+
childOf: parentSpan,
305+
})
306+
createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
307+
createNodesSpan.setTag(`sourceNodes.fetch.type`, `delta`)
308+
createNodesSpan.setTag(
309+
`sourceNodes.createNodes.count`,
310+
res.body.entities?.length
311+
)
260312

261313
// Process sync data from Drupal.
262314
const nodesToSync = res.body.entities
@@ -298,23 +350,34 @@ ${JSON.stringify(webhookBody, null, 4)}
298350
}
299351
}
300352

353+
createNodesSpan.finish()
301354
setPluginStatus({ lastFetched: res.body.timestamp })
302355
}
303356
} catch (e) {
304357
gracefullyRethrow(drupalFetchIncrementalActivity, e)
358+
359+
drupalFetchIncrementalActivity.end()
360+
fastBuildsSpan.finish()
305361
return
306362
}
307363

308364
drupalFetchIncrementalActivity.end()
365+
fastBuildsSpan.finish()
309366

310367
if (!requireFullRebuild) {
311368
return
312369
}
313370
}
314371

315372
const drupalFetchActivity = reporter.activityTimer(
316-
`Fetch all data from Drupal`
373+
`Fetch all data from Drupal`,
374+
{ parentSpan }
317375
)
376+
const fullFetchSpan = tracer.startSpan(`sourceNodes.fetch`, {
377+
childOf: parentSpan,
378+
})
379+
fullFetchSpan.setTag(`plugin`, `gatsby-source-drupal`)
380+
fullFetchSpan.setTag(`sourceNodes.fetch.type`, `full`)
318381

319382
// Fetch articles.
320383
reporter.info(`Starting to fetch all data from Drupal`)
@@ -332,6 +395,7 @@ ${JSON.stringify(webhookBody, null, 4)}
332395
headers,
333396
searchParams: params,
334397
responseType: `json`,
398+
parentSpan: fullFetchSpan,
335399
},
336400
])
337401
allData = await Promise.all(
@@ -380,6 +444,7 @@ ${JSON.stringify(webhookBody, null, 4)}
380444
password: basicAuth.password,
381445
headers,
382446
responseType: `json`,
447+
parentSpan: fullFetchSpan,
383448
},
384449
])
385450
} catch (error) {
@@ -479,6 +544,13 @@ ${JSON.stringify(webhookBody, null, 4)}
479544
}
480545

481546
drupalFetchActivity.end()
547+
fullFetchSpan.finish()
548+
549+
const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, {
550+
childOf: parentSpan,
551+
})
552+
createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
553+
createNodesSpan.setTag(`sourceNodes.fetch.type`, `full`)
482554

483555
const nodes = new Map()
484556

@@ -492,6 +564,8 @@ ${JSON.stringify(webhookBody, null, 4)}
492564
})
493565
})
494566

567+
createNodesSpan.setTag(`sourceNodes.createNodes.count`, nodes.size)
568+
495569
// second pass - handle relationships and back references
496570
nodes.forEach(node => {
497571
handleReferences(node, {
@@ -510,8 +584,10 @@ ${JSON.stringify(webhookBody, null, 4)}
510584
const fileNodes = [...nodes.values()].filter(isFileNode)
511585

512586
if (fileNodes.length) {
513-
const downloadingFilesActivity =
514-
reporter.activityTimer(`Remote file download`)
587+
const downloadingFilesActivity = reporter.activityTimer(
588+
`Remote file download`,
589+
{ parentSpan }
590+
)
515591
downloadingFilesActivity.start()
516592
try {
517593
await asyncPool(concurrentFileRequests, fileNodes, async node => {
@@ -545,6 +621,7 @@ ${JSON.stringify(webhookBody, null, 4)}
545621
// We're now done with the initial sourcing.
546622
initialSourcing = false
547623

624+
createNodesSpan.finish()
548625
return
549626
}
550627

‎packages/gatsby/src/utils/api-runner-node.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,7 @@ const runAPI = async (plugin, api, args, activity) => {
375375
const apiCallArgs = [
376376
{
377377
...args,
378+
parentSpan: pluginSpan,
378379
basePath: pathPrefix,
379380
pathPrefix: publicPath,
380381
actions,

‎yarn.lock‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3723,6 +3723,11 @@
37233723
resolved "https://registry.yarnpkg.com/@open-draft/until/-/until-1.0.3.tgz#db9cc719191a62e7d9200f6e7bab21c5b848adca"
37243724
integrity sha512-Aq58f5HiWdyDlFffbbSjAlv596h/cOnt2DO1w3DOC7OJ5EHs0hd/nycJfiu9RJbT6Yk6F1knnRRXNSpxoIVZ9Q==
37253725

3726+
"@opentelemetry/semantic-conventions@0.24.0":
3727+
version "0.24.0"
3728+
resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-0.24.0.tgz#1028ef0e0923b24916158d80d2ddfd67ea8b6740"
3729+
integrity sha512-a/szuMQV0Quy0/M7kKdglcbRSoorleyyOwbTNNJ32O+RBN766wbQlMTvdimImTmwYWGr+NJOni1EcC242WlRcA==
3730+
37263731
"@pmmmwh/react-refresh-webpack-plugin@^0.4.3":
37273732
version "0.4.3"
37283733
resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.4.3.tgz#1eec460596d200c0236bf195b078a5d1df89b766"

0 commit comments

Comments
 (0)