Segment Docs/Gainsight Integration

devguide.md

@@ -87,3 +87,13 @@ The current breadcrumb is currently determined based on the `page.path` and the
 ###  Searching
 We're using Algolia, which uses `algolia.js` and `algolia.css`. The index is updated as part of the build process on Netlify..
 
+### Gainsight
+We use federated search with Gainsight, which requires a similar indexing operation to Algolia. Here's a quick process overview:
+
+1. The current Gainsight indexing implementation reuses part of the `src/sitemap.xml` template to generate an array of objects in a JSON file - `src/gainsight-pages.json`. This file gets generated during the Jekyll build process.
+2. After a successful Netlify production deploy, a small Netlify build plugin (`netlify-plugins/gainsight`) reads the generated JSON file and sends a request to the Gainsight API to index the pages.
+3. Configuration in `netlify.toml` ensures that the build plugin only runs after production deploys.
+
+Gainsight requires the following environment variables to be configured in Netlify to work properly:
+- `GAINSIGHT_CLIENT_ID`
+- `GAINSIGHT_CLIENT_SECRET`

netlify-plugins/gainsight/index.js

@@ -0,0 +1,59 @@
+const fs = require('fs')
+const axios = require('axios')
+const { chunk } = require('lodash')
+
+// Limit number of URL objects per request
+const PAGE_SIZE = 50
+
+// Delay between requests to avoid rate limiting
+const REQUEST_DELAY = 50
+
+// Generate an OAuth2 token
+const generateToken = async () => {
+  const response = await axios.post(
+    'https://api2-us-west-2.insided.com/oauth2/token',
+
+    // Axios automatically sends URLSearchParams as form data
+    new URLSearchParams({
+      grant_type: 'client_credentials',
+      client_id: process.env.GAINSIGHT_CLIENT_ID,
+      client_secret: process.env.GAINSIGHT_CLIENT_SECRET,
+      scope: 'write'
+    })
+  )
+  return response.data.access_token
+}
+
+// Send a POST request to the Gainsight API to index the pages
+const index = async (pages) => {
+  const token = await generateToken()
+  const pageChunks = chunk(pages, PAGE_SIZE)
+  let currentPage = 0
+  console.log(`Gainsight: indexing ${pages.length} pages in ${pageChunks.length} chunks`)
+  for (const pageChunk of pageChunks) {
+    currentPage++
+    console.log(`Indexing page ${currentPage} of ${pageChunks.length}...`)
+    await axios.post(
+      'https://api2-us-west-2.insided.com/external-content/index',
+      { batch: pageChunk },
+      {
+        headers: {
+          'Authorization': `Bearer ${token}`,
+          'Content-Type': 'application/json',
+        }
+      }
+    )
+    await new Promise(resolve => setTimeout(resolve, REQUEST_DELAY))
+  }
+}
+
+// The `onSuccess` hook runs after the deploy is live. This matters because Gainsight will
+// access the URLs we upload, so the updated content has to be live before we index it.
+//
+// Copied file handling approach from Netlify's Brand Guardian plugin:
+// https://github.com/tzmanics/netlify-plugin-brand-guardian/blob/33f90f745086a2bc9ed9273b15340002960afdfa/index.js
+export const onSuccess = async ({ constants }) => {
+  const pagesJson = fs.readFileSync(`${constants.PUBLISH_DIR}/gainsight-pages.json`)
+  const pages = JSON.parse(pagesJson)
+  await index(pages)
+}

netlify-plugins/gainsight/manifest.yaml

@@ -0,0 +1 @@
+name: gainsight

netlify.toml

@@ -5,6 +5,10 @@
   # about the available build targets (develop, develop-inc, build)
   command = "jekyll algolia && yarn build"
 
+[[context.production.plugins]]
+  # Run the gainsight index script after a successful production deploy
+  package = "/netlify-plugins/gainsight"
+
 [context.deploy-preview]
   # For deploy previews, use the testing Jekyll environment, the develop build target calls
   command = "yarn develop"

package.json

@@ -22,7 +22,7 @@
     "@babel/plugin-proposal-object-rest-spread": "^7.15.6",
     "@babel/plugin-transform-runtime": "^7.15.0",
     "@babel/preset-env": "^7.15.6",
-    "axios": "^0.24.0",
+    "axios": "^1.6.7",
     "babel-loader": "^8.3.0",
     "concurrently": "^6.2.1",
     "front-matter": "^4.0.2",
@@ -58,6 +58,7 @@
     "globby": "11.0.4",
     "handlebars": "^4.7.7",
     "locate-path": "^7.1.0",
+    "lodash": "^4.17.21",
     "ms": "^2.1.3",
     "ora": "5.4.1",
     "p-locate": "5.0.0",

src/gainsight-pages.json

@@ -0,0 +1,15 @@
+---
+layout: null
+---
+[
+  {% assign pages = site.html_pages | where_exp:'doc','doc.sitemap != false' | where_exp:'doc','doc.url != "/404.html"' %}
+  {% for page in pages %}
+    {% unless page.hidden %}
+      {
+        "title": "{{ page.title | xml_escape }}",
+        "content": "{{ page.title | xml_escape }}",
+        "url": "{{ page.url | replace:'/index.html','/' | absolute_url | xml_escape }}"
+      }{% if forloop.last != true %},{% endif %}
+    {% endunless %}
+  {% endfor %}
+]

temp-test.js

@@ -0,0 +1,62 @@
+require('dotenv').config()
+const fs = require('fs')
+const axios = require('axios')
+const { chunk } = require('lodash')
+
+// Limit number of url objects per request
+const PAGE_SIZE = 50
+
+// Delay between requests to avoid rate limiting
+const REQUEST_DELAY = 50
+
+// Generate an OAuth2 token
+const generateToken = async () => {
+  const response = await axios.post(
+    'https://api2-us-west-2.insided.com/oauth2/token',
+
+    // Axios automatically sends URLSearchParams as form data
+    new URLSearchParams({
+      grant_type: 'client_credentials',
+      client_id: process.env.GAINSIGHT_CLIENT_ID,
+      client_secret: process.env.GAINSIGHT_CLIENT_SECRET,
+      scope: 'write'
+    })
+  )
+  return response.data.access_token
+}
+
+// Send a POST request to the Gainsight API to index the pages
+const index = async (pages) => {
+  const token = await generateToken()
+  const pageChunks = chunk(pages, PAGE_SIZE)
+  console.log(`Gainsight: indexing ${pages.length} pages in ${pageChunks.length} chunks`)
+  let currentPage = 0
+  for (const pageChunk of pageChunks) {
+    currentPage++
+    console.log(`Indexing page ${currentPage} of ${pageChunks.length}...`)
+    await axios.post(
+      'https://api2-us-west-2.insided.com/external-content/index',
+      { batch: pageChunk },
+      {
+        headers: {
+          'Authorization': `Bearer ${token}`,
+          'Content-Type': 'application/json',
+        }
+      }
+    )
+    await new Promise(resolve => setTimeout(resolve, REQUEST_DELAY))
+  }
+}
+
+// The `onSuccess` hook runs after the deploy is live. This matters because Gainsight will
+// access the URLs we upload, so the updated content has to be live before we index it.
+//
+// Copied file handling approach from Netlify's Brand Guardian plugin:
+// https://github.com/tzmanics/netlify-plugin-brand-guardian/blob/33f90f745086a2bc9ed9273b15340002960afdfa/index.js
+const onSuccess = async ({ constants }) => {
+  const pagesJson = fs.readFileSync(`${constants.PUBLISH_DIR}/gainsight-pages.json`)
+  const pages = JSON.parse(pagesJson)
+  await index(pages)
+}
+
+onSuccess({ constants: { PUBLISH_DIR: './_site' } })

yarn.lock

@@ -2353,12 +2353,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"axios@npm:^0.24.0":
-  version: 0.24.0
-  resolution: "axios@npm:0.24.0"
+"axios@npm:^1.6.7":
+  version: 1.6.7
+  resolution: "axios@npm:1.6.7"
   dependencies:
-    follow-redirects: ^1.14.4
-  checksum: 468cf496c08a6aadfb7e699bebdac02851e3043d4e7d282350804ea8900e30d368daa6e3cd4ab83b8ddb5a3b1e17a5a21ada13fc9cebd27b74828f47a4236316
+    follow-redirects: ^1.15.4
+    form-data: ^4.0.0
+    proxy-from-env: ^1.1.0
+  checksum: 87d4d429927d09942771f3b3a6c13580c183e31d7be0ee12f09be6d5655304996bb033d85e54be81606f4e89684df43be7bf52d14becb73a12727bf33298a082
   languageName: node
   linkType: hard
 
@@ -3885,7 +3887,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.14.4":
+"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.14.0":
   version: 1.14.7
   resolution: "follow-redirects@npm:1.14.7"
   peerDependenciesMeta:
@@ -3895,6 +3897,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"follow-redirects@npm:^1.15.4":
+  version: 1.15.5
+  resolution: "follow-redirects@npm:1.15.5"
+  peerDependenciesMeta:
+    debug:
+      optional: true
+  checksum: 5ca49b5ce6f44338cbfc3546823357e7a70813cecc9b7b768158a1d32c1e62e7407c944402a918ea8c38ae2e78266312d617dc68783fac502cbb55e1047b34ec
+  languageName: node
+  linkType: hard
+
 "form-data@npm:^3.0.0":
   version: 3.0.1
   resolution: "form-data@npm:3.0.1"
@@ -3906,6 +3918,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"form-data@npm:^4.0.0":
+  version: 4.0.0
+  resolution: "form-data@npm:4.0.0"
+  dependencies:
+    asynckit: ^0.4.0
+    combined-stream: ^1.0.8
+    mime-types: ^2.1.12
+  checksum: 01135bf8675f9d5c61ff18e2e2932f719ca4de964e3be90ef4c36aacfc7b9cb2fceb5eca0b7e0190e3383fe51c5b37f4cb80b62ca06a99aaabfcfd6ac7c9328c
+  languageName: node
+  linkType: hard
+
 "format@npm:^0.2.0":
   version: 0.2.2
   resolution: "format@npm:0.2.2"
@@ -6385,6 +6408,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"proxy-from-env@npm:^1.1.0":
+  version: 1.1.0
+  resolution: "proxy-from-env@npm:1.1.0"
+  checksum: ed7fcc2ba0a33404958e34d95d18638249a68c430e30fcb6c478497d72739ba64ce9810a24f53a7d921d0c065e5b78e3822759800698167256b04659366ca4d4
+  languageName: node
+  linkType: hard
+
 "pump@npm:^3.0.0":
   version: 3.0.0
   resolution: "pump@npm:3.0.0"
@@ -7685,7 +7715,7 @@ __metadata:
     ajv: ^6.10.2
     algoliasearch: ^4.10.5
     ansi-regex: ^6.0.1
-    axios: ^0.24.0
+    axios: ^1.6.7
     babel-loader: ^8.3.0
     browser-sync: ^2.29.1
     check-links: ^1.1.8
@@ -7703,6 +7733,7 @@ __metadata:
     handlebars: ^4.7.7
     js-yaml: ^4.1.0
     locate-path: ^7.1.0
+    lodash: ^4.17.21
     ms: ^2.1.3
     ora: 5.4.1
     p-locate: 5.0.0