SSR: request intereception optimization

Author: ebidel

public/app.js

@@ -23,6 +23,7 @@ dbHelper.setApp(firebase.initializeApp(shared.firebaseConfig));
 
 let _posts = [];
 let _filteringBy = null;
+const FILTERING_PARAMS = ['domain', 'author'];
 
 async function fetchPosts(url, maxResults = null) {
   try {
@@ -56,31 +57,39 @@ function handleDelete(el, dateStr, url) {
 }
 
 function filterBy(key, needle = null) {
+  if (key && !FILTERING_PARAMS.includes(key)) {
+    return;
+  }
+
   const currentURL = new URL(location.href);
+  const params = currentURL.searchParams;
   const filterEl = document.querySelector('#filtering');
   const needleEl = filterEl.querySelector('.filtering-needle');
-  let posts = _posts;
 
-  // TODO: this clears all params...even those unrelated to filtering.
-  for (const key of currentURL.searchParams.keys()) {
-    currentURL.searchParams.delete(key);
+  // Clear all previous filters.
+  for (const key of params.keys()) {
+    if (FILTERING_PARAMS.includes(key)) {
+      params.delete(key);
+    }
   }
 
+  let filteredPosts = _posts;
+
   // TODO: support filtering on more than one thing.
   if (needle === _filteringBy) {
-    currentURL.searchParams.delete(key);
+    params.delete(key);
     filterEl.classList.remove('on');
     _filteringBy = null;
   } else {
-    posts = posts.filter(post => post[key] === needle);
-    currentURL.searchParams.set(key, needle);
+    filteredPosts = filteredPosts.filter(post => post[key] === needle);
+    params.set(key, needle);
     needleEl.textContent = needle;
     filterEl.classList.add('on');
     _filteringBy = needle;
   }
 
   window.history.pushState(null, '', currentURL.href);
-  renderPosts(posts, container);
+  renderPosts(filteredPosts, container);
 }
 
 function clearFilters() {
@@ -176,17 +185,20 @@ async function getLatestPosts() {
 }
 
 (async() => {
-  const ssr = container.querySelector('#posts');
+  const PRE_RENDERED = container.querySelector('#posts'); // Already exists in DOM if we've SSR.
 
   try {
-    _posts = await getLatestPosts(); // populate cache
+    // Populates client-side cache for future realtime updates.
+    // Note: this basically results in 2x requests per page load, as we're
+    // making the same requests the server just made. Now repeating them client-side.
+    _posts = await getLatestPosts();
 
-    // Posts are already rendered in the DOM for the SSR case. Don't re-render.
-    if (!ssr) {
+     // Posts markup is already in place if we're SSRing. Don't re-render DOM.
+    if (!PRE_RENDERED) {
       renderPosts(_posts, container);
     }
 
-    // Subscribe to firestore updates.
+    // Subscribe to realtime firestore updates.
     realtimeUpdatePosts(util.currentYear);
 
     const params = new URL(location.href).searchParams;

server.mjs

@@ -18,6 +18,8 @@
 
 import fs from 'fs';
 import bodyParser from 'body-parser';
+import url from 'url';
+const URL = url.URL;
 // import compression from 'compression';
 // import minify from 'express-minify';
 import express from 'express';
@@ -42,21 +44,56 @@ function updateRSSFeedsDaily() {
   setTimeout(updateRSSFeedsDaily, dayInMilliseconds);
 }
 
-async function ssr(url) {
+/**
+ *
+ * @param url {string} URL to prerender.
+ * @param onlyCriticalRequests {boolean} Reduces the number of requests the
+ *     browser makes by aborting requests that are non-critical to rendering
+ *     the DOM of the page (stylesheets, images, media). True by default.
+ * @return {string} Serialized page output as an html string.
+ */
+async function ssr(url, onlyCriticalRequests = true) {
   if (RENDER_CACHE.has(url)) {
     return RENDER_CACHE.get(url);
   }
 
+  // Add param so client-side page can know it's being rendered by headless on the server.
+  const urlToFetch = new URL(url);
+  urlToFetch.searchParams.set('headless', '');
+
   const tic = Date.now();
   const browser = await puppeteer.launch({
-    args: ['--disable-dev-shm-usage']
+    args: ['--disable-dev-shm-usage'],
   });
   const page = await browser.newPage();
-  await page.goto(url, {waitUntil: 'domcontentloaded'});
+
+  // Small optimization. Since we only care about rendered DOM, ignore images,
+  // css, and other media that don't produce markup.
+  if (onlyCriticalRequests) {
+    await page.setRequestInterception(true);
+    page.on('request', req => {
+      const whitelist = ['document', 'script', 'xhr', 'fetch', 'websocket'];
+      if (whitelist.includes(req.resourceType())) {
+        req.continue();
+      } else {
+        req.abort();
+        // req.respond({
+        //   status: 200,
+        //   contentType: 'text/plain',
+        //   body: ''
+        // });
+      }
+    });
+  }
+
+  // TODO: another optimization may be to take enter page out of rendering pipeline.
+  // Add html { display: none } to page.
+
+  await page.goto(urlToFetch.href, {waitUntil: 'domcontentloaded'});
   await page.waitForSelector('#posts'); // wait for posts to be in filled in page.
-  const html = await page.content(); // Browser "SSR" page for us! Get serialized DOM.
+  const html = await page.content(); // Use browser to prerender page, get serialized DOM output!
   await browser.close();
-  console.info(`Headless chrome render time: ${Date.now() - tic}ms`);
+  console.info(`Headless rendered page in: ${Date.now() - tic}ms`);
 
   RENDER_CACHE.set(url, html); // cache rendered page.
 
@@ -99,7 +136,8 @@ app.use(express.static('node_modules/lit-html'));
 // });
 
 app.get('/ssr', async (req, res) => {
-  const html = await ssr(req.getOrigin());
+  const optimizeReqs = 'noreduce' in req.query ? false : true;
+  const html = await ssr(req.getOrigin(), optimizeReqs);
   res.status(200).send(html);
 });