Switch preview to chunked transfer (elastic#921)

Switches the PR preview infrastructure from buffering the entire file to
chunked transfer. This allows us to do two things:
1. Shrink the size of the node process significantly.
2. Handle large files.

Author: nik9000

build_docs.pl

@@ -598,7 +598,7 @@ sub preview {
         } else {
             close STDIN;
             open( STDIN, "</dev/null" );
-            exec( qw(node /docs_build/preview/preview.js) );
+            exec( qw(node --max-old-space-size=128 /docs_build/preview/preview.js) );
         }
     } else {
         close STDIN;

integtest/spec/preview_spec.rb

@@ -10,14 +10,18 @@
 # and the preview is designed to update itself as its target repo changes so
 # we start it once and play with the target repo during the tests.
 RSpec.describe 'previewing built docs', order: :defined do
+  very_large_text = 'muchtext' * 1024 * 1024 * 5 # 40mb
   repo_root = File.expand_path '../../', __dir__
+
   convert_before do |src, dest|
     repo = src.repo_with_index 'repo', <<~ASCIIDOC
       Some text.
 
       image::resources/cat.jpg[A cat]
+      image::resources/very_large.jpg[Not a jpg but very big]
     ASCIIDOC
     repo.cp "#{repo_root}/resources/cat.jpg", 'resources/cat.jpg'
+    repo.write 'resources/very_large.jpg', very_large_text
     repo.commit 'add cat image'
     book = src.book 'Test'
     book.source repo, 'index.asciidoc'
@@ -64,11 +68,14 @@ def get(watermark, branch, path)
   shared_context 'docs for branch' do
     watermark = SecureRandom.uuid
     let(:watermark) { watermark }
-    let(:root) { get watermark, branch, 'guide/index.html' }
     let(:current_url) { 'guide/test/current' }
+    let(:root) { get watermark, branch, 'guide/index.html' }
     let(:cat_image) do
       get watermark, branch, "#{current_url}/resources/cat.jpg"
     end
+    let(:very_large) do
+      get watermark, branch, "#{current_url}/resources/very_large.jpg"
+    end
   end
 
   it 'logs that the built docs are ready' do
@@ -106,7 +113,10 @@ def get(watermark, branch, path)
     include_examples 'serves the docs root'
     it 'serves an image' do
       bytes = File.open("#{repo_root}/resources/cat.jpg", 'rb', &:read)
-      expect(cat_image).to serve(doc_body(eq(bytes)))
+      expect(cat_image).to serve(eq(bytes))
+    end
+    it 'serves a very large file' do
+      expect(very_large).to serve(eq(very_large_text))
     end
   end
   describe 'for the test branch' do

lib/ES/Util.pm

@@ -607,6 +607,8 @@ http {
       proxy_http_version 1.1;
       proxy_set_header Host \$host;
       proxy_cache_bypass \$http_upgrade;
+      proxy_buffering off;
+      gzip on;
       add_header 'Access-Control-Allow-Origin' '*';
       if (\$request_method = 'OPTIONS') {
         add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';

preview/preview.js

@@ -1,3 +1,13 @@
+/*
+ * Little server that listens for requests in the form of
+ * `${branch}.host/guide/${doc}`, looks up the doc from git and streams
+ * it back over the response.
+ *
+ * It uses subprocesses to access git. It is tempting to use NodeGit to prevent
+ * the fork overhead but for the most part the subprocesses are fast enough and
+ * I'm worried that NodeGit's `Blog.getContent` methods are synchronous.
+ */
+
 const child_process = require('child_process');
 const http = require('http');
 const url = require('url');
@@ -9,8 +19,6 @@ const checkTypeOpts = {
 };
 const catOpts = {
   'cwd': '/docs_build/.repos/target_repo.git',
-  'encoding': 'buffer',
-  'max_buffer': 1024 * 1024 * 20,
 };
 
 const requestHandler = (request, response) => {
@@ -23,6 +31,7 @@ const requestHandler = (request, response) => {
   const path = 'html' + parsedUrl.pathname.substring('/guide'.length);
   const branch = gitBranch(request.headers['host']);
   const requestedObject = `${branch}:${path}`;
+  // TODO keepalive?
   // TODO include the commit info for the branch in a header
   child_process.execFile('git', ['cat-file', '-t', requestedObject], checkTypeOpts, (err, stdout, stderr) => {
     if (err) {
@@ -36,15 +45,51 @@ const requestHandler = (request, response) => {
       }
       return;
     }
+
+    response.setHeader('Transfer-Encoding', 'chunked');
     const showGitObject = gitShowObject(requestedObject, stdout.trim());
-    child_process.execFile('git', ['cat-file', 'blob', showGitObject], catOpts, (err, stdout, stderr) => {
-      if (err) {
-        console.warn('unhandled error', err);
-        response.statusCode = 500;
-        response.end(err.message);
+    const child = child_process.spawn(
+      'git', ['cat-file', 'blob', showGitObject], catOpts
+    );
+
+    // We spool stderr into a string because it is never super big.
+    child.stderr.setEncoding('utf8');
+    let childstderr = '';
+    child.stderr.addListener('data', chunk => {
+      childstderr += chunk;
+    });
+
+    /* Let node handle all the piping because it handles backpressure properly.
+     * We don't let the pipe emit the `end` event though because we'd like to
+     * do that manually when the process exits. The chunk size seems looks like
+     * it is 4k which looks to come from the child_process. */
+    child.stdout.pipe(response, {end: false});
+
+    let exited = false;
+    child.addListener('close', (code, signal) => {
+      /* This can get invoked multiple times but we can only do anything
+       * the first time so we just drop the second time on the floor. */
+      if (exited) return;
+      exited = true;
+
+      if (code === 0 && signal === null) {
+        response.end();
         return;
       }
-      response.end(stdout);
+      /* Note that we're playing a little fast and loose here with the status.
+       * If we've already sent a chunk we can't change the status code. We're
+       * out of luck. We just terminate the response anyway. The server log
+       * is the best we can do for tracking these. */
+      console.warn('unhandled error', code, signal, childstderr);
+      response.statusCode = 500;
+      response.end(childstderr);
+    });
+    child.addListener('error', err => {
+      child.stdout.destroy();
+      child.stderr.destroy();
+      console.warn('unhandled error', err);
+      response.statusCode = 500;
+      response.end(err.message);
     });
   });
 }

publish_docker.sh

@@ -3,7 +3,7 @@
 set -e
 
 export BUILD=docker.elastic.co/docs/build:1
-export PREVIEW=docker.elastic.co/docs/preview:1
+export PREVIEW=docker.elastic.co/docs/preview:2
 
 cd $(git rev-parse --show-toplevel)
 ./build_docs --just-build-image