Handle OOM in Java language server

Add parameters to the java server to make lemminx crash
when it runs out of memory.
Detect when the java server shuts down due to running out of memory,
display a message to the user that this happened,
and don't attempt to restart the server.

Closes #527

Signed-off-by: David Thompson <davthomp@redhat.com>

Author: datho7561

USAGE_DATA.md

@@ -18,6 +18,8 @@ vscode-xml has opt-in telemetry collection, provided by [vscode-redhat-telemetry
     * If the download fails, the associated error is attached to the telemetry event
  * A telemetry event is sent every time you click the "Open Proxy Configuration Documentation" link that is provided when the language server binary download fails due to a proxy related issue.
  * A telemetry event is sent every time you click the "Download Java" link that appears when you have [LemMinX extensions](./docs/Extensions.md) installed but don't have Java installed.
+ * A telemetry event is sent every time the Java XML language server crashes due to an Out Of Memory Error, which also contains the maximum heap space for the JVM (-Xmx) that you've set.
+ * A telemetry event is sent every time you click on the link to the documentation that appears after the Java XML language server crashes due to an Out Of Memory Error.
 
 ## What's included in the general telemetry data
 

docs/Troubleshooting.md

@@ -21,3 +21,22 @@ You can kill the process by:
 
   * on Windows OS: run `taskkill /F /PID ...` all instances
   * on other OS: run `kill -9 ...` all instances
+
+### The Language Server Crashes Due to an Out Of Memory Error
+
+If you are working with large XML files or referencing large schema files,
+this may lead to the language server running out of memory.
+The Java language server is more likely to run out memory than the binary language server.
+Switching to the binary language server
+or increasing the memory available to the Java language server could resolve this issue.
+
+If you get an Out of Memory Error, but aren't working with large XML files,
+then there may be a memory leak in the language server.
+Please [file a issue](https://github.com/redhat-developer/vscode-xml/issues) with a description of what you were doing if this is the case.
+
+#### How to increase the amount of memory available to the Java Language Server
+
+1. Go to settings
+2. Navigate to the setting `xml.server.vmargs`
+3. Add `-Xmx512m` to the setting string. This allows the the language server to use at most 512 megabytes of memory.
+4. If the problem persists, you can increase the `512m` to `1G` or higher

package-lock.json

@@ -67,7 +67,8 @@
     "Snippets"
   ],
   "devDependencies": {
-    "@types/fs-extra": "^8.0.0",
+    "@types/fs-extra": "^8.1.2",
+    "@types/glob": "^7.1.4",
     "@types/node": "^10.14.16",
     "@types/vscode": "^1.37.0",
     "@types/yauzl": "^2.9.1",

package.json

@@ -1,20 +1,30 @@
-import { window } from "vscode";
+import * as fs from "fs-extra";
+import { commands, ExtensionContext, window, workspace } from "vscode";
 import { CloseAction, ErrorAction, ErrorHandler, Message } from "vscode-languageclient";
+import { ClientCommandConstants } from "../commands/commandConstants";
+import { HEAP_DUMP_LOCATION } from "../server/java/jvmArguments";
+import { Telemetry } from "../telemetry";
+import glob = require("glob");
 
 /**
  * An error handler that restarts the language server,
- * unless it has been restarted 5 times in the last 3 minutes
+ * unless it has been restarted 5 times in the last 10 minutes,
+ * or if it crashed due to an Out Of Memory Error
  *
  * Adapted from [vscode-java](https://github.com/redhat-developer/vscode-java)
  */
 export class ClientErrorHandler implements ErrorHandler {
 
   private restarts: number[];
   private name: string;
+  private context: ExtensionContext;
+  private heapDumpFolder: string;
 
-  constructor(name: string) {
+  constructor(name: string, context: ExtensionContext) {
     this.name = name;
     this.restarts = [];
+    this.context = context;
+    this.heapDumpFolder = getHeapDumpFolderFromSettings() || context.globalStorageUri.fsPath;
   }
 
   error(_error: Error, _message: Message, _count: number): ErrorAction {
@@ -23,12 +33,24 @@ export class ClientErrorHandler implements ErrorHandler {
 
   closed(): CloseAction {
     this.restarts.push(Date.now());
+    const heapProfileGlob = new glob.GlobSync(`${this.heapDumpFolder}/java_*.hprof`);
+    if (heapProfileGlob.found.length) {
+      // Only clean heap dumps that are generated in the default location.
+      // The default location is the extension global storage
+      // This means that if users change the folder where the heap dumps are placed,
+      // then they will be able to read the heap dumps,
+      // since they aren't immediately deleted.
+      cleanUpHeapDumps(this.context);
+      Telemetry.sendTelemetry(Telemetry.JAVA_OOM_EVT, { 'jvm.xmx': getXmxFromSettings() });
+      showOOMMessage();
+      return CloseAction.DoNotRestart;
+    }
     if (this.restarts.length < 5) {
       return CloseAction.Restart;
     } else {
       const diff = this.restarts[this.restarts.length - 1] - this.restarts[0];
-      if (diff <= 3 * 60 * 1000) {
-        window.showErrorMessage(`The ${this.name} language server crashed 5 times in the last 3 minutes. The server will not be restarted.`);
+      if (diff <= 10 * 60 * 1000) {
+        window.showErrorMessage(`The ${this.name} language server crashed 5 times in the last 10 minutes. The server will not be restarted.`);
         return CloseAction.DoNotRestart;
       }
       this.restarts.shift();
@@ -37,3 +59,67 @@ export class ClientErrorHandler implements ErrorHandler {
   }
 
 }
+
+/**
+ * Deletes all the heap dumps generated by Out Of Memory errors
+ *
+ * @returns when the heap dumps have been deleted
+ */
+export async function cleanUpHeapDumps(context: ExtensionContext): Promise<void> {
+  const heapProfileGlob = new glob.GlobSync(`${context.globalStorageUri.fsPath}/java_*.hprof`);
+  for (let heapProfile of heapProfileGlob.found) {
+    await fs.remove(heapProfile);
+  }
+}
+
+/**
+ * Shows a message about the server crashing due to an out of memory issue
+ */
+async function showOOMMessage(): Promise<void> {
+  const DOCS = 'More info...';
+  const result = await window.showErrorMessage('The XML Language Server crashed due to an Out Of Memory Error, and will not be restarted. ', //
+    DOCS);
+  if (result === DOCS) {
+    Telemetry.sendTelemetry(Telemetry.OPEN_OOM_DOCS_EVT);
+    await commands.executeCommand(ClientCommandConstants.OPEN_DOCS,
+      {
+        page: 'Troubleshooting',
+        section: 'the-language-server-crashes-due-to-an-out-of-memory-error'
+      }
+    );
+  }
+}
+
+const HEAP_DUMP_FOLDER_EXTRACTOR = new RegExp(`${HEAP_DUMP_LOCATION}(?:'([^']+)'|"([^"]+)"|([^\\s]+))`);
+
+/**
+ * Returns the heap dump folder defined in the user's preferences, or undefined if the user does not set the heap dump folder
+ *
+ * @returns the heap dump folder defined in the user's preferences, or undefined if the user does not set the heap dump folder
+ */
+function getHeapDumpFolderFromSettings(): string {
+  const jvmArgs: string = workspace.getConfiguration('xml.server').get('vmargs');
+  const results = HEAP_DUMP_FOLDER_EXTRACTOR.exec(jvmArgs);
+  if (!results || !results[0]) {
+    return undefined;
+  }
+  return results[1] || results[2] || results[3];
+}
+
+const XMX_EXTRACTOR = /-Xmx([^\s]+)/;
+
+/**
+ * Returns the value that the user set for Xmx, or DEFAULT if the user didn't set Xmx
+ *
+ * @returns the value that the user set for Xmx, or DEFAULT if the user didn't set Xmx
+ */
+function getXmxFromSettings(): string {
+  const vmargs: string = workspace.getConfiguration('xml.server').get('vmargs', null);
+  if (vmargs != null) {
+    const extractOfVmargs: RegExpExecArray = XMX_EXTRACTOR.exec(vmargs);
+    if (extractOfVmargs.length && extractOfVmargs[1]) {
+      return extractOfVmargs[1];
+    }
+  }
+  return 'DEFAULT';
+}

src/client/clientErrorHandler.ts

@@ -3,7 +3,7 @@ import { commands, ExtensionContext, extensions, Position, TextDocument, TextEdi
 import { Command, ConfigurationParams, ConfigurationRequest, DidChangeConfigurationNotification, ExecuteCommandParams, LanguageClientOptions, MessageType, NotificationType, RequestType, RevealOutputChannelOn, TextDocumentPositionParams } from "vscode-languageclient";
 import { Executable, LanguageClient } from 'vscode-languageclient/node';
 import { XMLFileAssociation } from '../api/xmlExtensionApi';
-import { ClientCommandConstants, ServerCommandConstants } from '../commands/commandConstants';
+import { ServerCommandConstants } from '../commands/commandConstants';
 import { registerClientServerCommands } from '../commands/registerCommands';
 import { onExtensionChange } from '../plugin';
 import { RequirementsData } from "../server/requirements";
@@ -37,7 +37,7 @@ let languageClient: LanguageClient;
 
 export async function startLanguageClient(context: ExtensionContext, executable: Executable, logfile: string, externalXmlSettings: ExternalXmlSettings, requirementsData: RequirementsData): Promise<LanguageClient> {
 
-  const languageClientOptions: LanguageClientOptions = getLanguageClientOptions(logfile, externalXmlSettings, requirementsData);
+  const languageClientOptions: LanguageClientOptions = getLanguageClientOptions(logfile, externalXmlSettings, requirementsData, context);
   languageClient = new LanguageClient('xml', 'XML Support', executable, languageClientOptions);
 
   languageClient.onTelemetry(async (e: TelemetryEvent) => {
@@ -106,7 +106,11 @@ export async function startLanguageClient(context: ExtensionContext, executable:
   return languageClient;
 }
 
-function getLanguageClientOptions(logfile: string, externalXmlSettings: ExternalXmlSettings, requirementsData: RequirementsData): LanguageClientOptions {
+function getLanguageClientOptions(
+    logfile: string,
+    externalXmlSettings: ExternalXmlSettings,
+    requirementsData: RequirementsData,
+    context: ExtensionContext): LanguageClientOptions {
   return {
     // Register the server for xml and xsl
     documentSelector: [
@@ -135,7 +139,7 @@ function getLanguageClientOptions(logfile: string, externalXmlSettings: External
         shouldLanguageServerExitOnShutdown: true
       }
     },
-    errorHandler: new ClientErrorHandler('XML'),
+    errorHandler: new ClientErrorHandler('XML', context),
     synchronize: {
       //preferences starting with these will trigger didChangeConfiguration
       configurationSection: ['xml', '[xml]', 'files.trimFinalNewlines', 'files.trimTrailingWhitespace', 'files.insertFinalNewline']

src/client/xmlClient.ts

@@ -10,21 +10,23 @@
  *  Microsoft Corporation - Auto Closing Tags
  */
 
+import * as fs from 'fs-extra';
 import * as os from 'os';
 import * as path from 'path';
 import { ExtensionContext, extensions, languages } from "vscode";
 import { Executable, LanguageClient } from 'vscode-languageclient/node';
 import { XMLExtensionApi } from './api/xmlExtensionApi';
 import { getXmlExtensionApiImplementation } from './api/xmlExtensionApiImplementation';
+import { cleanUpHeapDumps } from './client/clientErrorHandler';
 import { getIndentationRules } from './client/indentation';
 import { startLanguageClient } from './client/xmlClient';
+import { registerClientOnlyCommands } from './commands/registerCommands';
 import { collectXmlJavaExtensions } from './plugin';
 import * as requirements from './server/requirements';
 import { prepareExecutable } from './server/serverStarter';
 import { ExternalXmlSettings } from "./settings/externalXmlSettings";
 import { getXMLConfiguration } from './settings/settings';
 import { Telemetry } from './telemetry';
-import { registerClientOnlyCommands } from './commands/registerCommands';
 
 let languageClient: LanguageClient;
 
@@ -52,6 +54,8 @@ export async function activate(context: ExtensionContext): Promise<XMLExtensionA
     storagePath = os.homedir() + "/.lemminx";
   }
   const logfile = path.resolve(storagePath + '/lemminx.log');
+  await fs.ensureDir(context.globalStorageUri.fsPath);
+  await cleanUpHeapDumps(context);
 
   const externalXmlSettings: ExternalXmlSettings = new ExternalXmlSettings();
 

src/extension.ts

@@ -13,7 +13,7 @@ import { getProxySettings, getProxySettingsAsEnvironmentVariables, ProxySettings
 import { getXMLConfiguration } from "../../settings/settings";
 import { Telemetry } from '../../telemetry';
 import { addTrustedHash, getTrustedHashes } from './binaryHashManager';
-const glob = require('glob');
+import glob = require('glob');
 
 const HTTPS_PATTERN = /^https:\/\//;
 const JAR_ZIP_AND_HASH_REJECTOR = /(?:\.jar)|(?:\.zip)|(?:\.sha256)$/;
@@ -430,4 +430,4 @@ async function acceptBinaryDownloadResponse(response: http.IncomingMessage): Pro
 
 async function openProxyDocumentation(): Promise<void> {
   await commands.executeCommand(ClientCommandConstants.OPEN_DOCS, { page: "Proxy" });
-}
+}