HADOOP-18055. Async Profiler endpoint for Hadoop daemons (#3824)

Reviewed-by: Akira Ajisaka <aajisaka@apache.org>

Author: virajjasani

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -27,6 +27,7 @@
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
@@ -771,6 +772,26 @@ private void initializeWebServer(String name, String hostName,
 
     addDefaultServlets();
     addPrometheusServlet(conf);
+    addAsyncProfilerServlet(contexts);
+  }
+
+  private void addAsyncProfilerServlet(ContextHandlerCollection contexts) throws IOException {
+    final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome();
+    if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) {
+      addServlet("prof", "/prof", ProfileServlet.class);
+      Path tmpDir = Paths.get(ProfileServlet.OUTPUT_DIR);
+      if (Files.notExists(tmpDir)) {
+        Files.createDirectories(tmpDir);
+      }
+      ServletContextHandler genCtx = new ServletContextHandler(contexts, "/prof-output-hadoop");
+      genCtx.addServlet(ProfileOutputServlet.class, "/*");
+      genCtx.setResourceBase(tmpDir.toAbsolutePath().toString());
+      genCtx.setDisplayName("prof-output-hadoop");
+    } else {
+      addServlet("prof", "/prof", ProfilerDisabledServlet.class);
+      LOG.info("ASYNC_PROFILER_HOME environment variable and async.profiler.home system property "
+          + "not specified. Disabling /prof endpoint.");
+    }
   }
 
   private void addPrometheusServlet(Configuration conf) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.http;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.regex.Pattern;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Servlet to serve files generated by {@link ProfileServlet}.
+ */
+@InterfaceAudience.Private
+public class ProfileOutputServlet extends DefaultServlet {
+
+  private static final long serialVersionUID = 1L;
+
+  private static final Logger LOG = LoggerFactory.getLogger(ProfileOutputServlet.class);
+  // default refresh period 2 sec
+  private static final int REFRESH_PERIOD = 2;
+  // Alphanumeric characters, plus percent (url-encoding), equals, ampersand, dot and hyphen
+  private static final Pattern ALPHA_NUMERIC = Pattern.compile("[a-zA-Z0-9%=&.\\-]*");
+
+  @Override
+  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
+      throws ServletException, IOException {
+    if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
+      resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+      ProfileServlet.setResponseHeader(resp);
+      resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
+      return;
+    }
+
+    String absoluteDiskPath = getServletContext().getRealPath(req.getPathInfo());
+    File requestedFile = new File(absoluteDiskPath);
+    // async-profiler version 1.4 writes 'Started [cpu] profiling' to output file when profiler is
+    // running which gets replaced by final output. If final output is not ready yet, the file size
+    // will be <100 bytes (in all modes).
+    if (requestedFile.length() < 100) {
+      LOG.info("{} is incomplete. Sending auto-refresh header.", requestedFile);
+      String refreshUrl = req.getRequestURI();
+      // Rebuild the query string (if we have one)
+      if (req.getQueryString() != null) {
+        refreshUrl += "?" + sanitize(req.getQueryString());
+      }
+      ProfileServlet.setResponseHeader(resp);
+      resp.setHeader("Refresh", REFRESH_PERIOD + ";" + refreshUrl);
+      resp.getWriter().write("This page will be auto-refreshed every " + REFRESH_PERIOD
+          + " seconds until the output file is ready. Redirecting to " + refreshUrl);
+    } else {
+      super.doGet(req, resp);
+    }
+  }
+
+  static String sanitize(String input) {
+    // Basic test to try to avoid any XSS attacks or HTML content showing up.
+    // Duplicates HtmlQuoting a little, but avoid destroying ampersand.
+    if (ALPHA_NUMERIC.matcher(input).matches()) {
+      return input;
+    }
+    throw new RuntimeException("Non-alphanumeric data found in input, aborting.");
+  }
+}