From b67d8e36bcd45b5501d1ae3a6bcb98a312d13f6e Mon Sep 17 00:00:00 2001 From: mmacik Date: Fri, 30 Jun 2017 13:47:09 +0200 Subject: [PATCH] MID-2929, MID-3230 addition of http client and a request interceptor for executing operations on remote cluster nodes --- .../impl/util/ReportPeerQueryInterceptor.java | 173 ++++++++++++++++++ gui/admin-gui/src/main/webapp/WEB-INF/web.xml | 11 ++ model/report-impl/pom.xml | 23 ++- .../report/impl/ReportManagerImpl.java | 46 ++++- .../midpoint/report/impl/ReportNodeUtils.java | 140 ++++++++++++++ 5 files changed, 381 insertions(+), 12 deletions(-) create mode 100644 gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/util/ReportPeerQueryInterceptor.java create mode 100644 model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportNodeUtils.java diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/util/ReportPeerQueryInterceptor.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/util/ReportPeerQueryInterceptor.java new file mode 100644 index 00000000000..716b6f35726 --- /dev/null +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/util/ReportPeerQueryInterceptor.java @@ -0,0 +1,173 @@ +package com.evolveum.midpoint.gui.impl.util; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.prism.query.ObjectQuery; +import com.evolveum.midpoint.prism.query.TypeFilter; +import com.evolveum.midpoint.repo.api.RepositoryService; +import com.evolveum.midpoint.schema.result.OperationResult; +import com.evolveum.midpoint.task.api.TaskManager; +import com.evolveum.midpoint.util.logging.LoggingUtils; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.web.security.MidPointApplication; +import com.evolveum.midpoint.xml.ns._public.common.common_3.NodeType; + +import org.apache.commons.io.IOUtils; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URLDecoder; +import java.util.List; + +public class ReportPeerQueryInterceptor extends HttpServlet { + + private static final long serialVersionUID = 7612750211021974750L; + private static String MIDPOINT_HOME = System.getProperty("midpoint.home"); + private static String EXPORT_DIR = MIDPOINT_HOME + "export/"; + private static String HEADER_USERAGENT = "mp-cluser-peer-client"; + private static String DEFAULTMIMETYPE = "application/octet-stream"; + private static String FILENAMEPARAMETER = "fname"; + private static String URLENCODING = "UTF-8"; + + private static final String INTERCEPTOR_CLASS = ReportPeerQueryInterceptor.class.getName() + "."; + private static final String OPERATION_LIST_NODES = INTERCEPTOR_CLASS + "listNodes"; + + private static final Trace LOGGER = TraceManager.getTrace(ReportPeerQueryInterceptor.class); + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String userAgent = request.getHeader("User-Agent"); + String remoteName = request.getRemoteHost(); + if (!HEADER_USERAGENT.equals(userAgent)) { + LOGGER.debug("Invalid user-agent: {}", userAgent); + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + } else if (!isKnownNode(remoteName, "File retrieval")) { + LOGGER.debug("Unknown node, host: {} ", remoteName); + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + } else { + String fileName = request.getParameter(FILENAMEPARAMETER); + fileName = URLDecoder.decode(fileName, URLENCODING); + StringBuilder buildfilePath = new StringBuilder(EXPORT_DIR).append(fileName); + String filePath = buildfilePath.toString(); + + File loadedFile = new File(filePath); + if (!loadedFile.exists()) { + StringBuilder errorBuilder = new StringBuilder("Download operation not successful. The file: ") + .append(fileName).append(" was not found on the filesystem"); + LOGGER.warn(errorBuilder.toString()); + response.setStatus(HttpServletResponse.SC_NOT_FOUND); + } else if (loadedFile.isDirectory()) { + StringBuilder errorBuilder = new StringBuilder("Download operation not successful. Attempt to download a directory with the name: ") + .append(fileName).append(" this operation is prohibited."); + LOGGER.warn(errorBuilder.toString()); + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + } else { + FileInputStream fileInputStream = new FileInputStream(filePath); + + ServletContext context = getServletContext(); + String mimeType = context.getMimeType(filePath); + + if (mimeType == null) { + // MIME mapping not found + mimeType = DEFAULTMIMETYPE; + } + + response.setContentType(mimeType); + response.setContentLength((int) loadedFile.length()); + + StringBuilder headerValue = new StringBuilder("attachment; filename=\"%s\"").append(fileName); + response.setHeader("Content-Disposition", headerValue.toString()); + + OutputStream outputStream = response.getOutputStream(); + + byte[] buffer = new byte[1024]; + int len; + while ((len = fileInputStream.read(buffer)) > -1) { + outputStream.write(buffer, 0, len); + } + IOUtils.closeQuietly(fileInputStream); + IOUtils.closeQuietly(outputStream); + LOGGER.trace("The file {} has been dispatched to the client.", fileName); + } + } + } + + @Override + protected void doDelete(HttpServletRequest request, HttpServletResponse response) throws ServletException, java.io.IOException { + + String userAgent = request.getHeader("User-Agent"); + String remoteName = request.getRemoteHost(); + if (!HEADER_USERAGENT.equals(userAgent)) { + LOGGER.debug("Invalid user-agent: {}", userAgent); + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + } else if (!isKnownNode(remoteName, "File deletion")) { + LOGGER.debug("Unknown node, host: {} ", remoteName); + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + } else { + String fileName = request.getParameter(FILENAMEPARAMETER); + fileName = URLDecoder.decode(fileName, URLENCODING); + StringBuilder buildfilePath = new StringBuilder(EXPORT_DIR).append(fileName); + String filePath = buildfilePath.toString(); + File reportFile = new File(filePath); + if (!reportFile.exists()) { + StringBuilder errorBuilder = new StringBuilder("Delete operation not successful. The file: ").append(fileName) + .append(" was not found on the filesystem."); + LOGGER.warn(errorBuilder.toString()); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + + } else if (reportFile.isDirectory()) { + StringBuilder errorBuilder = new StringBuilder("Delete operation not successful. Attempt to Delete a directory with the name: ") + .append(fileName).append(" This operation is prohibited."); + LOGGER.warn(errorBuilder.toString()); + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + } else { + reportFile.delete(); + response.setStatus(HttpServletResponse.SC_NO_CONTENT); + } + LOGGER.trace("Deletion of the file {} has finished.", fileName); + } + } + + protected RepositoryService getRepositoryService() { + MidPointApplication application = (MidPointApplication) MidPointApplication.get(); + return application.getRepositoryService(); + } + + protected TaskManager getTaskManager() { + MidPointApplication application = (MidPointApplication) MidPointApplication.get(); + return application.getTaskManager(); + } + + protected Boolean isKnownNode(String remoteName, String operation) { + + LOGGER.info("Checking if {} is a known node", remoteName); + OperationResult result = new OperationResult(OPERATION_LIST_NODES); + try { + ObjectQuery query = new ObjectQuery(); + TypeFilter filter = TypeFilter.createType(NodeType.COMPLEX_TYPE, null); + query.addFilter(filter); + List> knownNodes = getRepositoryService().searchObjects(NodeType.class, null, null, result); + for (PrismObject node : knownNodes) { + NodeType actualNode = node.asObjectable(); + if (remoteName.equals(actualNode.getHostname())) { + LOGGER.trace("The node {} was recognized as a known node. Attempting to execute the requested operation: {} ", actualNode.getHostname(), operation); + return true; + } + } + + } catch (Exception e) { + LOGGER.error("Unhandled exception when listing nodes"); + LoggingUtils.logUnexpectedException(LOGGER, "Unhandled exception when listing nodes", e); + } + return false; + } + +} diff --git a/gui/admin-gui/src/main/webapp/WEB-INF/web.xml b/gui/admin-gui/src/main/webapp/WEB-INF/web.xml index af31c84207b..a220a9e0fca 100644 --- a/gui/admin-gui/src/main/webapp/WEB-INF/web.xml +++ b/gui/admin-gui/src/main/webapp/WEB-INF/web.xml @@ -148,6 +148,17 @@ 1 + + + ReportPeerQueryInterceptor + com.evolveum.midpoint.gui.impl.util.ReportPeerQueryInterceptor + + + + + ReportPeerQueryInterceptor + /report + CXFServlet diff --git a/model/report-impl/pom.xml b/model/report-impl/pom.xml index d29ac277810..1a7ac7f676c 100644 --- a/model/report-impl/pom.xml +++ b/model/report-impl/pom.xml @@ -143,7 +143,7 @@ 20060411 runtime - + org.apache.cxf cxf-core @@ -165,8 +165,23 @@ javax.annotation javax.annotation-api - - + + Dependency definnition added, client to fetch reports <--> + + + org.apache.httpcomponents + httpclient + 4.5.2 + + + + org.apache.httpcomponents + httpcore + 4.4.6 + + + + com.evolveum.midpoint.repo repo-cache @@ -281,5 +296,5 @@ 1.2.1 test - + diff --git a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportManagerImpl.java b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportManagerImpl.java index 65b415f20f7..fc568ba9908 100644 --- a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportManagerImpl.java +++ b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportManagerImpl.java @@ -62,7 +62,6 @@ import javax.xml.datatype.XMLGregorianCalendar; import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -331,11 +330,27 @@ public void deleteReportOutput(ReportOutputType reportOutput, OperationResult pa parentResult.addSubresult(task.getResult()); OperationResult result = parentResult.createSubresult(DELETE_REPORT_OUTPUT); + + String filePath = reportOutput.getFilePath(); result.addParam("oid", oid); try { - File reportFile = new File(reportOutput.getFilePath()); - reportFile.delete(); - + File reportFile = new File(filePath); + + if(reportFile.exists()){ + reportFile.delete(); + }else { + ObjectReferenceType nodeRef =reportOutput.getNodeRef(); + String nodeOid= nodeRef.getOid(); + NodeType node = modelService.getObject(NodeType.class, nodeOid,null,null,parentResult).asObjectable(); + SystemConfigurationType systemConfig = modelService.getObject(SystemConfigurationType.class, SystemObjectsType.SYSTEM_CONFIGURATION.value(),null,task,result).asObjectable(); + String icUrlPattern = systemConfig.getInfrastructure().getIntraClusterHttpUrlPattern(); + String hostName = node.getHostname();; + ReportNodeUtils nodeUtils = new ReportNodeUtils(); + String [] splitted = filePath.split("/"); + String filename= splitted[splitted.length-1]; + nodeUtils.executeOperation(hostName,filename,icUrlPattern,"DELETE"); + } + ObjectDelta delta = ObjectDelta.createDeleteDelta(ReportOutputType.class, oid, prismContext); Collection> deltas = MiscSchemaUtil.createCollection(delta); @@ -369,12 +384,27 @@ public InputStream getReportOutputData(String reportOutputOid, OperationResult p return null; } File file = new File(filePath); - reportData = FileUtils.openInputStream(file); - + if(file.exists()){ + + reportData = FileUtils.openInputStream(file); + + }else{ + ObjectReferenceType nodeRef =reportOutput.getNodeRef(); + String nodeOid= nodeRef.getOid(); + NodeType node = modelService.getObject(NodeType.class, nodeOid,null,null,parentResult).asObjectable(); + String hostName = node.getHostname(); + + SystemConfigurationType systemConfig = modelService.getObject(SystemConfigurationType.class, SystemObjectsType.SYSTEM_CONFIGURATION.value(),null,task,result).asObjectable(); + String icUrlPattern = systemConfig.getInfrastructure().getIntraClusterHttpUrlPattern(); + ReportNodeUtils nodeUtils = new ReportNodeUtils(); + String [] splitted = filePath.split("/"); + String filename= splitted[splitted.length-1]; + reportData = nodeUtils.executeOperation(hostName,filename,icUrlPattern,"GET"); + } result.recordSuccessIfUnknown(); } catch (IOException ex) { - LOGGER.error("Report doesn't exist anymore."); - result.recordPartialError("Report doesn't exist anymore."); + LOGGER.error("Error while fetching file. File might not exist on the corresponding file system"); + result.recordPartialError("Error while fetching file. File might not exist on the corresponding file system."); throw ex; } catch (ObjectNotFoundException | SchemaException | SecurityViolationException | CommunicationException | ConfigurationException | ExpressionEvaluationException e) { diff --git a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportNodeUtils.java b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportNodeUtils.java new file mode 100644 index 00000000000..14dd8fc69cd --- /dev/null +++ b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportNodeUtils.java @@ -0,0 +1,140 @@ +package com.evolveum.midpoint.report.impl; +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; + +import com.evolveum.midpoint.util.exception.CommunicationException; +import com.evolveum.midpoint.util.exception.ConfigurationException; +import com.evolveum.midpoint.util.exception.ObjectNotFoundException; +import com.evolveum.midpoint.util.exception.SecurityViolationException; +import com.evolveum.midpoint.util.logging.LoggingUtils; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.HttpClientBuilder; + + +public class ReportNodeUtils { + + private static final Trace LOGGER = TraceManager.getTrace(ReportNodeUtils.class); + private static final String SPACE = "%20"; + private static final String HEADER_USERAGENT = "mp-cluser-peer-client"; + private static final String ENDPOINTURIPATH = "midpoint/report"; + private static String URLENCODING = "UTF-8"; + private static String FILENAMEPARAMETER = "fname"; + private static final Integer DEFAULTPORT = 80; + + public InputStream executeOperation(String host, String fileName, String intraClusterHttpUrlPattern, String operation) throws CommunicationException, SecurityViolationException, ObjectNotFoundException, ConfigurationException, IOException { + fileName = fileName.replaceAll("\\s", SPACE); + StringBuilder path = new StringBuilder(ENDPOINTURIPATH); + InputStream inputStream = null; + InputStream entityContent = null; + LOGGER.trace("About to initiate connection with {}", host); + try { + if (intraClusterHttpUrlPattern != null && !(intraClusterHttpUrlPattern.isEmpty())) { + LOGGER.trace("The cluster uri pattern: {} ", intraClusterHttpUrlPattern.toString()); + String[] splitted = intraClusterHttpUrlPattern.split("/"); + if (!(splitted.length > 3)) { // https://$host/midpoint + StringBuilder errorBuilder = new StringBuilder("Non valid IntraClusterHttpUrlPattern parameter value: ").append(intraClusterHttpUrlPattern); + throw new ConfigurationException(errorBuilder.toString()); + } + URIBuilder ubilder = new URIBuilder(); + String scheme = splitted[0].substring(0, splitted[0].length() - 1); + ubilder.setScheme(scheme).setHost(host); + String hostPart = splitted[2]; + + String[] hostAndPort = hostPart.split("\\:");//host:port + if (hostAndPort.length > 1) { + String port = hostAndPort[1]; + if (NumberUtils.isDigits(port)) { + ubilder.setPort(Integer.parseInt(port)); + } + } else { + ubilder.setPort(DEFAULTPORT); + } + ubilder.setPath(path.toString()).setParameter(FILENAMEPARAMETER, + fileName); + URI requestUri = ubilder.build(); + fileName = URLDecoder.decode(fileName, URLENCODING); + LOGGER.debug("Sending request to the following uri: {} ", requestUri.toString()); + HttpRequestBase httpRequest = buildHttpRequest(operation); + httpRequest.setURI(requestUri); + httpRequest.setHeader("User-Agent", HEADER_USERAGENT); + HttpClient client = HttpClientBuilder.create().build(); + try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpRequest)) { + HttpEntity entity = response.getEntity(); + Integer statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode == HttpStatus.SC_OK) { + LOGGER.info("Response OK, the file successfully returned by the cluster peer. "); + if (entity != null) { + entityContent = entity.getContent(); + ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int len; + while ((len = entityContent.read(buffer)) > -1) { + arrayOutputStream.write(buffer, 0, len); + } + arrayOutputStream.flush(); + inputStream = new ByteArrayInputStream(arrayOutputStream.toByteArray()); + } + } else if (statusCode == HttpStatus.SC_NO_CONTENT) { + if (HttpDelete.METHOD_NAME.equals(operation)) { + LOGGER.info("Deletion of the file {} was successful.", fileName); + } + + } else if (statusCode == HttpStatus.SC_FORBIDDEN) { + + StringBuilder errorBuilder = new StringBuilder("The access to the report ").append(fileName) + .append(" is forbidden."); + LOGGER.error("The access to the report with the name {} is forbidden.", fileName); + + throw new SecurityViolationException(errorBuilder.toString()); + } else if (statusCode == HttpStatus.SC_NOT_FOUND) { + StringBuilder errorBuilder = new StringBuilder("The report file ").append(fileName) + .append(" was not found on the originating nodes filesystem."); + throw new ObjectNotFoundException(errorBuilder.toString()); + } + } catch (ClientProtocolException e) { + + StringBuilder errorBuilder = new StringBuilder("An exception with the communication protocol has occurred during a query to the cluster peer. ") + .append(e.getLocalizedMessage()); + throw new CommunicationException(errorBuilder.toString()); + } + } + } catch (URISyntaxException e1) { + StringBuilder errorBuilder = new StringBuilder("Invalid uri syntax: ").append(e1.getLocalizedMessage()); + throw new CommunicationException(errorBuilder.toString()); + } catch (UnsupportedEncodingException e) { + LOGGER.error("Unhandled exception when listing nodes"); + LoggingUtils.logUnexpectedException(LOGGER, "Unhandled exception when listing nodes", e); + } finally { + IOUtils.closeQuietly(entityContent); + } + return inputStream; + } + + public HttpRequestBase buildHttpRequest(String typeOfRequest) { + HttpRequestBase httpRequest; + + if (HttpDelete.METHOD_NAME.equals(typeOfRequest)) { + httpRequest = new HttpDelete(); + + } else { + httpRequest = new HttpGet(); + } + return httpRequest; + } + +}