fix(pp) fixing edge push cases where content and folders from non-pushed hosts gets dragged along by dependency, and containers in a drafted state fail to publish

dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/remote/handler/ContainerHandler.java

@@ -63,6 +63,7 @@
 import com.dotmarketing.business.APILocator;
 import com.dotmarketing.business.CacheLocator;
 import com.dotmarketing.business.UserAPI;
+import com.dotmarketing.common.db.DotConnect;
 import com.dotmarketing.exception.DotDataException;
 import com.dotmarketing.portlets.containers.business.ContainerAPI;
 import com.dotmarketing.portlets.containers.model.Container;
@@ -72,6 +73,7 @@
 import com.dotmarketing.util.PushPublishLogger;
 import com.dotmarketing.util.PushPublishLogger.PushPublishAction;
 import com.dotmarketing.util.PushPublishLogger.PushPublishHandler;
+import com.dotmarketing.util.UtilMethods;
 import com.liferay.portal.model.User;
 import com.liferay.util.FileUtil;
 import com.liferay.util.StringPool;
@@ -84,6 +86,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * This handler class is part of the Push Publishing mechanism that deals with Container-related information inside a
@@ -174,6 +177,14 @@ private void handleContainers(final Collection<File> containers) throws DotPubli
         unpublish = containerWrapper.getOperation().equals(Operation.UNPUBLISH);
 
         Host localHost = APILocator.getHostAPI().find(containerId.getHostId(), systemUser, false);
+        if(UtilMethods.isEmpty(()->localHost.getIdentifier())){
+          final Container finalContainer = container;
+          Logger.warn(this.getClass(), "Ignoring container on non-existing host.  Id:" + container.getIdentifier() + ", title:" + Try.of(()->finalContainer.getTitle()).getOrElse("unknown") + ". Unable to find referenced host id:" + containerId.getHostId());
+          continue;
+        }
+
+
+
 
         if (containerWrapper.getOperation().equals(PushPublisherConfig.Operation.UNPUBLISH)) {
           String containerIden = container.getIdentifier();
@@ -184,12 +195,12 @@ private void handleContainers(final Collection<File> containers) throws DotPubli
         } else {
           // save if it doesn't exists
           final Container existing = containerAPI.find(container.getInode(), systemUser, false);
-          if (existing == null || !InodeUtils.isSet(existing.getIdentifier())) {
+          if (UtilMethods.isEmpty(()->existing.getIdentifier())) {
             containerAPI.save(container, containerWrapper.getCsList(), localHost, systemUser, false);
             PushPublishLogger.log(getClass(), PushPublishHandler.CONTAINER, PushPublishAction.PUBLISH_CREATE, container.getIdentifier(),
                 container.getInode(), container.getName(), config.getId());
           } else {
-            containerAPI.save(existing, containerWrapper.getCsList(), localHost, systemUser, false);
+            container= containerAPI.save(existing, containerWrapper.getCsList(), localHost, systemUser, false);
             PushPublishLogger.log(getClass(), PushPublishHandler.CONTAINER, PushPublishAction.PUBLISH_UPDATE, container.getIdentifier(),
                 container.getInode(), container.getName(), config.getId());
           }
@@ -201,6 +212,19 @@ private void handleContainers(final Collection<File> containers) throws DotPubli
 
         if (!unpublish) {
           final VersionInfo info = containerWrapper.getCvi();
+          if(!Objects.equals(info.getWorkingInode(), info.getLiveInode())){
+            boolean workingNotExists = new DotConnect()
+                    .setSQL("select inode from dot_containers where inode=?")
+                    .addParam(info.getWorkingInode())
+                    .loadResults()
+                    .isEmpty();
+            if(workingNotExists){
+              info.setWorkingInode(container.getInode());
+            }
+          }
+
+
+          //info.setWorkingInode(container.getInode());
           if (info.isLocked() && info.getLockedBy() != null) {
             final User user = Try.of(()-> APILocator.getUserAPI().loadUserById(info.getLockedBy())).getOrElse(systemUser);
             info.setLockedBy(user.getUserId());

dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/remote/handler/ContentHandler.java

@@ -120,6 +120,7 @@
 import com.thoughtworks.xstream.XStream;
 import io.vavr.Lazy;
 import io.vavr.control.Try;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.io.File;
@@ -218,7 +219,7 @@ public void handle(final File bundleFolder, final Boolean isHost) throws Excepti
 		List<File> contents = isHost?FileUtil.listFilesRecursively(bundleFolder, new HostBundler().getFileFilter()):
 				FileUtil.listFilesRecursively(bundleFolder, new ContentBundler().getFileFilter());
 		Collections.sort(contents);
-
+		contents = contents.stream().filter(File::isFile).collect(Collectors.toList());
 		handleContents(contents, bundleFolder, isHost);
 		HandlerUtil.setExistingContent(config.getId(), existingContentMap);
 
@@ -241,11 +242,22 @@ public void handle(final File bundleFolder, final Boolean isHost) throws Excepti
 		}
 	}
 
+
+
+	private boolean ignoreContent(Contentlet contentlet){
+
+		// if a host does not exist on target, skip content
+		Host localHost = Try.of(()->APILocator.getHostAPI().find(contentlet.getHost(), APILocator.systemUser(), false)).getOrNull();
+		return UtilMethods.isEmpty(()->localHost.getIdentifier());
+
+	}
+
+
 	/**
 	 * Reads the information of the contentlets contained in the bundle and
 	 * saves them in the destination server.
 	 * 
-	 * @param contents
+	 * @param contentsIn
 	 *            The list of data files containing the contentlet information.
 	 * @param folderOut
 	 *            - The location of the bundle in the file system.
@@ -257,23 +269,22 @@ public void handle(final File bundleFolder, final Boolean isHost) throws Excepti
 	 * @throws DotDataException
 	 *             An error occurred when interacting with the database.
 	 */
-	private void handleContents(final Collection<File> contents, final File folderOut, final Boolean isHost) throws DotPublishingException, DotDataException{
+	private void handleContents(final Collection<File> contentsIn, final File folderOut, final Boolean isHost) throws DotPublishingException, DotDataException{
 	    if(LicenseUtil.getLevel() < LicenseLevel.PROFESSIONAL.level) {
 			throw new RuntimeException("need an enterprise pro license to run this");
 		}
 	    final User systemUser = userAPI.getSystemUser();
         File workingOn=null;
         Contentlet content = null;
 		ContentWrapper wrapper = null;
+		final Collection<File> contents = contentsIn.stream().filter(File::isFile).collect(Collectors.toList());
     	try{
 	        final XStream xstream = XStreamHandler.newXStreamInstance();
 			final Set<Pair<String,Long>> pushedIdsToIgnore = new HashSet<>();
             for (final File contentFile : contents) {
                 workingOn=contentFile;
                 content = null;
-                if ( contentFile.isDirectory() ) {
-                    continue;
-                }
+
                 try(final InputStream input = Files.newInputStream(contentFile.toPath())){
                     wrapper = (ContentWrapper) xstream.fromXML(input);
                 }
@@ -311,6 +322,15 @@ private void handleContents(final Collection<File> contents, final File folderOu
                     content.getMap().remove(PAGE_FRIENDLY_NAME_FIELD_VAR.toLowerCase());
                 }
 
+
+				// if a host does not exist on target, skip content
+				if(ignoreContent(content)){
+					Contentlet finalContent = content;
+					Logger.warn(this.getClass(), "Ignoring contentlet:" + content.getIdentifier() + " | " + Try.of(
+                            finalContent::getTitle).getOrElse("unknown")  + " . Unable to find referenced host id:" + content.getHost());
+					continue;
+				}
+
 				content.setVariantId(wrapper.getContent().getVariantId());
 
                 // get the local language and assign it to the version info, and content, since the id's might be different
@@ -387,6 +407,15 @@ private void handleContents(final Collection<File> contents, final File folderOu
 					final Language remoteLang = wrapper.getLanguage();
 					final Pair<Long,Long> remoteLocalLanguages = this.existingContentMap.getRemoteLocalLanguages(wrapper);
 
+
+					// if a host does not exist on target, skip content
+					if(ignoreContent(content)){
+						Contentlet finalContent = content;
+						Logger.warn(this.getClass(), "Ignoring contentlet:" + content.getIdentifier() + " | " + Try.of(()->finalContent.getTitle()).getOrElse("unknown")  + " . Unable to find referenced host id:" + content.getHost());
+						continue;
+					}
+
+
 					if(UtilMethods.isSet(remoteLang) && remoteLang.getId() > 0) {
 						// This should take care of solving any existing conflicts. Previously solved by the Language Handler.
 						final Language mappedRemoteLanguage = config.getMappedRemoteLanguage(remoteLang.getId());
@@ -781,6 +810,7 @@ public void run () {
                 }
             } );
         }
+
         //Saving the content
         if (content.isArchived()){
             this.contentletAPI.unarchive(content, userToUse, !RESPECT_FRONTEND_ROLES);

dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/remote/handler/FolderHandler.java

@@ -183,7 +183,7 @@ private void handleFolders(Collection<File> folders) throws DotPublishingExcepti
 					continue;
 				}
 
-	        	folderName = Try.of(()-> folder.getPath()).getOrNull();
+	        	folderName = Try.of(folder::getPath).getOrNull();
 	        	folderId = folderWrapper.getFolderId();
 	        	host = folderWrapper.getHost();
 
@@ -194,7 +194,12 @@ private void handleFolders(Collection<File> folders) throws DotPublishingExcepti
 
 	        	//Check Host if exists otherwise create
 	        	Host localHost = APILocator.getHostAPI().find(host.getIdentifier(), systemUser, false);
+				if(UtilMethods.isEmpty(()->localHost.getIdentifier())){
+					Logger.warn(FolderHandler.class, "Unable to publish folder:" + folderName + ". Unable to find referenced host id:" + folder.getHostId());
+					Logger.warn(FolderHandler.class, "Make sure the host exists with the id:" + folder.getHostId() + " before pushing the folder or run the integrity checker before pushing.");
+					continue;
 
+				}
 	        	temp = fAPI.findFolderByPath(folderId.getPath(), localHost, systemUser, false);
 	        	if(folderWrapper.getOperation().equals(PushPublisherConfig.Operation.UNPUBLISH)) {
 	        		String folderIden = temp.getIdentifier();
@@ -328,7 +333,7 @@ else if(!folderId.getAssetName().equals(id.getAssetName())) {
             final String errorMsg = String.format("An error occurred when processing Folder in '%s': %s", workingOn,
                     e.getMessage());
             Logger.error(this.getClass(), errorMsg);
-			Logger.error(this, "-- Local Folder: " + (UtilMethods.isSet(temp) ? temp.toString() : "- object is null -"));
+			Logger.error(this, "-- Local Folder: " + (UtilMethods.isSet(temp) ? temp : "- object is null -"));
 			if(UtilMethods.isSet(temp) && UtilMethods.isSet(folderName)) {
 			    Logger.error(this, "-- folderName:" + folderName);
 			}

dotCMS/src/main/java/com/dotcms/publisher/ajax/RemotePublishAjaxAction.java

@@ -41,6 +41,7 @@
 import com.dotmarketing.portlets.folders.model.Folder;
 import com.dotmarketing.portlets.workflows.model.WorkflowActionFailureException;
 import com.dotmarketing.servlets.ajax.AjaxAction;
+import com.dotmarketing.util.Config;
 import com.dotmarketing.util.ConfigUtils;
 import com.dotmarketing.util.DateUtil;
 import com.dotmarketing.util.FileUtil;
@@ -653,6 +654,13 @@ public void downloadUnpushedBundle ( HttpServletRequest request, HttpServletResp
      */
     public void uploadBundle ( HttpServletRequest request, HttpServletResponse response ) throws FileUploadException, IOException{
 
+        if(!Config.getBooleanProperty("ENABLE_OLD_BUNDLE_UPLOAD_ENDPOINT", false)){
+            response.getWriter().println("Endpoint disabled, POST your bundle to: /api/bundle/sync or set DOT_ENABLE_OLD_BUNDLE_UPLOAD_ENDPOINT=true to use this old endpoint.");
+        }
+
+
+
+
     	try {
 			if(!APILocator.getLayoutAPI().doesUserHaveAccessToPortlet("publishing-queue", getUser())){
 				response.sendError(401);

dotCMS/src/main/java/com/dotcms/publisher/receiver/BundlePublisher.java

@@ -118,10 +118,8 @@ public PublisherConfig init(PublisherConfig config) throws DotPublishingExceptio
         handlers.add(new HostHandler(config));
         handlers.add(new FolderHandler(config));
         handlers.add(new WorkflowHandler(config));
-        if (Config.getBooleanProperty("PUSH_PUBLISHING_PUSH_STRUCTURES", true)) {
-            handlers.add(new ContentTypeHandler(config));
-            handlers.add(new RelationshipHandler(config));
-        }
+        handlers.add(new ContentTypeHandler(config));
+        handlers.add(new RelationshipHandler(config));
         handlers.add(new ContainerHandler(config));
         handlers.add(new TemplateHandler(config));
         handlers.add(new LanguageHandler(config));

dotCMS/src/main/java/com/dotcms/rest/BundleResource.java

@@ -923,15 +923,15 @@ public final Response uploadBundleSync(@Context final HttpServletRequest request
                 response.setContentType("text/html; charset=utf-8");
                 PublishAuditStatus previousStatus = PublishAuditAPI
                         .getInstance().updateAuditTable(endpointId, endpointId, bundleFolder);
-
-                final PublisherConfig config = !previousStatus.getStatus().equals(Status.PUBLISHING_BUNDLE)?
-                        new PushPublisherJob().processBundle(bundleName, previousStatus): null;
-
-                final String finalStatus = config != null ?
-                        config.getPublishAuditStatus().getStatus().name():
-                        Status.RECEIVED_BUNDLE.name();
-
-                return Response.ok(ImmutableMap.of("bundleName", bundleName, "status", finalStatus))
+                DotConcurrentFactory.getInstance().getSubmitter().submit(()-> {
+                    final PublisherConfig config = !previousStatus.getStatus().equals(Status.PUBLISHING_BUNDLE) ?
+                            new PushPublisherJob().processBundle(bundleName, previousStatus) : null;
+
+                    final String finalStatus = config != null ?
+                            config.getPublishAuditStatus().getStatus().name() :
+                            Status.RECEIVED_BUNDLE.name();
+                });
+                return Response.ok(ImmutableMap.of("bundleName", bundleName, "status", Status.RECEIVED_BUNDLE.name()))
                         .build();
             } catch (IOException e) {
                 Logger.error(this, "Unable to import Bundle", e);
@@ -943,6 +943,7 @@ public final Response uploadBundleSync(@Context final HttpServletRequest request
         return Response.ok().build();
     } // uploadBundleSync.
 
+    @Path("/async")
     @POST
     @JSONP
     @NoCache