fix(push publishing) fixes #30998 : Filter option relationships is no…

…t being respected (#31008) ### Proposed Changes * Passing down the Push Publishing Filter that was used when generating the bundle that is being sent to the receiving endpoints. * When a filter with `relationships: false` is used, DO NOT re-generate the `tree` table as this may also clear relationships of the specified content with parents that we don't have records of at that point. --------- Co-authored-by: erickgonzalez <[email protected]>
dotCMS · Dec 27, 2024 · 489be3e · 489be3e
1 parent 35f8c1b
commit 489be3e
Show file tree

Hide file tree

Showing 10 changed files with 201 additions and 85 deletions.
diff --git a/...S/src/enterprise/java/com/dotcms/enterprise/publishing/remote/handler/ContentHandler.java b/...S/src/enterprise/java/com/dotcms/enterprise/publishing/remote/handler/ContentHandler.java
@@ -55,11 +55,15 @@
 import com.dotcms.enterprise.publishing.remote.bundler.ContentBundler;
 import com.dotcms.enterprise.publishing.remote.bundler.HostBundler;
 import com.dotcms.enterprise.publishing.remote.handler.HandlerUtil.HandlerType;
+import com.dotcms.publisher.bundle.bean.Bundle;
+import com.dotcms.publisher.bundle.business.BundleAPI;
 import com.dotcms.publisher.pusher.PushPublisherConfig;
 import com.dotcms.publisher.pusher.wrapper.ContentWrapper;
 import com.dotcms.publisher.receiver.handler.IHandler;
 import com.dotcms.publishing.DotPublishingException;
+import com.dotcms.publishing.FilterDescriptor;
 import com.dotcms.publishing.PublisherConfig;
+import com.dotcms.publishing.PublisherFilter;
 import com.dotcms.rendering.velocity.services.PageLoader;
 import com.dotcms.repackage.com.google.common.base.Strings;
 import com.dotcms.storage.FileMetadataAPI;
@@ -120,7 +124,6 @@
 import com.thoughtworks.xstream.XStream;
 import io.vavr.Lazy;
 import io.vavr.control.Try;
-import java.util.stream.Collectors;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.io.File;
@@ -131,12 +134,16 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static com.dotcms.contenttype.model.type.PageContentType.PAGE_FRIENDLY_NAME_FIELD_VAR;
+import static com.dotcms.publishing.FilterDescriptor.RELATIONSHIPS_KEY;
+import static com.liferay.util.StringPool.BLANK;
 
 /**
  * This handler deals with Contentlet-related information inside a bundle and
@@ -165,6 +172,7 @@ public class ContentHandler implements IHandler {
 	private final VersionableAPI versionableAPI = APILocator.getVersionableAPI();
 	private final FileMetadataAPI fileMetadataAPI = APILocator.getFileMetadataAPI();
 	private final Lazy<MultiTreeAPI> multiTreeAPI = Lazy.of(APILocator::getMultiTreeAPI);
+	private final Lazy<BundleAPI> bundleAPI = Lazy.of(APILocator::getBundleAPI);
 
 	private final Map<String,Long> infoToRemove = new HashMap<>();
 	private final ExistingContentMapping existingContentMap = new ExistingContentMapping();
@@ -818,10 +826,18 @@ public void run () {
         }
         content = this.contentletAPI.checkin(content, userToUse, !RESPECT_FRONTEND_ROLES);
 
-        //First we need to remove the "old" trees in order to add this new ones
-        cleanTrees( content );
-		regenerateTree(wrapper,remoteLocalLanguages.getLeft());
-
+		final String filterKey = this.getFilterKeyFromBundle();
+		Logger.debug(this, () -> "Filter Key: " + filterKey);
+		final FilterDescriptor filterDescriptor = APILocator.getPublisherAPI().getFilterDescriptorByKey(filterKey);
+		boolean isRelationshipsFilter = filterDescriptor.getFilters().containsKey(RELATIONSHIPS_KEY) ? Boolean.class.cast(filterDescriptor.getFilters()
+				.get(FilterDescriptor.RELATIONSHIPS_KEY)) : true;
+		Logger.debug(this, () -> "Relationships Filter: " + isRelationshipsFilter);
+        if (isRelationshipsFilter) {
+			// Depending on the selected Push Publishing Filter, we need to remove the "old" trees
+			// in order to add the new ones, if the relationships filter is set to false, we shouldn't remove the trees
+			this.cleanTrees(content);
+			this.regenerateTree(wrapper, remoteLocalLanguages.getLeft());
+		}
         // Categories
         if (UtilMethods.isSet(wrapper.getCategories())) {
             handleContentCategories(content.getInode(), wrapper.getCategories());
@@ -877,6 +893,20 @@ public void run () {
 				PushPublishAction.PUBLISH, content.getIdentifier(), content.getInode(), content.getName(), config.getId());
     }
 
+	/**
+	 * Retrieves the Push Publishing Filter that was selected to generate the current Bundle.
+	 *
+	 * @return The Push Publishing Filter key. But, if the bundle doesn't exist, returns an empty
+	 * String.
+	 *
+	 * @throws DotDataException An error occurred when interacting with the data source.
+	 */
+	private String getFilterKeyFromBundle() throws DotDataException {
+		final Bundle bundle =
+				this.bundleAPI.get().getBundleById(com.dotmarketing.util.FileUtil.removeExtension(this.config.getId()));
+		return null != bundle ? bundle.getFilterKey() : BLANK;
+	}
+
 	/**
 	 * Invalidates the respective MultiTree cache entry when the pushed Contentlet is the child of an existing record
 	 * but wasn't pushed back when such a record was created.

diff --git a/dotCMS/src/main/java/com/dotcms/publisher/pusher/PushPublisher.java b/dotCMS/src/main/java/com/dotcms/publisher/pusher/PushPublisher.java
@@ -23,8 +23,15 @@
 import com.dotcms.enterprise.publishing.remote.bundler.VariantBundler;
 import com.dotcms.enterprise.publishing.remote.bundler.WorkflowBundler;
 import com.dotcms.publisher.bundle.bean.Bundle;
-import com.dotcms.publisher.business.*;
+import com.dotcms.publisher.business.DotPublisherException;
+import com.dotcms.publisher.business.EndpointDetail;
+import com.dotcms.publisher.business.PublishAuditAPI;
+import com.dotcms.publisher.business.PublishAuditHistory;
+import com.dotcms.publisher.business.PublishAuditStatus;
 import com.dotcms.publisher.business.PublishAuditStatus.Status;
+import com.dotcms.publisher.business.PublishQueueElement;
+import com.dotcms.publisher.business.PublisherAPI;
+import com.dotcms.publisher.business.PublisherQueueJob;
 import com.dotcms.publisher.endpoint.bean.PublishingEndPoint;
 import com.dotcms.publisher.endpoint.business.PublishingEndPointAPI;
 import com.dotcms.publisher.environment.bean.Environment;
@@ -39,14 +46,14 @@
 import com.dotcms.publishing.output.BundleOutput;
 import com.dotcms.publishing.output.TarGzipBundleOutput;
 import com.dotcms.repackage.org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.io.FileUtils;
 import com.dotcms.rest.ResourceResponse;
 import com.dotcms.rest.RestClientBuilder;
 import com.dotcms.system.event.local.business.LocalSystemEventsAPI;
 import com.dotcms.system.event.local.type.pushpublish.AllPushPublishEndpointsFailureEvent;
 import com.dotcms.system.event.local.type.pushpublish.AllPushPublishEndpointsSuccessEvent;
 import com.dotcms.system.event.local.type.pushpublish.SinglePushPublishEndpointFailureEvent;
 import com.dotcms.util.CloseUtils;
+import com.dotcms.util.EnterpriseFeature;
 import com.dotmarketing.business.APILocator;
 import com.dotmarketing.cms.factories.PublicEncryptionFactory;
 import com.dotmarketing.exception.DotDataException;
@@ -55,8 +62,10 @@
 import com.dotmarketing.util.Config;
 import com.dotmarketing.util.Logger;
 import com.dotmarketing.util.PushPublishLogger;
+import com.dotmarketing.util.UtilMethods;
 import com.liferay.portal.language.LanguageException;
 import com.liferay.portal.language.LanguageUtil;
+import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.ThreadContext;
 import org.glassfish.jersey.client.ClientProperties;
 import org.quartz.JobDetail;
@@ -150,7 +159,7 @@ public PublisherConfig process ( final PublishStatus status ) throws DotPublishi
 		try {
 			//Compressing bundle
 			File bundleRoot = BundlerUtil.getBundleRoot(this.config.getName(), false);
-			ArrayList<File> list = new ArrayList<>(1);
+			final List<File> list = new ArrayList<>(1);
 			list.add(bundleRoot);
 			File bundleFile = new File(bundleRoot + ".tar.gz");
 
@@ -165,7 +174,7 @@ public PublisherConfig process ( final PublishStatus status ) throws DotPublishi
 			currentStatusHistory = pubAuditAPI.getPublishAuditStatus(this.config.getId()).getStatusPojo();
 			Map<String, Map<String, EndpointDetail>> endpointsMap = currentStatusHistory.getEndpointsMap();
 			// If not empty, don't overwrite publish history already set via the PublisherQueueJob
-			boolean isHistoryEmpty = endpointsMap.size() == 0;
+			boolean isHistoryEmpty = endpointsMap.isEmpty();
 			currentStatusHistory.setPublishStart(new Date());
 			PushPublishLogger.log(this.getClass(), "Status Update: Sending to all environments");
 			pubAuditAPI.updatePublishAuditStatus(this.config.getId(), PublishAuditStatus.Status.SENDING_TO_ENDPOINTS, currentStatusHistory);
@@ -181,28 +190,27 @@ public PublisherConfig process ( final PublishStatus status ) throws DotPublishi
 
 				Map<String, EndpointDetail> endpointsDetail = endpointsMap.get(environment.getId());
 				//Filter Endpoints list and push only to those that are enabled and are Dynamic (not S3 at the moment)
-				for(PublishingEndPoint ep : allEndpoints) {
-					if(ep.isEnabled() && getProtocols().contains(ep.getProtocol())) {
-						// If pushing a bundle for the first time, always add
-						// all end-points
-						if (null == endpointsDetail || endpointsDetail.size() == 0) {
-							endpoints.add(ep);
-						} else {
-							EndpointDetail epDetail = endpointsDetail.get(ep.getId());
-							// If re-trying a bundle or just re-attempting to
-							// install a bundle, send it only to those
-							// end-points whose status IS NOT success
-							if (DeliveryStrategy.ALL_ENDPOINTS.equals(this.config.getDeliveryStrategy())
-									|| (DeliveryStrategy.FAILED_ENDPOINTS.equals(this.config.getDeliveryStrategy())
-									&& PublishAuditStatus.Status.SUCCESS.getCode() != epDetail.getStatus()
-                                    && Status.SUCCESS_WITH_WARNINGS.getCode() != epDetail.getStatus()
-									&& PublishAuditStatus.Status.BUNDLE_SENT_SUCCESSFULLY.getCode() != epDetail.getStatus())) {
+				if (null != allEndpoints) {
+					for (PublishingEndPoint ep : allEndpoints) {
+						if (ep.isEnabled() && getProtocols().contains(ep.getProtocol())) {
+							// If pushing a bundle for the first time, always add all end-points
+							if (null == endpointsDetail || endpointsDetail.isEmpty()) {
 								endpoints.add(ep);
+							} else {
+								EndpointDetail epDetail = endpointsDetail.get(ep.getId());
+								// If re-trying a bundle or just re-attempting to install a bundle,
+								// send it only to those end-points whose status IS NOT success
+								if (DeliveryStrategy.ALL_ENDPOINTS.equals(this.config.getDeliveryStrategy())
+										|| (DeliveryStrategy.FAILED_ENDPOINTS.equals(this.config.getDeliveryStrategy())
+										&& PublishAuditStatus.Status.SUCCESS.getCode() != epDetail.getStatus()
+										&& Status.SUCCESS_WITH_WARNINGS.getCode() != epDetail.getStatus()
+										&& PublishAuditStatus.Status.BUNDLE_SENT_SUCCESSFULLY.getCode() != epDetail.getStatus())) {
+									endpoints.add(ep);
+								}
 							}
 						}
 					}
 				}
-
 				boolean failedEnvironment = false;
 				if(!environment.getPushToAll()) {
 					Collections.shuffle(endpoints);
@@ -224,9 +232,10 @@ public PublisherConfig process ( final PublishStatus status ) throws DotPublishi
 
 						if (endpoint.hasAuthKey()) {
 							PushPublishLogger.log(this.getClass(), "Status Update: Sending Bundle");
-
+							final String filterKey = bundle.getFilterKey();
 							WebTarget webTarget = client.target(endpoint.toURL() + "/api/bundlePublisher/publish")
-									.queryParam("FORCE_PUSH", bundle.isForcePush());
+									.queryParam("FORCE_PUSH", bundle.isForcePush())
+									.queryParam("filterkey", filterKey);
 
 							Response response = webTarget.request(MediaType.APPLICATION_JSON)
 									.header("Content-Disposition", contentDisposition)

diff --git a/dotCMS/src/main/java/com/dotcms/publishing/PublisherAPIImpl.java b/dotCMS/src/main/java/com/dotcms/publishing/PublisherAPIImpl.java
@@ -31,7 +31,6 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -308,13 +307,14 @@ public FilterDescriptor getFilterDescriptorByKey(final String filterKey) {
                 this.filterList.stream().filter(filter -> filterKey.equalsIgnoreCase(filter.getKey())).findFirst().orElse(defaultFilter);
     }
 
+    @SuppressWarnings("unchecked")
     @CloseDBIfOpened
     @Override
     public PublisherFilter createPublisherFilter(final String bundleId) throws DotDataException, DotSecurityException {
 
         final String filterKey = APILocator.getBundleAPI().getBundleById(bundleId).getFilterKey();
         final FilterDescriptor filterDescriptor = this.getFilterDescriptorByKey(filterKey);
-        final PublisherFilterImpl publisherFilter = new PublisherFilterImpl((Boolean)filterDescriptor.getFilters().getOrDefault(FilterDescriptor.DEPENDENCIES_KEY,true),
+        final PublisherFilterImpl publisherFilter = new PublisherFilterImpl(filterKey, (Boolean)filterDescriptor.getFilters().getOrDefault(FilterDescriptor.DEPENDENCIES_KEY,true),
                 (Boolean)filterDescriptor.getFilters().getOrDefault(FilterDescriptor.RELATIONSHIPS_KEY,true));
 
         if(filterDescriptor.getFilters().containsKey(FilterDescriptor.EXCLUDE_CLASSES_KEY)){

diff --git a/dotCMS/src/main/java/com/dotcms/publishing/PublisherFilter.java b/dotCMS/src/main/java/com/dotcms/publishing/PublisherFilter.java
@@ -8,6 +8,19 @@
  */
 public interface PublisherFilter {
 
+    String CONTENT_ONLY_KEY = "ContentOnly.yml";
+    String FORCE_PUSH_KEY = "ForcePush.yml";
+    String INTELLIGENT__KEY = "Intelligent.yml";
+    String SHALLOW_PUSH_KEY = "ShallowPush.yml";
+    String WEB_CONTENT_KEY = "WebContentOnly.yml";
+
+    /**
+     * Returns the key of the specified Push Publishing Filter.
+     *
+     * @return The Push Publishing Filter key.
+     */
+    String key();
+
     /**
      * Check if the asset needs to be excluded from the bundle.
      * This because the excludeClasses contains the type of the asset.

diff --git a/dotCMS/src/main/java/com/dotcms/publishing/PublisherFilterImpl.java b/dotCMS/src/main/java/com/dotcms/publishing/PublisherFilterImpl.java
@@ -1,5 +1,6 @@
 package com.dotcms.publishing;
 
+import com.liferay.util.StringPool;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -12,18 +13,29 @@
  */
 public class PublisherFilterImpl implements PublisherFilter{
 
+    private final String key;
     private final Set<String> excludeClassesSet = new HashSet<>();
-    private final Set<String>excludeDependencyClassesSet = new HashSet<>();
-    private final Set<String>excludeQueryAssetIdSet = new HashSet<>();
-    private final Set<String>excludeDependencyQueryAssetIdSet = new HashSet<>();
+    private final Set<String> excludeDependencyClassesSet = new HashSet<>();
+    private final Set<String> excludeQueryAssetIdSet = new HashSet<>();
+    private final Set<String> excludeDependencyQueryAssetIdSet = new HashSet<>();
     private final boolean dependencies;
     private final boolean relationships;
 
     public PublisherFilterImpl(final boolean dependencies, final boolean relationships) {
+        this(StringPool.BLANK, dependencies, relationships);
+    }
+
+    public PublisherFilterImpl(final String key, final boolean dependencies, final boolean relationships) {
+        this.key = key;
         this.dependencies = dependencies;
         this.relationships = relationships;
     }
 
+    @Override
+    public String key() {
+        return this.key;
+    }
+
     @Override
     public boolean isDependencies() {
         return dependencies;
@@ -70,16 +82,17 @@ public boolean doesExcludeDependencyClassesContainsType(final String pusheableAs
         return this.excludeDependencyClassesSet.contains(pusheableAssetType.toLowerCase());
     }
 
-    public String toString(){
-        return "PublisherFilter {" +
-                " excludeClassesSet = " + this.excludeClassesSet.toString() +
-                " , excludeDependencyClassesSet = " + this.excludeDependencyClassesSet.toString() +
-                " , excludeQueryIds = " + this.excludeQueryAssetIdSet.toString() +
-                " , excludeDependencyQueryIds = " + this.excludeDependencyQueryAssetIdSet.toString() +
-                " , relationships = " + this.relationships +
-                " , dependencies = " + this.dependencies +
-                "}";
-
-
+    @Override
+    public String toString() {
+        return "PublisherFilterImpl{" +
+                "key='" + key + '\'' +
+                ", excludeClassesSet=" + excludeClassesSet +
+                ", excludeDependencyClassesSet=" + excludeDependencyClassesSet +
+                ", excludeQueryAssetIdSet=" + excludeQueryAssetIdSet +
+                ", excludeDependencyQueryAssetIdSet=" + excludeDependencyQueryAssetIdSet +
+                ", dependencies=" + dependencies +
+                ", relationships=" + relationships +
+                '}';
     }
+
 }