Skip to content

Commit

Permalink
Always include dependency reference in PP manifest (#29823)
Browse files Browse the repository at this point in the history 
Closes #28214

### Proposed Changes
* With this change the dependency reference is always included for all
the files listed in the bundle manifest, even if the files were excluded
from the bundle because of a filter or because the file wasn't changed
after the last PP.
* This allows to trace why every file was listed in the manifest even
when the file was excluded from the bundle that was sent to the receiver

### Checklist
- [x] Tests

---------

Co-authored-by: freddyDOTCMS <147462678+freddyDOTCMS@users.noreply.github.com>
Co-authored-by: Humberto Morera <31667212+hmoreras@users.noreply.github.com>
Co-authored-by: Will Ezell <will@dotcms.com>
Co-authored-by: Daniel Enrique Colina Rodríguez <daniel.colina@dotcms.com>
Co-authored-by: Nicolas Molina Monroy <hi@nicobytes.com>
Co-authored-by: Victor Alfaro <victor.alfaro@dotcms.com>
Co-authored-by: Kevin Davila <144152756+KevinDavilaDotCMS@users.noreply.github.com>
Co-authored-by: Kevin Davila <kfariid@gmail.com>
Co-authored-by: Jonathan <jonathan.sanchez@dotcms.com>
Co-authored-by: Freddy Montes <751424+fmontes@users.noreply.github.com>
Co-authored-by: Fabrizzio Araya <37148755+fabrizzio-dotCMS@users.noreply.github.com>
Co-authored-by: spbolton <steve.bolton@dotcms.com>
Co-authored-by: Jalinson Diaz <zjaaaldev@gmail.com>
Co-authored-by: Jonathan Gamba <jonathan.gamba@dotcms.com>
Co-authored-by: Jose Castro <jose.castro@dotcms.com>
Co-authored-by: Rafael Velazco <rjvelazco21@gmail.com>
Co-authored-by: Nollymar Longa <nollymar.longa@dotcms.com>
Co-authored-by: Arcadio Quintero <oidacra@gmail.com>
Co-authored-by: Valentino Giardino <77643678+valentinogiardino@users.noreply.github.com>
Co-authored-by: Geronimo Ortiz <geronimo.ortiz@dotcms.com>
Co-authored-by: erickgonzalez <erick.gonzalez@dotcms.com>
  • Loading branch information
@dsolistorres @freddyDOTCMS
@hmoreras @wezell @dcolina @nicobytes @victoralfaro-dotcms @KevinDavilaDotCMS @kevindaviladev @jdotcms @fmontes @fabrizzio-dotCMS @spbolton @zJaaal @jgambarios @jcastro-dotcms @rjvelazco @nollymar @oidacra @valentinogiardino @gortiz-dotcms @erickgonzalez
22 people authored Sep 20, 2024
1 parent 1ca1832 commit b59cf7d
Show file tree
Hide file tree
Showing 10 changed files with 367 additions and 110 deletions.
61 changes: 51 additions & 10 deletions dotCMS/src/main/java/com/dotcms/publisher/pusher/PushPublisherConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,16 @@ public BundleAssets dependencySet(){
return bundleAssets;
}

/**
* Add an asset to the bundle. If the asset is already added, it will not be added again.
* It will be included in the dependency processor, so its dependencies are included too.
* @param asset The asset to be added
* @param pusheableAsset The object type of the asset
* @param evaluateReason The reason why the asset is being added
* @return true if the asset was added, false otherwise
*/
public <T> boolean addWithDependencies(final T asset, final PusheableAsset pusheableAsset,
final String reason) {
final String evaluateReason) {
final String key = DependencyManager.getBundleKey(asset);
final boolean added = bundleAssets.isAdded(key, pusheableAsset);
final boolean isAlreadyAdded = bundleAssets.isDependenciesAdded(key, pusheableAsset);
Expand All @@ -278,57 +286,90 @@ public <T> boolean addWithDependencies(final T asset, final PusheableAsset pushe
this.dependencyProcessor.addAsset(asset, pusheableAsset);

if (!added) {
writeIncludeManifestItem(asset, reason);
writeIncludeManifestItem(asset, evaluateReason);
}
}

return !isAlreadyAdded;
}

public <T> boolean add(final T asset, final PusheableAsset pusheableAsset, final String reason) {
/**
* Add an asset to the bundle. If the asset is already added, it will not be added again.
* @param asset The asset to be added
* @param pusheableAsset The object type of the asset
* @param evaluateReason The reason why the asset is being added
* @return true if the asset was added, false otherwise
*/
public <T> boolean add(final T asset, final PusheableAsset pusheableAsset, final String evaluateReason) {
final String key = DependencyManager.getBundleKey(asset);

if(!bundleAssets.isAdded(key, pusheableAsset)) {
bundleAssets.add(key, pusheableAsset);
writeIncludeManifestItem(asset, reason);
writeIncludeManifestItem(asset, evaluateReason);
return true;
} else {
return false;
}
}

/**
* Returns true if the asset is already added to the bundle, false otherwise.
* @param asset The asset to be checked
* @param pusheableAsset The object type of the asset
* @return true if the asset is already added, false otherwise
*/
public <T> boolean contains(final T asset, final PusheableAsset pusheableAsset) {
final String key = DependencyManager.getBundleKey(asset);
return bundleAssets.isAdded(key, pusheableAsset);
}

public <T> boolean exclude(final T asset, final PusheableAsset pusheableAsset, final String reason) {
/**
* Add an asset to the list of assets that will be excluded from the bundle.
* If the asset is already excluded, it will not be excluded again.
* @param asset The asset to be excluded
* @param pusheableAsset The object type of the asset
* @param evaluateReason The reason why the asset is being evaluated
* @param excludeReason The reason why the asset is being excluded
* @return true if the asset was excluded, false otherwise
*/
public <T> boolean exclude(final T asset, final PusheableAsset pusheableAsset, final String evaluateReason, final String excludeReason) {
final String key = DependencyManager.getBundleKey(asset);

if(!excludes.contains(key)) {
excludes.add(key);
writeExcludeManifestItem(asset, reason);
writeExcludeManifestItem(asset, evaluateReason, excludeReason);
return true;
} else {
return false;
}
}

public <T> void writeIncludeManifestItem(final T asset, final String reason) {
/**
* Write a line to the manifest file as an included item.
* @param asset The asset to be included
* @param evaluateReason The reason why the asset is being included
*/
public <T> void writeIncludeManifestItem(final T asset, final String evaluateReason) {
if (ManifestItem.class.isAssignableFrom(asset.getClass())) {
if (UtilMethods.isSet(manifestBuilder)) {
manifestBuilder.include((ManifestItem) asset, reason);
manifestBuilder.include((ManifestItem) asset, evaluateReason);
}
} else {
Logger.warn(PushPublisherConfig.class,
String.format("It is not possible add %s into the manifest", asset));
}
}

private <T> void writeExcludeManifestItem(final T asset, final String reason) {
/**
* Write a line to the manifest file as an excluded item.
* @param asset The asset to be excluded
* @param evaluateReason The reason why the asset is being evaluated
* @param excludeReason The reason why the asset is being excluded
*/
private <T> void writeExcludeManifestItem(final T asset, final String evaluateReason, final String excludeReason) {
if (ManifestItem.class.isAssignableFrom(asset.getClass())) {
if (UtilMethods.isSet(manifestBuilder)) {
manifestBuilder.exclude((ManifestItem) asset, reason);
manifestBuilder.exclude((ManifestItem) asset, evaluateReason, excludeReason);
}
} else {
Logger.warn(PushPublisherConfig.class,
Expand Down
72 changes: 52 additions & 20 deletions ...src/main/java/com/dotcms/publisher/util/dependencies/PushPublishigDependencyProcesor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -906,8 +906,17 @@ public void waitUntilResolveAllDependencies() throws ExecutionException {
dependencyProcessor.waitUntilResolveAllDependencies();
}

private <T> boolean add(final PusheableAsset pusheableAsset, final T asset, final String reason) {
final boolean isAdded = config.add(asset, pusheableAsset, reason);
/**
* This method will add the specified dotCMS to the bundle if it has not been added already.
* If the asses was already added to the bundle, then {@code false} will be returned,
* otherwise, the asset will be added to the bundle and {@code true} will be returned.
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param asset The actual dotCMS object that is being added.
* @param evaluateReason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for more details.
* @return If the asset was added to the bundle, {@code true} will be returned. Otherwise, {@code false} will be returned.
*/
private <T> boolean add(final PusheableAsset pusheableAsset, final T asset, final String evaluateReason) {
final boolean isAdded = config.add(asset, pusheableAsset, evaluateReason);

if (isAdded) {
pushedAssetUtil.savePushedAssetForAllEnv(asset, pusheableAsset);
Expand All @@ -916,27 +925,42 @@ private <T> boolean add(final PusheableAsset pusheableAsset, final T asset, fina
return isAdded;
}

/**
* This method tries to add the specified set of dotCMS asset to the bundle,
* it also processes the dependencies of the assets.
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param assets The actual dotCMS objects that are being added.
* @param evaluateReason The reason why this set of assets is being added to the bundle. Refer to {@link ManifestReason} for more details.
*/
private <T> void tryToAddAllAndProcessDependencies(
final PusheableAsset pusheableAsset, final Iterable<T> assets, final String reason)
final PusheableAsset pusheableAsset, final Iterable<T> assets, final String evaluateReason)
throws DotDataException, DotSecurityException {

if (UtilMethods.isSet(assets)) {
for (T asset : assets) {
tryToAddAndProcessDependencies(pusheableAsset, asset, reason);
tryToAddAndProcessDependencies(pusheableAsset, asset, evaluateReason);
}
}
}

/**
* This method tries to add the specified set of dotCMS assets to the bundle.
* For assets that cannot be added to the bundle, a new entry will be added to the bundle's MANIFEST file indicating the reason why.
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param assets The actual dotCMS objects that are being added.
* @param evaluateReason The reason why this set of assets is being added to the bundle. Refer to {@link ManifestReason} for more details.
* @return A collection of assets that were added to the bundle.
*/
private <T> Collection<T> tryToAddAll(final PusheableAsset pusheableAsset,
final Collection<T> assets, final String reason)
final Collection<T> assets, final String evaluateReason)
throws DotDataException, DotSecurityException {

if (assets != null) {
return assets.stream()
.filter(asset -> {
try {
final TryToAddResult tryToAddResult = tryToAdd(pusheableAsset, asset,
reason);
evaluateReason);
return TryToAddResult.Result.INCLUDE == tryToAddResult.result;
} catch (AssetExcludeException e) {
return false;
Expand All @@ -948,6 +972,14 @@ private <T> Collection<T> tryToAddAll(final PusheableAsset pusheableAsset,
}
}

/**
* This method tries to add the specified dotCMS asset dependency to the bundle.
* If the asset cannot be added to the bundle, a new entry will be added to the bundle's MANIFEST file indicating the reason why.
* Additionally, the very same asset will be added to the Dependency Processor queue in order to determine what other dependent assets might need to be added to the bundle as well.
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param dependency The actual dotCMS object that is being added.
* @param from The asset that is including the dependency.
*/
private void tryToAddAsDependency(final PusheableAsset pusheableAsset,
final ManifestItem dependency, final ManifestItem from)
throws DotDataException, DotSecurityException {
Expand All @@ -964,14 +996,14 @@ private void tryToAddAsDependency(final PusheableAsset pusheableAsset,
*
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param asset The actual dotCMS object that is being added.
* @param reason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for
* @param evaluateReason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for
* more details.
*/
private <T> void tryToAddAndProcessDependencies(final PusheableAsset pusheableAsset,
final T asset, final String reason) {
final T asset, final String evaluateReason) {
if (UtilMethods.isSet(asset)) {
try {
final TryToAddResult tryToAddResult = tryToAdd(pusheableAsset, asset, reason);
final TryToAddResult tryToAddResult = tryToAdd(pusheableAsset, asset, evaluateReason);

if (shouldIncludeDependency(tryToAddResult)) {
dependencyProcessor.addAsset(asset, pusheableAsset);
Expand Down Expand Up @@ -1000,19 +1032,19 @@ private static boolean shouldIncludeDependency(final TryToAddResult tryToAddResu
*
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param asset The actual dotCMS object that is being added.
* @param reason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for
* @param evaluateReason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for
* more details.
*
* @return If the asset was added to the bundle, {@code true} will be returned. Otherwise, {@code false} will be
* returned.
*/
private <T> boolean tryToAddSilently (
final PusheableAsset pusheableAsset, final T asset, final String reason) {
final PusheableAsset pusheableAsset, final T asset, final String evaluateReason) {
if (null == asset) {
return false;
}
try {
final TryToAddResult tryToAddResult = tryToAdd(pusheableAsset, asset, reason);
final TryToAddResult tryToAddResult = tryToAdd(pusheableAsset, asset, evaluateReason);
return TryToAddResult.Result.INCLUDE == tryToAddResult.result;
} catch (final AssetExcludeException e) {
Logger.debug(PushPublishigDependencyProcesor.class,
Expand All @@ -1027,7 +1059,7 @@ private <T> boolean tryToAddSilently (
*
* @param pusheableAsset The type of asset that is being added to the bundle.
* @param asset The actual dotCMS object that is being added.
* @param reason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for
* @param evaluateReason The reason why this asset is being added to the bundle. Refer to {@link ManifestReason} for
* more details.
*
* @return An instance of the {@link TryToAddResult} class indicating if the asset was included, excluded, and the
Expand All @@ -1036,7 +1068,7 @@ private <T> boolean tryToAddSilently (
* @throws AssetExcludeException
*/
private synchronized <T> TryToAddResult tryToAdd(final PusheableAsset pusheableAsset, final T asset,
final String reason)
final String evaluateReason)
throws AssetExcludeException {
if (config.contains(asset, pusheableAsset)) {
return new TryToAddResult(TryToAddResult.Result.ALREADY_INCLUDE);
Expand All @@ -1047,33 +1079,33 @@ private synchronized <T> TryToAddResult tryToAdd(final PusheableAsset pusheableA
}

if (isSystemObject(asset)) {
config.exclude(asset, pusheableAsset, ManifestReason.EXCLUDE_SYSTEM_OBJECT.getMessage());
config.exclude(asset, pusheableAsset, evaluateReason, ManifestReason.EXCLUDE_SYSTEM_OBJECT.getMessage());
return new TryToAddResult(TryToAddResult.Result.EXCLUDE, ManifestReason.EXCLUDE_SYSTEM_OBJECT);
}

if (!isTemplateLayout(asset) && isExcludeByFilter(pusheableAsset)) {
config.exclude(asset, pusheableAsset, ManifestReason.EXCLUDE_BY_FILTER.getMessage());
config.exclude(asset, pusheableAsset, evaluateReason, ManifestReason.EXCLUDE_BY_FILTER.getMessage());
return new TryToAddResult(TryToAddResult.Result.EXCLUDE, ManifestReason.EXCLUDE_BY_FILTER);
}

if ( config.getOperation() != Operation.PUBLISH ) {
config.exclude(asset, pusheableAsset, ManifestReason.EXCLUDE_BY_OPERATION.getMessage(config.getOperation()));
config.exclude(asset, pusheableAsset, evaluateReason, ManifestReason.EXCLUDE_BY_OPERATION.getMessage(config.getOperation()));
return new TryToAddResult(TryToAddResult.Result.EXCLUDE, ManifestReason.EXCLUDE_BY_OPERATION);
}

if (Contentlet.class.isInstance(asset) && !Contentlet.class.cast(asset).isHost() &&
publisherFilter.doesExcludeDependencyQueryContainsContentletId(
((Contentlet) asset).getIdentifier())) {
config.exclude(asset, pusheableAsset, ManifestReason.EXCLUDE_BY_FILTER.getMessage());
config.exclude(asset, pusheableAsset, evaluateReason, ManifestReason.EXCLUDE_BY_FILTER.getMessage());
return new TryToAddResult(TryToAddResult.Result.EXCLUDE, ManifestReason.EXCLUDE_BY_FILTER);
}

if (!shouldCheckModDate(asset) ||
!dependencyModDateUtil.excludeByModDate(asset, pusheableAsset)) {
add(pusheableAsset, asset, reason);
add(pusheableAsset, asset, evaluateReason);
return new TryToAddResult(TryToAddResult.Result.INCLUDE);
} else {
config.exclude(asset, pusheableAsset,
config.exclude(asset, pusheableAsset, evaluateReason,
ManifestReason.EXCLUDE_BY_MOD_DATE.getMessage(asset.getClass()));
return new TryToAddResult(TryToAddResult.Result.EXCLUDE, ManifestReason.EXCLUDE_BY_MOD_DATE);
}
Expand Down
Loading

0 comments on commit b59cf7d

Please sign in to comment.