Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
Expand All @@ -59,13 +58,13 @@
import org.apache.maven.buildcache.RemoteCacheRepository;
import org.apache.maven.buildcache.ScanConfigProperties;
import org.apache.maven.buildcache.Xpp3DomUtils;
import org.apache.maven.buildcache.checksum.exclude.ExclusionResolver;
import org.apache.maven.buildcache.hash.HashAlgorithm;
import org.apache.maven.buildcache.hash.HashChecksum;
import org.apache.maven.buildcache.xml.CacheConfig;
import org.apache.maven.buildcache.xml.DtoUtils;
import org.apache.maven.buildcache.xml.build.DigestItem;
import org.apache.maven.buildcache.xml.build.ProjectsInputInfo;
import org.apache.maven.buildcache.xml.config.Exclude;
import org.apache.maven.buildcache.xml.config.Include;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.model.Dependency;
Expand Down Expand Up @@ -108,21 +107,11 @@ public class MavenProjectInput {
* property name to pass glob value. The glob to be used to list directory files in plugins scanning
*/
private static final String CACHE_INPUT_GLOB_NAME = "maven.build.cache.input.glob";
/**
* default glob, bbsdk/abfx specific
*/
public static final String DEFAULT_GLOB = "{*.java,*.groovy,*.yaml,*.svcd,*.proto,*assembly.xml,assembly"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was not used except in a UT.

+ "*.xml,*logback.xml,*.vm,*.ini,*.jks,*.properties,*.sh,*.bat}";
/**
* property name prefix to pass input files with project properties. smth like maven.build.cache.input.1 will be
* accepted
*/
private static final String CACHE_INPUT_NAME = "maven.build.cache.input";
/**
* property name prefix to exclude files from input. smth like maven.build.cache.exclude.1 should be set in project
* props
*/
private static final String CACHE_EXCLUDE_NAME = "maven.build.cache.exclude";
/**
* Flag to control if we should check values from plugin configs as file system objects
*/
Expand All @@ -136,12 +125,17 @@ public class MavenProjectInput {
private final RepositorySystem repoSystem;
private final CacheConfig config;
private final PathIgnoringCaseComparator fileComparator;
private final List<Path> filteredOutPaths;
private final NormalizedModelProvider normalizedModelProvider;
private final MultiModuleSupport multiModuleSupport;
private final ProjectInputCalculator projectInputCalculator;
private final Path baseDirPath;
private final String dirGlob;
/**
* The project glob to use every time there is no override
*/
private final String projectGlob;

private final ExclusionResolver exclusionResolver;

private final boolean processPlugins;
private final String tmpDir;

Expand All @@ -165,41 +159,12 @@ public MavenProjectInput(
this.repoSystem = repoSystem;
this.remoteCache = remoteCache;
Properties properties = project.getProperties();
this.dirGlob = properties.getProperty(CACHE_INPUT_GLOB_NAME, config.getDefaultGlob());
this.projectGlob = properties.getProperty(CACHE_INPUT_GLOB_NAME, config.getDefaultGlob());
this.processPlugins =
Boolean.parseBoolean(properties.getProperty(CACHE_PROCESS_PLUGINS, config.isProcessPlugins()));
this.tmpDir = System.getProperty("java.io.tmpdir");

org.apache.maven.model.Build build = project.getBuild();
filteredOutPaths = new ArrayList<>(Arrays.asList(
normalizedPath(build.getDirectory()), // target by default
normalizedPath(build.getOutputDirectory()),
normalizedPath(build.getTestOutputDirectory())));

List<Exclude> excludes = config.getGlobalExcludePaths();
for (Exclude excludePath : excludes) {
filteredOutPaths.add(Paths.get(excludePath.getValue()));
}

for (String propertyName : properties.stringPropertyNames()) {
if (propertyName.startsWith(CACHE_EXCLUDE_NAME)) {
String propertyValue = properties.getProperty(propertyName);
Path path = Paths.get(propertyValue);
filteredOutPaths.add(path);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
"Adding an excludePath from property '{}', values is '{}', path is '{}' ",
propertyName,
propertyValue,
path);
}
}
}
CacheUtils.debugPrintCollection(
LOGGER,
filteredOutPaths,
"List of excluded paths (checked either by fileName or by startsWith prefix)",
"Path entry");
this.exclusionResolver = new ExclusionResolver(project, config);

this.fileComparator = new PathIgnoringCaseComparator();
}
Expand Down Expand Up @@ -354,28 +319,28 @@ private SortedSet<Path> getInputFiles() {
org.apache.maven.model.Build build = project.getBuild();

final boolean recursive = true;
startWalk(Paths.get(build.getSourceDirectory()), dirGlob, recursive, collectedFiles, visitedDirs);
startWalk(Paths.get(build.getSourceDirectory()), projectGlob, recursive, collectedFiles, visitedDirs);
for (Resource resource : build.getResources()) {
startWalk(Paths.get(resource.getDirectory()), dirGlob, recursive, collectedFiles, visitedDirs);
startWalk(Paths.get(resource.getDirectory()), projectGlob, recursive, collectedFiles, visitedDirs);
}

startWalk(Paths.get(build.getTestSourceDirectory()), dirGlob, recursive, collectedFiles, visitedDirs);
startWalk(Paths.get(build.getTestSourceDirectory()), projectGlob, recursive, collectedFiles, visitedDirs);
for (Resource testResource : build.getTestResources()) {
startWalk(Paths.get(testResource.getDirectory()), dirGlob, recursive, collectedFiles, visitedDirs);
startWalk(Paths.get(testResource.getDirectory()), projectGlob, recursive, collectedFiles, visitedDirs);
}

Properties properties = project.getProperties();
for (String name : properties.stringPropertyNames()) {
if (name.startsWith(CACHE_INPUT_NAME)) {
String path = properties.getProperty(name);
startWalk(Paths.get(path), dirGlob, recursive, collectedFiles, visitedDirs);
startWalk(Paths.get(path), projectGlob, recursive, collectedFiles, visitedDirs);
}
}

List<Include> includes = config.getGlobalIncludePaths();
for (Include include : includes) {
final String path = include.getValue();
final String glob = defaultIfEmpty(include.getGlob(), dirGlob);
final String glob = defaultIfEmpty(include.getGlob(), projectGlob);
startWalk(Paths.get(path), glob, include.isRecursive(), collectedFiles, visitedDirs);
}

Expand Down Expand Up @@ -410,13 +375,17 @@ private SortedSet<Path> getInputFiles() {
return sorted;
}

private Path convertToAbsolutePath(Path path) {
Path resolvedPath = path.isAbsolute() ? path : baseDirPath.resolve(path);
return resolvedPath.toAbsolutePath().normalize();
}

/**
* entry point for directory walk
*/
private void startWalk(
Path candidate, String glob, boolean recursive, List<Path> collectedFiles, Set<WalkKey> visitedDirs) {
Path normalized = candidate.isAbsolute() ? candidate : baseDirPath.resolve(candidate);
normalized = normalized.toAbsolutePath().normalize();
Path normalized = convertToAbsolutePath(candidate);
WalkKey key = new WalkKey(normalized, glob, recursive);
if (visitedDirs.contains(key) || !Files.exists(normalized)) {
return;
Expand All @@ -433,17 +402,13 @@ private void startWalk(
throw new RuntimeException(e);
}
} else {
if (!isFilteredOutSubpath(normalized)) {
if (!exclusionResolver.excludesPath(normalized)) {
LOGGER.debug("Adding: {}", normalized);
collectedFiles.add(normalized);
}
}
}

private Path normalizedPath(String directory) {
return Paths.get(directory).normalize();
}

private void collectFromPlugins(List<Path> files, HashSet<WalkKey> visitedDirs) {
List<Plugin> plugins = project.getBuild().getPlugins();
for (Plugin plugin : plugins) {
Expand Down Expand Up @@ -498,16 +463,15 @@ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFil
} else if (!isReadable(path)) {
LOGGER.debug("Skipping subtree (not readable): {}", path);
return FileVisitResult.SKIP_SUBTREE;
} else if (isFilteredOutSubpath(path)) {
} else if (exclusionResolver.excludesPath(path)) {
LOGGER.debug("Skipping subtree (blacklisted): {}", path);
return FileVisitResult.SKIP_SUBTREE;
} else if (visitedDirs.contains(currentDirKey)) {
LOGGER.debug("Skipping subtree (visited): {}", path);
return FileVisitResult.SKIP_SUBTREE;
}

walkDirectoryFiles(path, collectedFiles, key.getGlob(), entry -> filteredOutPaths.stream()
.anyMatch(it -> it.getFileName().equals(entry.getFileName())));
walkDirectoryFiles(path, collectedFiles, key.getGlob(), entry -> exclusionResolver.excludesPath(entry));

if (!key.isRecursive()) {
LOGGER.debug("Skipping subtree (non recursive): {}", path);
Expand Down Expand Up @@ -549,7 +513,7 @@ private void addInputsFromPluginConfigs(
addInputsFromPluginConfigs(Xpp3DomUtils.getChildren(configChild), scanConfig, files, visitedDirs);

final ScanConfigProperties propertyConfig = scanConfig.getTagScanProperties(tagName);
final String glob = defaultIfEmpty(propertyConfig.getGlob(), dirGlob);
final String glob = defaultIfEmpty(propertyConfig.getGlob(), projectGlob);
if ("true".equals(Xpp3DomUtils.getAttribute(configChild, CACHE_INPUT_NAME))) {
LOGGER.info(
"Found tag marked with {} attribute. Tag: {}, value: {}", CACHE_INPUT_NAME, tagName, tagValue);
Expand Down Expand Up @@ -632,16 +596,6 @@ private static boolean isReadable(Path entry) throws IOException {
return Files.isReadable(entry);
}

private boolean isFilteredOutSubpath(Path path) {
Path normalized = path.normalize();
for (Path filteredOutDir : filteredOutPaths) {
if (normalized.startsWith(filteredOutDir)) {
return true;
}
}
return false;
}

private SortedMap<String, String> getMutableDependencies() throws IOException {
SortedMap<String, String> result = new TreeMap<>();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.maven.buildcache.checksum.exclude;

import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.maven.buildcache.xml.config.Exclude;

public class Exclusion {

/**
* Glob prefix for path matchers
*/
private static final String GLOB_PX = "glob:";

private static final String GLOB_ALL_PATHS = "**";
private static final String GLOB_ALL_NAMES = "*";

/**
* Default glob
*/
private static final String DEFAULT_GLOB = GLOB_ALL_PATHS;

private final Path absolutePath;
private final PathMatcher matcher;

private final MatcherType matcherType;

private final EntryType entryType;

/**
* Denormalization to increase pathmatching resolution speed if the glob obviously match all files
*/
private boolean matchesAllNames;

/**
* Denormalization to increase pathmatching resolution speed if the glob obviously match all paths
*/
private boolean matchesAllPaths;

/**
* True if the configured value was already an absolute path
*/
private final boolean configuredAsAbsolute;

public Exclusion(Path basedir, Exclude exclude) {

if (StringUtils.isNotBlank(exclude.getValue())) {
Path candidate = Paths.get(FilenameUtils.separatorsToSystem(exclude.getValue()));
configuredAsAbsolute = candidate.isAbsolute();
Path resolvedPath = configuredAsAbsolute ? candidate : basedir.resolve(candidate);
this.absolutePath = resolvedPath.toAbsolutePath().normalize();
} else {
configuredAsAbsolute = false;
this.absolutePath = basedir;
}
// Unix style glob is correctly interpreted on windows by the corresponding pathMatcher implementation.
String unixStyleGlob = convertGlobToUnixStyle(exclude.getGlob());
this.matcher = FileSystems.getDefault().getPathMatcher(GLOB_PX + unixStyleGlob);
this.matcherType = MatcherType.valueOf(exclude.getMatcherType().toUpperCase());
this.entryType = EntryType.valueOf(exclude.getEntryType().toUpperCase());
computeMatcherDenormalization(unixStyleGlob);
}

public Exclusion(Path absolutePath, MatcherType resolutionType, EntryType entryType) {
this.configuredAsAbsolute = false;
this.absolutePath = absolutePath;
this.matcher = absolutePath.getFileSystem().getPathMatcher(GLOB_PX + DEFAULT_GLOB);
this.matcherType = resolutionType;
this.entryType = entryType;
computeMatcherDenormalization(DEFAULT_GLOB);
}

private String convertGlobToUnixStyle(String glob) {
return glob.replace("\\\\", "/");
}

private void computeMatcherDenormalization(String glob) {
if (GLOB_ALL_PATHS.equals(glob)) {
matchesAllPaths = true;
} else if (GLOB_ALL_NAMES.equals(glob)) {
matchesAllNames = true;
}
}

public Path getAbsolutePath() {
return absolutePath;
}

public EntryType getEntryType() {
return entryType;
}

/**
* True if the exclusion applies to the given path (does not indicate that the path is excluded)
* @param path a visited path
* @return true if the exclusion applies to the given path
*/
private boolean applies(Path path) {
return path.startsWith(this.absolutePath);
}

public boolean excludesPath(Path parentPath, Path path) {
if (applies(path)) {
switch (matcherType) {
case FILENAME:
if (matchesAllPaths || matchesAllNames || matcher.matches(path.getFileName())) {
return true;
}
break;
case PATH:
// If path is configured relative, matching has to be done relatively to the project directory in
// order to be independent from
// the project location on the disk.
if (matchesAllPaths || matcher.matches(configuredAsAbsolute ? path : parentPath.relativize(path))) {
return true;
}
break;
default:
throw new RuntimeException("Exclusion resolution type not handled.");
}
}
return false;
}

public enum MatcherType {
FILENAME,
PATH;
}

public enum EntryType {
FILE,
DIRECTORY,
ALL;
}
}
Loading