Skip to content

Commit

Permalink
Concurrent build support
Browse files Browse the repository at this point in the history
  • Loading branch information
@gnodet
gnodet committed Sep 21, 2019
0 parents commit 844f3dd
Show file tree
Hide file tree
Showing 37 changed files with 2,396 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
.idea
target
*.iml
142 changes: 142 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

<modelVersion>4.0.0</modelVersion>

<groupId>org.jboss.fuse.mvnd</groupId>
<artifactId>mvnd</artifactId>
<version>0.1-SNAPSHOT</version>

<packaging>takari-maven-component</packaging>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

<jlineVersion>3.12.1</jlineVersion>
<junitVersion>4.12</junitVersion>
<logbackVersion>1.2.3</logbackVersion>
<mavenVersion>3.6.2</mavenVersion>
<pluginTestingVersion>2.9.2</pluginTestingVersion>
<slf4jVersion>1.7.28</slf4jVersion>
<takariLifecycleVersion>1.13.9</takariLifecycleVersion>
<takariProvisioVersion>0.1.56</takariProvisioVersion>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-embedder</artifactId>
<version>${mavenVersion}</version>
<scope>provided</scope>
</dependency>

<!-- Logging -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logbackVersion}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>${slf4jVersion}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4jVersion}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
<version>${slf4jVersion}</version>
</dependency>

<dependency>
<groupId>org.jline</groupId>
<artifactId>jline-terminal</artifactId>
<version>${jlineVersion}</version>
</dependency>

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junitVersion}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.takari.maven.plugins</groupId>
<artifactId>takari-plugin-testing</artifactId>
<version>${pluginTestingVersion}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.takari.maven.plugins</groupId>
<artifactId>takari-plugin-integration-testing</artifactId>
<version>${pluginTestingVersion}</version>
<type>pom</type>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>io.takari.maven.plugins</groupId>
<artifactId>takari-lifecycle-plugin</artifactId>
<version>${takariLifecycleVersion}</version>
<extensions>true</extensions>
<configuration>
<proc>none</proc>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>maven-distro</id>
<build>
<plugins>
<plugin>
<groupId>io.takari.maven.plugins</groupId>
<artifactId>provisio-maven-plugin</artifactId>
<version>${takariProvisioVersion}</version>
<executions>
<execution>
<id>maven-distro</id>
<phase>package</phase>
<goals>
<goal>provision</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/maven-distro</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<executions>
<execution>
<id>integration-test</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<includes>**/*IT.java</includes>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

</project>
21 changes: 21 additions & 0 deletions src/main/distro/bin/mvnd
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/sh

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

"`dirname "$0"`/mvn" --builder smart --threads 8 --define buildtime.output.log --define maven.logging=smart "$@"

50 changes: 50 additions & 0 deletions src/main/java/org/jboss/fuse/mvnd/builder/DependencyGraph.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package org.jboss.fuse.mvnd.builder;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.maven.execution.ProjectDependencyGraph;
import org.apache.maven.project.MavenProject;

interface DependencyGraph<K> {

static DependencyGraph<MavenProject> fromMaven(ProjectDependencyGraph graph) {
List<MavenProject> projects = graph.getAllProjects();
Map<MavenProject, List<MavenProject>> upstreams = projects.stream()
.collect(Collectors.toMap(p -> p, p -> graph.getUpstreamProjects(p, false)));
Map<MavenProject, List<MavenProject>> downstreams = projects.stream()
.collect(Collectors.toMap(p -> p, p -> graph.getDownstreamProjects(p, false)));
return new DependencyGraph<MavenProject>() {
@Override
public Stream<MavenProject> getDownstreamProjects(MavenProject project) {
return downstreams.get(project).stream();
}

@Override
public Stream<MavenProject> getProjects() {
return projects.stream();
}

@Override
public Stream<MavenProject> getUpstreamProjects(MavenProject project) {
return upstreams.get(project).stream();
}

@Override
public boolean isRoot(MavenProject project) {
return upstreams.get(project).isEmpty();
}
};
}

Stream<K> getProjects();

boolean isRoot(K project);

Stream<K> getDownstreamProjects(K project);

Stream<K> getUpstreamProjects(K project);

}
113 changes: 113 additions & 0 deletions src/main/java/org/jboss/fuse/mvnd/builder/ProjectComparator.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package org.jboss.fuse.mvnd.builder;

import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.function.ToLongFunction;

import org.apache.maven.project.MavenProject;

/**
* Project comparator (factory) that uses project build time to establish build order.
* <p>
* Internally, each project is assigned a weight, which is calculated as sum of project build time
* and maximum weight of any of the project's downstream dependencies. The project weights are
* calculated by recursively traversing project dependency graph starting from build root projects,
* i.e. projects that do not have any upstream dependencies.
* <p>
* Project build times are estimated based on values persisted during a previous build. Average
* build time is used for projects that do not have persisted build time.
* <p>
* If there are no persisted build times, all projects build times are assumed the same (arbitrary)
* value of 1. This means that the project with the longest downstream dependency trail will be
* built first.
* <p>
* Currently, historical build times are stored in
* <code>${session.request/baseDirectory}/.mvn/timing.properties</code> file. The timings file is
* written only if <code>${session.request/baseDirectory}/.mvn</code> directory is already present.
*/
class ProjectComparator {

public static Comparator<MavenProject> create(DependencyGraph<MavenProject> graph) {
return create0(graph, Collections.emptyMap(), ProjectComparator::id);
}

static <K> Comparator<K> create0(DependencyGraph<K> dependencyGraph,
Map<String, AtomicLong> historicalServiceTimes,
Function<K, String> toKey) {
final long defaultServiceTime = average(historicalServiceTimes.values());

final Map<K, Long> serviceTimes = new HashMap<>();

final Set<K> rootProjects = new HashSet<>();
dependencyGraph.getProjects().forEach(project -> {
long serviceTime = getServiceTime(historicalServiceTimes, project, defaultServiceTime, toKey);
serviceTimes.put(project, serviceTime);
if (dependencyGraph.isRoot(project)) {
rootProjects.add(project);
}
});

final Map<K, Long> projectWeights =
calculateWeights(dependencyGraph, serviceTimes, rootProjects);

return Comparator.comparingLong((ToLongFunction<K>) projectWeights::get)
.thenComparing(toKey, String::compareTo)
.reversed();
}

private static long average(Collection<AtomicLong> values) {
return (long) (values.stream().mapToLong(AtomicLong::longValue)
.average().orElse(1.0d));
}

private static <K> long getServiceTime(Map<String, AtomicLong> serviceTimes, K project,
long defaultServiceTime, Function<K, String> toKey) {
AtomicLong serviceTime = serviceTimes.get(toKey.apply(project));
return serviceTime != null ? serviceTime.longValue() : defaultServiceTime;
}

private static <K> Map<K, Long> calculateWeights(DependencyGraph<K> dependencyGraph,
Map<K, Long> serviceTimes, Collection<K> rootProjects) {
Map<K, Long> weights = new HashMap<>();
for (K rootProject : rootProjects) {
calculateWeights(dependencyGraph, serviceTimes, rootProject, weights);
}
return weights;
}

/**
* Returns the maximum sum of build time along a path from the project to an exit project. An
* "exit project" is a project without downstream dependencies.
*/
private static <K> long calculateWeights(DependencyGraph<K> dependencyGraph,
Map<K, Long> serviceTimes, K project, Map<K, Long> weights) {
long weight = serviceTimes.get(project)
+ dependencyGraph.getDownstreamProjects(project)
.mapToLong(successor -> {
long successorWeight;
if (weights.containsKey(successor)) {
successorWeight = weights.get(successor);
} else {
successorWeight = calculateWeights(dependencyGraph, serviceTimes, successor, weights);
}
return successorWeight;
})
.max().orElse(0);
weights.put(project, weight);
return weight;
}

static String id(MavenProject project) {
return project.getGroupId() +
':' + project.getArtifactId() +
':' + project.getVersion();
}

}
Loading

0 comments on commit 844f3dd

Please sign in to comment.