Add support for unpacking nested JARs

Update the executable JAR code to automatically unpack any entries
which include an entry comment starting `UNPACK:` to the temp folder.
The existing Maven and Gradle plugins have been updated with new
configuration options and the `spring-boot-tools` project has been
updated to write the appropriate entry comment based on a flag passed
in via the `Library` class.

This support has been added to allow libraries such a JRuby (which
assumes that `jruby-complete.jar` is always accessible as file) to work
with Spring Boot executable jars.

Fixes gh-1070

Author: Phillip Webb

spring-boot-cli/src/main/java/org/springframework/boot/cli/command/jar/JarCommand.java

@@ -56,6 +56,8 @@
 import org.springframework.boot.loader.tools.JarWriter;
 import org.springframework.boot.loader.tools.Layout;
 import org.springframework.boot.loader.tools.Layouts;
+import org.springframework.boot.loader.tools.Library;
+import org.springframework.boot.loader.tools.LibraryScope;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.springframework.util.Assert;
@@ -248,7 +250,8 @@ private void addDependencies(JarWriter writer, List<URL> urls)
 		private void addDependency(JarWriter writer, File dependency)
 				throws FileNotFoundException, IOException {
 			if (dependency.isFile()) {
-				writer.writeNestedLibrary("lib/", dependency);
+				writer.writeNestedLibrary("lib/", new Library(dependency,
+						LibraryScope.COMPILE));
 			}
 		}
 

spring-boot-docs/src/main/asciidoc/build-tool-plugins.adoc

@@ -511,6 +511,11 @@ The following configuration options are available:
 |`layout`
 |The type of archive, corresponding to how the dependencies are laid out inside
  (defaults to a guess based on the archive type).
+
+|`requiresUnpack`
+|A list of dependencies (in the form ``groupId:artifactId'' that must be unpacked from
+ fat jars in order to run. Items are still packaged into the fat jar, but they will be
+ automatically unpacked when it runs.
 |===
 
 

spring-boot-docs/src/main/asciidoc/howto.adoc

@@ -1618,6 +1618,50 @@ For Gradle users the steps are similar. Example:
 
 
 
+[[howto-extract-specific-libraries-when-an-executable-jar-runs]]
+=== Extract specific libraries when an executable jar runs
+Most nested libraries in an executable jar do not need to be unpacked in order to run,
+however, certain libraries can have problems. For example, JRuby includes its own nested
+jar support which assumes that the `jruby-complete.jar` is always directly available as a
+file in its own right.
+
+To deal with any problematic libraries, you can flag that specific nested jars should be
+automatically unpacked to the ``temp folder'' when the executable jar first runs.
+
+For example, to indicate that JRuby should be flagged for unpack using the Maven Plugin
+you would add the following configuration:
+
+[source,xml,indent=0,subs="verbatim,quotes,attributes"]
+----
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+				<configuration>
+					<requiresUnpack>
+						<dependency>
+							<groupId>org.jruby</groupId>
+							<artifactId>jruby-complete</artifactId>
+						</dependency>
+					</requiresUnpack>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+----
+
+And to do that same with Gradle:
+
+[source,groovy,indent=0,subs="verbatim,attributes"]
+----
+	springBoot  {
+		requiresUnpack = ['org.jruby:jruby-complete']
+	}
+----
+
+
+
 [[howto-create-a-nonexecutable-jar]]
 === Create a non-executable JAR with exclusions
 Often if you have an executable and a non-executable jar as build products, the executable

spring-boot-tools/spring-boot-gradle-plugin/src/main/groovy/org/springframework/boot/gradle/SpringBootPluginExtension.groovy

@@ -107,6 +107,12 @@ public class SpringBootPluginExtension {
 		(layout == null ? null : layout.layout)
 	}
 
+	/**
+	 * Libraries that must be unpacked from fat jars in order to run. Use Strings in the
+	 * form {@literal groupId:artifactId}.
+	 */
+	Set<String> requiresUnpack;
+
 	/**
 	 * Location of an agent jar to attach to the VM when running the application with runJar task.
 	 */
@@ -121,4 +127,5 @@ public class SpringBootPluginExtension {
 	 * If exclude rules should be applied to dependencies based on the spring-dependencies-bom
 	 */
 	boolean applyExcludeRules = true;
+
 }

spring-boot-tools/spring-boot-gradle-plugin/src/main/groovy/org/springframework/boot/gradle/repackage/ProjectLibraries.java

@@ -18,10 +18,15 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
 
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.file.FileCollection;
+import org.gradle.api.artifacts.ModuleVersionIdentifier;
+import org.gradle.api.artifacts.ResolvedArtifact;
+import org.springframework.boot.gradle.SpringBootPluginExtension;
 import org.springframework.boot.loader.tools.Libraries;
 import org.springframework.boot.loader.tools.Library;
 import org.springframework.boot.loader.tools.LibraryCallback;
@@ -37,22 +42,24 @@ class ProjectLibraries implements Libraries {
 
 	private final Project project;
 
+	private final SpringBootPluginExtension extension;
+
 	private String providedConfigurationName = "providedRuntime";
 
 	private String customConfigurationName = null;
 
 	/**
 	 * Create a new {@link ProjectLibraries} instance of the specified {@link Project}.
-	 *
 	 * @param project the gradle project
+	 * @param extension the extension
 	 */
-	public ProjectLibraries(Project project) {
+	public ProjectLibraries(Project project, SpringBootPluginExtension extension) {
 		this.project = project;
+		this.extension = extension;
 	}
 
 	/**
 	 * Set the name of the provided configuration. Defaults to 'providedRuntime'.
-	 *
 	 * @param providedConfigurationName the providedConfigurationName to set
 	 */
 	public void setProvidedConfigurationName(String providedConfigurationName) {
@@ -65,27 +72,20 @@ public void setCustomConfigurationName(String customConfigurationName) {
 
 	@Override
 	public void doWithLibraries(LibraryCallback callback) throws IOException {
-
-		FileCollection custom = this.customConfigurationName != null ? this.project
-				.getConfigurations().findByName(this.customConfigurationName) : null;
-
+		Set<ResolvedArtifact> custom = getArtifacts(this.customConfigurationName);
 		if (custom != null) {
 			libraries(LibraryScope.CUSTOM, custom, callback);
 		}
 		else {
-			FileCollection compile = this.project.getConfigurations()
-					.getByName("compile");
-
-			FileCollection runtime = this.project.getConfigurations()
-					.getByName("runtime");
-			runtime = runtime.minus(compile);
+			Set<ResolvedArtifact> compile = getArtifacts("compile");
 
-			FileCollection provided = this.project.getConfigurations()
-					.findByName(this.providedConfigurationName);
+			Set<ResolvedArtifact> runtime = getArtifacts("runtime");
+			runtime = minus(runtime, compile);
 
+			Set<ResolvedArtifact> provided = getArtifacts(this.providedConfigurationName);
 			if (provided != null) {
-				compile = compile.minus(provided);
-				runtime = runtime.minus(provided);
+				compile = minus(compile, provided);
+				runtime = minus(runtime, provided);
 			}
 
 			libraries(LibraryScope.COMPILE, compile, callback);
@@ -94,12 +94,47 @@ public void doWithLibraries(LibraryCallback callback) throws IOException {
 		}
 	}
 
-	private void libraries(LibraryScope scope, FileCollection files,
+	private Set<ResolvedArtifact> getArtifacts(String configurationName) {
+		Configuration configuration = (configurationName == null ? null : this.project
+				.getConfigurations().findByName(configurationName));
+		return (configuration == null ? null : configuration.getResolvedConfiguration()
+				.getResolvedArtifacts());
+	}
+
+	private Set<ResolvedArtifact> minus(Set<ResolvedArtifact> source,
+			Set<ResolvedArtifact> toRemove) {
+		if (source == null || toRemove == null) {
+			return source;
+		}
+		Set<File> filesToRemove = new HashSet<File>();
+		for (ResolvedArtifact artifact : toRemove) {
+			filesToRemove.add(artifact.getFile());
+		}
+		Set<ResolvedArtifact> result = new LinkedHashSet<ResolvedArtifact>();
+		for (ResolvedArtifact artifact : source) {
+			if (!toRemove.contains(artifact.getFile())) {
+				result.add(artifact);
+			}
+		}
+		return result;
+	}
+
+	private void libraries(LibraryScope scope, Set<ResolvedArtifact> artifacts,
 			LibraryCallback callback) throws IOException {
-		if (files != null) {
-			for (File file: files) {
-				callback.library(new Library(file, scope));
+		if (artifacts != null) {
+			for (ResolvedArtifact artifact : artifacts) {
+				callback.library(new Library(artifact.getFile(), scope, isUnpackRequired(artifact)));
 			}
 		}
 	}
+
+	private boolean isUnpackRequired(ResolvedArtifact artifact) {
+		if (this.extension.getRequiresUnpack() != null) {
+			ModuleVersionIdentifier id = artifact.getModuleVersion().getId();
+			return this.extension.getRequiresUnpack().contains(
+					id.getGroup() + ":" + id.getName());
+		}
+		return false;
+	}
+
 }

spring-boot-tools/spring-boot-gradle-plugin/src/main/groovy/org/springframework/boot/gradle/repackage/RepackageTask.java

@@ -101,7 +101,7 @@ public ProjectLibraries getLibraries() {
 		Project project = getProject();
 		SpringBootPluginExtension extension = project.getExtensions().getByType(
 				SpringBootPluginExtension.class);
-		ProjectLibraries libraries = new ProjectLibraries(project);
+		ProjectLibraries libraries = new ProjectLibraries(project, extension);
 		if (extension.getProvidedConfiguration() != null) {
 			libraries.setProvidedConfigurationName(extension.getProvidedConfiguration());
 		}

spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/FileUtils.java

@@ -17,13 +17,19 @@
 package org.springframework.boot.loader.tools;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 
 /**
  * Utilities for manipulating files and directories in Spring Boot tooling.
  * 
  * @author Dave Syer
+ * @author Phillip Webb
  */
-public class FileUtils {
+public abstract class FileUtils {
 
 	/**
 	 * Utility to remove duplicate files from an "output" directory if they already exist
@@ -50,4 +56,37 @@ public static void removeDuplicatesFromOutputDirectory(File outputDirectory,
 		}
 	}
 
+	/**
+	 * Generate a SHA.1 Hash for a given file.
+	 * @param file the file to hash
+	 * @return the hash value as a String
+	 * @throws IOException
+	 */
+	public static String sha1Hash(File file) throws IOException {
+		try {
+			DigestInputStream inputStream = new DigestInputStream(new FileInputStream(
+					file), MessageDigest.getInstance("SHA-1"));
+			try {
+				byte[] buffer = new byte[4098];
+				while (inputStream.read(buffer) != -1) {
+					// Read the entire stream
+				}
+				return bytesToHex(inputStream.getMessageDigest().digest());
+			}
+			finally {
+				inputStream.close();
+			}
+		}
+		catch (NoSuchAlgorithmException ex) {
+			throw new IllegalStateException(ex);
+		}
+	}
+
+	private static String bytesToHex(byte[] bytes) {
+		StringBuilder hex = new StringBuilder();
+		for (byte b : bytes) {
+			hex.append(String.format("%02x", b));
+		}
+		return hex.toString();
+	}
 }

spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/JarWriter.java

@@ -50,7 +50,7 @@ public class JarWriter {
 
 	private static final String NESTED_LOADER_JAR = "META-INF/loader/spring-boot-loader.jar";
 
-	private static final int BUFFER_SIZE = 4096;
+	private static final int BUFFER_SIZE = 32 * 1024;
 
 	private final JarOutputStream jarOutput;
 
@@ -122,11 +122,16 @@ public void writeEntry(String entryName, InputStream inputStream) throws IOExcep
 	/**
 	 * Write a nested library.
 	 * @param destination the destination of the library
-	 * @param file the library file
+	 * @param library the library
 	 * @throws IOException if the write fails
 	 */
-	public void writeNestedLibrary(String destination, File file) throws IOException {
+	public void writeNestedLibrary(String destination, Library library)
+			throws IOException {
+		File file = library.getFile();
 		JarEntry entry = new JarEntry(destination + file.getName());
+		if (library.isUnpackRequired()) {
+			entry.setComment("UNPACK:" + FileUtils.sha1Hash(file));
+		}
 		new CrcAndSize(file).setupStoredEntry(entry);
 		writeEntry(entry, new InputStreamEntryWriter(new FileInputStream(file), true));
 	}

spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/Library.java

@@ -31,14 +31,27 @@ public class Library {
 
 	private final LibraryScope scope;
 
+	private final boolean unpackRequired;
+
 	/**
 	 * Create a new {@link Library}.
 	 * @param file the source file
 	 * @param scope the scope of the library
 	 */
 	public Library(File file, LibraryScope scope) {
+		this(file, scope, false);
+	}
+
+	/**
+	 * Create a new {@link Library}.
+	 * @param file the source file
+	 * @param scope the scope of the library
+	 * @param unpackRequired if the library needs to be unpacked before it can be used
+	 */
+	public Library(File file, LibraryScope scope, boolean unpackRequired) {
 		this.file = file;
 		this.scope = scope;
+		this.unpackRequired = unpackRequired;
 	}
 
 	/**
@@ -55,4 +68,12 @@ public LibraryScope getScope() {
 		return this.scope;
 	}
 
+	/**
+	 * @return if the file cannot be used directly as a nested jar and needs to be
+	 * unpacked.
+	 */
+	public boolean isUnpackRequired() {
+		return this.unpackRequired;
+	}
+
 }

spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/Repackager.java

@@ -147,7 +147,7 @@ public void library(Library library) throws IOException {
 						String destination = Repackager.this.layout
 								.getLibraryDestination(file.getName(), library.getScope());
 						if (destination != null) {
-							writer.writeNestedLibrary(destination, file);
+							writer.writeNestedLibrary(destination, library);
 						}
 					}
 				}