1515 * limitations under the License.
1616 */
1717
18- package org .apache .spark .deploy
18+ package org .apache .spark .util
1919
2020import java .io .File
2121import java .net .URI
@@ -25,12 +25,140 @@ import org.apache.hadoop.conf.Configuration
2525import org .apache .hadoop .fs .{FileSystem , Path }
2626
2727import org .apache .spark .{SecurityManager , SparkConf , SparkException }
28+ import org .apache .spark .deploy .SparkSubmitUtils
2829import org .apache .spark .internal .Logging
29- import org .apache .spark .util .{MutableURLClassLoader , Utils }
3030
31- private [deploy] object DependencyUtils extends Logging {
31+ case class IvyProperties (
32+ packagesExclusions : String ,
33+ packages : String ,
34+ repositories : String ,
35+ ivyRepoPath : String ,
36+ ivySettingsPath : String )
37+
38+ private [spark] object DependencyUtils extends Logging {
39+
40+ def getIvyProperties (): IvyProperties = {
41+ val Seq (packagesExclusions, packages, repositories, ivyRepoPath, ivySettingsPath) = Seq (
42+ " spark.jars.excludes"
43+ " spark.jars.packages"
44+ " spark.jars.repositories"
45+ " spark.jars.ivy"
46+ " spark.jars.ivySettings"
47+ ).map(sys.props.get(_).orNull)
48+ IvyProperties (packagesExclusions, packages, repositories, ivyRepoPath, ivySettingsPath)
49+ }
50+
51+ private def isInvalidQueryString (tokens : Array [String ]): Boolean = {
52+ tokens.length != 2 || StringUtils .isBlank(tokens(0 )) || StringUtils .isBlank(tokens(1 ))
53+ }
54+
55+ /**
56+ * Parse URI query string's parameter value of `transitive` and `exclude`.
57+ * Other invalid parameters will be ignored.
58+ *
59+ * @param uri Ivy URI need to be downloaded.
60+ * @return Tuple value of parameter `transitive` and `exclude` value.
61+ *
62+ * 1. transitive: whether to download dependency jar of Ivy URI, default value is false
63+ * and this parameter value is case-sensitive. Invalid value will be treat as false.
64+ * Example: Input: exclude=org.mortbay.jetty:jetty&transitive=true
65+ * Output: true
66+ *
67+ * 2. exclude: comma separated exclusions to apply when resolving transitive dependencies,
68+ * consists of `group:module` pairs separated by commas.
69+ * Example: Input: excludeorg.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http
70+ * Output: [org.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http]
71+ */
72+ private def parseQueryParams (uri : URI ): (Boolean , String ) = {
73+ val uriQuery = uri.getQuery
74+ if (uriQuery == null ) {
75+ (false , " "
76+ } else {
77+ val mapTokens = uriQuery.split(" &" " ="
78+ if (mapTokens.exists(isInvalidQueryString)) {
79+ throw new IllegalArgumentException (
80+ s " Invalid query string in Ivy URI ${uri.toString}: $uriQuery" )
81+ }
82+ val groupedParams = mapTokens.map(kv => (kv(0 ), kv(1 ))).groupBy(_._1)
83+
84+ // Parse transitive parameters (e.g., transitive=true) in an Ivy URI, default value is false
85+ val transitiveParams = groupedParams.get(" transitive"
86+ if (transitiveParams.map(_.size).getOrElse(0 ) > 1 ) {
87+ logWarning(" It's best to specify `transitive` parameter in ivy URI query only once." +
88+ " If there are multiple `transitive` parameter, we will select the last one"
89+ }
90+ val transitive =
91+ transitiveParams.flatMap(_.takeRight(1 ).map(_._2 == " true" false )
92+
93+ // Parse an excluded list (e.g., exclude=org.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http)
94+ // in an Ivy URI. When download Ivy URI jar, Spark won't download transitive jar
95+ // in a excluded list.
96+ val exclusionList = groupedParams.get(" exclude" =>
97+ params.map(_._2).flatMap { excludeString =>
98+ val excludes = excludeString.split(" ,"
99+ if (excludes.map(_.split(" :"
100+ throw new IllegalArgumentException (
101+ s " Invalid exclude string in Ivy URI ${uri.toString}: " +
102+ " expected 'org:module,org:module,..', found " + excludeString)
103+ }
104+ excludes
105+ }.mkString(" ,"
106+ }.getOrElse(" "
107+
108+ val validParams = Set (" transitive" " exclude"
109+ val invalidParams = groupedParams.keys.filterNot(validParams.contains).toSeq
110+ if (invalidParams.nonEmpty) {
111+ logWarning(s " Invalid parameters ` ${invalidParams.sorted.mkString(" ," ` found " +
112+ s " in Ivy URI query ` $uriQuery`. " )
113+ }
114+
115+ (transitive, exclusionList)
116+ }
117+ }
118+
119+ /**
120+ * Download Ivy URI's dependency jars.
121+ *
122+ * @param uri Ivy URI need to be downloaded. The URI format should be:
123+ * `ivy://group:module:version[?query]`
124+ * Ivy URI query part format should be:
125+ * `parameter=value¶meter=value...`
126+ * Note that currently Ivy URI query part support two parameters:
127+ * 1. transitive: whether to download dependent jars related to your Ivy URI.
128+ * transitive=false or `transitive=true`, if not set, the default value is false.
129+ * 2. exclude: exclusion list when download Ivy URI jar and dependency jars.
130+ * The `exclude` parameter content is a ',' separated `group:module` pair string :
131+ * `exclude=group:module,group:module...`
132+ * @return Comma separated string list of jars downloaded.
133+ */
134+ def resolveMavenDependencies (uri : URI ): Seq [String ] = {
135+ val ivyProperties = DependencyUtils .getIvyProperties()
136+ val authority = uri.getAuthority
137+ if (authority == null ) {
138+ throw new IllegalArgumentException (
139+ s " Invalid Ivy URI authority in uri ${uri.toString}: " +
140+ " Expected 'org:module:version', found null."
141+ }
142+ if (authority.split(" :" != 3 ) {
143+ throw new IllegalArgumentException (
144+ s " Invalid Ivy URI authority in uri ${uri.toString}: " +
145+ s " Expected 'org:module:version', found $authority. " )
146+ }
147+
148+ val (transitive, exclusionList) = parseQueryParams(uri)
149+
150+ resolveMavenDependencies(
151+ transitive,
152+ exclusionList,
153+ authority,
154+ ivyProperties.repositories,
155+ ivyProperties.ivyRepoPath,
156+ Option (ivyProperties.ivySettingsPath)
157+ ).split(" ,"
158+ }
32159
33160 def resolveMavenDependencies (
161+ packagesTransitive : Boolean ,
34162 packagesExclusions : String ,
35163 packages : String ,
36164 repositories : String ,
@@ -51,7 +179,8 @@ private[deploy] object DependencyUtils extends Logging {
51179 SparkSubmitUtils .buildIvySettings(Option (repositories), Option (ivyRepoPath))
52180 }
53181
54- SparkSubmitUtils .resolveMavenCoordinates(packages, ivySettings, exclusions = exclusions)
182+ SparkSubmitUtils .resolveMavenCoordinates(packages, ivySettings,
183+ transitive = packagesTransitive, exclusions = exclusions)
55184 }
56185
57186 def resolveAndDownloadJars (
0 commit comments