Skip to content

Commit 398cb95

Browse files
committed
Fix looking up shuffle manager in conf
1 parent ca3efd9 commit 398cb95

File tree

1 file changed

+11
-7
lines changed

1 file changed

+11
-7
lines changed

core/src/main/scala/org/apache/spark/SparkEnv.scala

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -156,11 +156,9 @@ object SparkEnv extends Logging {
156156
conf.set("spark.driver.port", boundPort.toString)
157157
}
158158

159-
// Create an instance of the class named by the given Java system property, or by
160-
// defaultClassName if the property is not set, and return it as a T
161-
def instantiateClass[T](propertyName: String, defaultClassName: String = null): T = {
162-
val name = conf.get(propertyName, defaultClassName)
163-
val cls = Class.forName(name, true, Utils.getContextOrSparkClassLoader)
159+
// Create an instance of the class with the given name, possibly initializing it with our conf
160+
def instantiateClass[T](className: String): T = {
161+
val cls = Class.forName(className, true, Utils.getContextOrSparkClassLoader)
164162
// Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just
165163
// SparkConf, then one taking no arguments
166164
try {
@@ -178,11 +176,17 @@ object SparkEnv extends Logging {
178176
}
179177
}
180178

181-
val serializer = instantiateClass[Serializer](
179+
// Create an instance of the class named by the given SparkConf property, or defaultClassName
180+
// if the property is not set, possibly initializing it with our conf
181+
def instantiateClassFromConf[T](propertyName: String, defaultClassName: String): T = {
182+
instantiateClass[T](conf.get(propertyName, defaultClassName))
183+
}
184+
185+
val serializer = instantiateClassFromConf[Serializer](
182186
"spark.serializer", "org.apache.spark.serializer.JavaSerializer")
183187
logDebug(s"Using serializer: ${serializer.getClass}")
184188

185-
val closureSerializer = instantiateClass[Serializer](
189+
val closureSerializer = instantiateClassFromConf[Serializer](
186190
"spark.closure.serializer", "org.apache.spark.serializer.JavaSerializer")
187191

188192
def registerOrLookup(name: String, newActor: => Actor): ActorRef = {

0 commit comments

Comments
 (0)