Skip to content

Commit 67d5114

Browse files
yhuairxin
authored andcommitted
Give Databricks-specific messages when new SparkContext or SQLContext is created
## What changes were proposed in this pull request? This commit hard-codes allowMultipleContexts to false and change the error message to a DBC-specific one. This is originally authored by yhuai. Author: Yin Huai <yhuai@databricks.com> Closes apache#43 from rxin/rxin-dbc-sparkcontext-msg.
1 parent fecf8d0 commit 67d5114

File tree

1 file changed

+17
-13
lines changed

1 file changed

+17
-13
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2215,9 +2215,11 @@ object SparkContext extends Logging {
22152215
allowMultipleContexts: Boolean): Unit = {
22162216
SPARK_CONTEXT_CONSTRUCTOR_LOCK.synchronized {
22172217
Option(activeContext.get()).filter(_ ne sc).foreach { ctx =>
2218-
val errMsg = "Only one SparkContext may be running in this JVM (see SPARK-2243)." +
2219-
" To ignore this error, set spark.driver.allowMultipleContexts = true. " +
2220-
s"The currently running SparkContext was created at:\n${ctx.creationSite.longForm}"
2218+
val ctx = activeContext.get()
2219+
val errMsg = "In Databricks, developers should utilize the shared SparkContext " +
2220+
"instead of creating one using the constructor. In Scala and Python notebooks, " +
2221+
"the shared context can be accessed as sc. When running a job, " +
2222+
"you can access the shared context by calling SparkContext.getOrCreate()."
22212223
val exception = new SparkException(errMsg)
22222224
if (allowMultipleContexts) {
22232225
logWarning("Multiple running SparkContexts detected in the same JVM!", exception)
@@ -2226,16 +2228,18 @@ object SparkContext extends Logging {
22262228
}
22272229
}
22282230

2229-
contextBeingConstructed.filter(_ ne sc).foreach { otherContext =>
2230-
// Since otherContext might point to a partially-constructed context, guard against
2231-
// its creationSite field being null:
2232-
val otherContextCreationSite =
2233-
Option(otherContext.creationSite).map(_.longForm).getOrElse("unknown location")
2234-
val warnMsg = "Another SparkContext is being constructed (or threw an exception in its" +
2235-
" constructor). This may indicate an error, since only one SparkContext may be" +
2236-
" running in this JVM (see SPARK-2243)." +
2237-
s" The other SparkContext was created at:\n$otherContextCreationSite"
2238-
logWarning(warnMsg)
2231+
contextBeingConstructed.foreach { otherContext =>
2232+
if (otherContext ne sc) { // checks for reference equality
2233+
// Since otherContext might point to a partially-constructed context, guard against
2234+
// its creationSite field being null:
2235+
val otherContextCreationSite =
2236+
Option(otherContext.creationSite).map(_.longForm).getOrElse("unknown location")
2237+
val warnMsg = "Another SparkContext is being constructed (or threw an exception in its" +
2238+
" constructor). This may indicate an error, since only one SparkContext may be" +
2239+
" running in this JVM (see SPARK-2243)." +
2240+
s" The other SparkContext was created at:\n$otherContextCreationSite"
2241+
logWarning(warnMsg)
2242+
}
22392243
}
22402244
}
22412245
}

0 commit comments

Comments
 (0)