Skip to content

Commit d434150

Browse files
ScrapCodespwendell
authored andcommitted
[SPARK-1199][REPL] Remove VALId and use the original import style for defined classes.
This is an alternate solution to apache#1176. Author: Prashant Sharma <prashant.s@imaginea.com> Closes apache#1179 from ScrapCodes/SPARK-1199/repl-fix-second-approach and squashes the following commits: 820b34b [Prashant Sharma] Here we generate two kinds of import wrappers based on whether it is a class or not.
1 parent 5448804 commit d434150

File tree

3 files changed

+31
-11
lines changed

3 files changed

+31
-11
lines changed

repl/src/main/scala/org/apache/spark/repl/SparkIMain.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -744,7 +744,7 @@ import org.apache.spark.util.Utils
744744
*
745745
* Read! Eval! Print! Some of that not yet centralized here.
746746
*/
747-
class ReadEvalPrint(lineId: Int) {
747+
class ReadEvalPrint(val lineId: Int) {
748748
def this() = this(freshLineId())
749749

750750
private var lastRun: Run = _
@@ -1241,7 +1241,10 @@ import org.apache.spark.util.Utils
12411241
// old style
12421242
beSilentDuring(parse(code)) foreach { ts =>
12431243
ts foreach { t =>
1244-
withoutUnwrapping(logDebug(asCompactString(t)))
1244+
if (isShow || isShowRaw)
1245+
withoutUnwrapping(echo(asCompactString(t)))
1246+
else
1247+
withoutUnwrapping(logDebug(asCompactString(t)))
12451248
}
12461249
}
12471250
}

repl/src/main/scala/org/apache/spark/repl/SparkImports.scala

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -182,15 +182,26 @@ trait SparkImports {
182182
// ambiguity errors will not be generated. Also, quote
183183
// the name of the variable, so that we don't need to
184184
// handle quoting keywords separately.
185+
case x: ClassHandler =>
186+
// I am trying to guess if the import is a defined class
187+
// This is an ugly hack, I am not 100% sure of the consequences.
188+
// Here we, let everything but "defined classes" use the import with val.
189+
// The reason for this is, otherwise the remote executor tries to pull the
190+
// classes involved and may fail.
191+
for (imv <- x.definedNames) {
192+
val objName = req.lineRep.readPath
193+
code.append("import " + objName + ".INSTANCE" + req.accessPath + ".`" + imv + "`\n")
194+
}
195+
185196
case x =>
186197
for (imv <- x.definedNames) {
187198
if (currentImps contains imv) addWrapper()
188199
val objName = req.lineRep.readPath
189-
val valName = "$VAL" + newValId();
200+
val valName = "$VAL" + req.lineRep.lineId
190201

191202
if(!code.toString.endsWith(".`" + imv + "`;\n")) { // Which means already imported
192-
code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
193-
code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
203+
code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
204+
code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
194205
}
195206
// code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
196207
// code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
@@ -211,10 +222,4 @@ trait SparkImports {
211222
private def membersAtPickler(sym: Symbol): List[Symbol] =
212223
beforePickler(sym.info.nonPrivateMembers.toList)
213224

214-
private var curValId = 0
215-
216-
private def newValId(): Int = {
217-
curValId += 1
218-
curValId
219-
}
220225
}

repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,18 @@ class ReplSuite extends FunSuite {
235235
assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output)
236236
}
237237

238+
test("SPARK-1199-simple-reproduce") {
239+
val output = runInterpreter("local-cluster[1,1,512]",
240+
"""
241+
|case class Sum(exp: String, exp2: String)
242+
|val a = Sum("A", "B")
243+
|def b(a: Sum): String = a match { case Sum(_, _) => "Found Sum" }
244+
|b(a)
245+
""".stripMargin)
246+
assertDoesNotContain("error:", output)
247+
assertDoesNotContain("Exception", output)
248+
}
249+
238250
if (System.getenv("MESOS_NATIVE_LIBRARY") != null) {
239251
test("running on Mesos") {
240252
val output = runInterpreter("localquiet",

0 commit comments

Comments
 (0)