Skip to content

Commit b9be160

Browse files
pwendellpwendell
committed
SPARK-782 Clean up for ASM dependency.
This makes two changes. 1) Spark uses the shaded version of asm that is (conveniently) published with Kryo. 2) Existing exclude rules around asm are updated to reflect the new groupId of `org.ow2.asm`. This made all of the old rules not work with newer Hadoop versions that pull in new asm versions. Author: Patrick Wendell <pwendell@gmail.com> Closes apache#100 from pwendell/asm and squashes the following commits: 9235f3f [Patrick Wendell] SPARK-782 Clean up for ASM dependency.
1 parent faf4cad commit b9be160

File tree

6 files changed

+53
-25
lines changed

6 files changed

+53
-25
lines changed

‎core/pom.xml

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,6 @@
102102
<groupId>org.xerial.snappy</groupId>
103103
<artifactId>snappy-java</artifactId>
104104
</dependency>
105-
<dependency>
106-
<groupId>org.ow2.asm</groupId>
107-
<artifactId>asm</artifactId>
108-
</dependency>
109105
<dependency>
110106
<groupId>com.twitter</groupId>
111107
<artifactId>chill_${scala.binary.version}</artifactId>

‎core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
2222
import scala.collection.mutable.Map
2323
import scala.collection.mutable.Set
2424

25-
import org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor, Type}
26-
import org.objectweb.asm.Opcodes._
25+
import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor, Type}
26+
import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._
2727

2828
import org.apache.spark.Logging
2929

‎graphx/src/main/scala/org/apache/spark/graphx/util/BytecodeUtils.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ import scala.collection.mutable.HashSet
2323

2424
import org.apache.spark.util.Utils
2525

26-
import org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor}
27-
import org.objectweb.asm.Opcodes._
26+
import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor}
27+
import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._
2828

2929

3030
/**

‎pom.xml

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,6 @@
221221
<artifactId>snappy-java</artifactId>
222222
<version>1.0.5</version>
223223
</dependency>
224-
<dependency>
225-
<groupId>org.ow2.asm</groupId>
226-
<artifactId>asm</artifactId>
227-
<version>4.0</version>
228-
</dependency>
229224
<dependency>
230225
<groupId>com.clearspring.analytics</groupId>
231226
<artifactId>stream</artifactId>
@@ -245,11 +240,31 @@
245240
<groupId>com.twitter</groupId>
246241
<artifactId>chill_${scala.binary.version}</artifactId>
247242
<version>0.3.1</version>
243+
<exclusions>
244+
<exclusion>
245+
<groupId>org.ow2.asm</groupId>
246+
<artifactId>asm</artifactId>
247+
</exclusion>
248+
<exclusion>
249+
<groupId>org.ow2.asm</groupId>
250+
<artifactId>asm-commons</artifactId>
251+
</exclusion>
252+
</exclusions>
248253
</dependency>
249254
<dependency>
250255
<groupId>com.twitter</groupId>
251256
<artifactId>chill-java</artifactId>
252257
<version>0.3.1</version>
258+
<exclusions>
259+
<exclusion>
260+
<groupId>org.ow2.asm</groupId>
261+
<artifactId>asm</artifactId>
262+
</exclusion>
263+
<exclusion>
264+
<groupId>org.ow2.asm</groupId>
265+
<artifactId>asm-commons</artifactId>
266+
</exclusion>
267+
</exclusions>
253268
</dependency>
254269
<dependency>
255270
<groupId>${akka.group}</groupId>
@@ -435,6 +450,10 @@
435450
<groupId>asm</groupId>
436451
<artifactId>asm</artifactId>
437452
</exclusion>
453+
<exclusion>
454+
<groupId>org.ow2.asm</groupId>
455+
<artifactId>asm</artifactId>
456+
</exclusion>
438457
<exclusion>
439458
<groupId>org.jboss.netty</groupId>
440459
<artifactId>netty</artifactId>
@@ -474,6 +493,10 @@
474493
<groupId>asm</groupId>
475494
<artifactId>asm</artifactId>
476495
</exclusion>
496+
<exclusion>
497+
<groupId>org.ow2.asm</groupId>
498+
<artifactId>asm</artifactId>
499+
</exclusion>
477500
<exclusion>
478501
<groupId>org.jboss.netty</groupId>
479502
<artifactId>netty</artifactId>
@@ -489,6 +512,10 @@
489512
<groupId>asm</groupId>
490513
<artifactId>asm</artifactId>
491514
</exclusion>
515+
<exclusion>
516+
<groupId>org.ow2.asm</groupId>
517+
<artifactId>asm</artifactId>
518+
</exclusion>
492519
<exclusion>
493520
<groupId>org.jboss.netty</groupId>
494521
<artifactId>netty</artifactId>
@@ -505,6 +532,10 @@
505532
<groupId>asm</groupId>
506533
<artifactId>asm</artifactId>
507534
</exclusion>
535+
<exclusion>
536+
<groupId>org.ow2.asm</groupId>
537+
<artifactId>asm</artifactId>
538+
</exclusion>
508539
<exclusion>
509540
<groupId>org.jboss.netty</groupId>
510541
<artifactId>netty</artifactId>

‎project/SparkBuild.scala

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,8 @@ object SparkBuild extends Build {
257257
val slf4jVersion = "1.7.5"
258258

259259
val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
260-
val excludeAsm = ExclusionRule(organization = "asm")
260+
val excludeAsm = ExclusionRule(organization = "org.ow2.asm")
261+
val excludeOldAsm = ExclusionRule(organization = "asm")
261262
val excludeCommonsLogging = ExclusionRule(organization = "commons-logging")
262263
val excludeSLF4J = ExclusionRule(organization = "org.slf4j")
263264
val excludeScalap = ExclusionRule(organization = "org.scala-lang", artifact = "scalap")
@@ -280,7 +281,6 @@ object SparkBuild extends Build {
280281
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
281282
"com.ning" % "compress-lzf" % "1.0.0",
282283
"org.xerial.snappy" % "snappy-java" % "1.0.5",
283-
"org.ow2.asm" % "asm" % "4.0",
284284
"org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
285285
"org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
286286
"org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test",
@@ -291,15 +291,15 @@ object SparkBuild extends Build {
291291
"commons-net" % "commons-net" % "2.2",
292292
"net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging),
293293
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
294-
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J),
294+
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
295295
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),
296296
"com.codahale.metrics" % "metrics-core" % "3.0.0",
297297
"com.codahale.metrics" % "metrics-jvm" % "3.0.0",
298298
"com.codahale.metrics" % "metrics-json" % "3.0.0",
299299
"com.codahale.metrics" % "metrics-ganglia" % "3.0.0",
300300
"com.codahale.metrics" % "metrics-graphite" % "3.0.0",
301-
"com.twitter" %% "chill" % "0.3.1",
302-
"com.twitter" % "chill-java" % "0.3.1",
301+
"com.twitter" %% "chill" % "0.3.1" excludeAll(excludeAsm),
302+
"com.twitter" % "chill-java" % "0.3.1" excludeAll(excludeAsm),
303303
"com.clearspring.analytics" % "stream" % "2.5.1"
304304
),
305305
libraryDependencies ++= maybeAvro
@@ -320,7 +320,7 @@ object SparkBuild extends Build {
320320
name := "spark-examples",
321321
libraryDependencies ++= Seq(
322322
"com.twitter" %% "algebird-core" % "0.1.11",
323-
"org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging),
323+
"org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging),
324324
"org.apache.cassandra" % "cassandra-all" % "1.2.6"
325325
exclude("com.google.guava", "guava")
326326
exclude("com.googlecode.concurrentlinkedhashmap", "concurrentlinkedhashmap-lru")
@@ -397,10 +397,10 @@ object SparkBuild extends Build {
397397
def yarnEnabledSettings = Seq(
398398
libraryDependencies ++= Seq(
399399
// Exclude rule required for all ?
400-
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm),
401-
"org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm),
402-
"org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm),
403-
"org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm)
400+
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm),
401+
"org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm),
402+
"org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm),
403+
"org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm)
404404
)
405405
)
406406

‎repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ import org.apache.hadoop.fs.{FileSystem, Path}
2727
import org.apache.spark.SparkEnv
2828
import org.apache.spark.util.Utils
2929

30-
import org.objectweb.asm._
31-
import org.objectweb.asm.Opcodes._
30+
31+
import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm._
32+
import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._
3233

3334

3435
/**

0 commit comments

Comments
 (0)