Skip to content

Commit 04e61c5

Browse files
peter-tothpeter-toth
committed
Merge branch 'master' into SPARK-34581-keep-grouping-expressions
# Conflicts: # sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
2 parents 3de19ca + 93a5d34 commit 04e61c5

File tree

102 files changed

+2320
-1568
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+2320
-1568
lines changed

‎core/src/main/java/org/apache/spark/memory/MemoryConsumer.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ protected MemoryConsumer(TaskMemoryManager taskMemoryManager, long pageSize, Mem
4040
this.mode = mode;
4141
}
4242

43-
protected MemoryConsumer(TaskMemoryManager taskMemoryManager) {
44-
this(taskMemoryManager, taskMemoryManager.pageSizeBytes(), MemoryMode.ON_HEAP);
43+
protected MemoryConsumer(TaskMemoryManager taskMemoryManager, MemoryMode mode) {
44+
this(taskMemoryManager, taskMemoryManager.pageSizeBytes(), mode);
4545
}
4646

4747
/**

‎core/src/main/scala/org/apache/spark/ContextCleaner.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -172,18 +172,18 @@ private[spark] class ContextCleaner(
172172
registerForCleanup(rdd, CleanCheckpoint(parentId))
173173
}
174174

175-
/** Register an object for cleanup. */
176-
private def registerForCleanup(objectForCleanup: AnyRef, task: CleanupTask): Unit = {
177-
referenceBuffer.add(new CleanupTaskWeakReference(task, objectForCleanup, referenceQueue))
178-
}
179-
180175
/** Register a SparkListener to be cleaned up when its owner is garbage collected. */
181176
def registerSparkListenerForCleanup(
182177
listenerOwner: AnyRef,
183178
listener: SparkListener): Unit = {
184179
registerForCleanup(listenerOwner, CleanSparkListener(listener))
185180
}
186181

182+
/** Register an object for cleanup. */
183+
private def registerForCleanup(objectForCleanup: AnyRef, task: CleanupTask): Unit = {
184+
referenceBuffer.add(new CleanupTaskWeakReference(task, objectForCleanup, referenceQueue))
185+
}
186+
187187
/** Keep cleaning RDD, shuffle, and broadcast state. */
188188
private def keepCleaning(): Unit = Utils.tryOrStopSparkContext(sc) {
189189
while (!stopped) {

‎core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1584,7 +1584,11 @@ class SparkContext(config: SparkConf) extends Logging {
15841584
path: String, recursive: Boolean, addedOnSubmit: Boolean, isArchive: Boolean = false
15851585
): Unit = {
15861586
val uri = if (!isArchive) {
1587-
new Path(path).toUri
1587+
if (Utils.isAbsoluteURI(path) && path.contains("%")) {
1588+
new URI(path)
1589+
} else {
1590+
new Path(path).toUri
1591+
}
15881592
} else {
15891593
Utils.resolveURI(path)
15901594
}
@@ -1619,10 +1623,8 @@ class SparkContext(config: SparkConf) extends Logging {
16191623
env.rpcEnv.fileServer.addFile(new File(uri.getPath))
16201624
} else if (uri.getScheme == null) {
16211625
schemeCorrectedURI.toString
1622-
} else if (isArchive) {
1623-
uri.toString
16241626
} else {
1625-
path
1627+
uri.toString
16261628
}
16271629

16281630
val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
@@ -1977,7 +1979,11 @@ class SparkContext(config: SparkConf) extends Logging {
19771979
// For local paths with backslashes on Windows, URI throws an exception
19781980
(addLocalJarFile(new File(path)), "local")
19791981
} else {
1980-
val uri = new Path(path).toUri
1982+
val uri = if (Utils.isAbsoluteURI(path) && path.contains("%")) {
1983+
new URI(path)
1984+
} else {
1985+
new Path(path).toUri
1986+
}
19811987
// SPARK-17650: Make sure this is a valid URL before adding it to the list of dependencies
19821988
Utils.validateURL(uri)
19831989
val uriScheme = uri.getScheme

‎core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import org.apache.ivy.core.report.ResolveReport
4444
import org.apache.ivy.core.resolve.ResolveOptions
4545
import org.apache.ivy.core.retrieve.RetrieveOptions
4646
import org.apache.ivy.core.settings.IvySettings
47-
import org.apache.ivy.plugins.matcher.{GlobPatternMatcher, PatternMatcher}
47+
import org.apache.ivy.plugins.matcher.GlobPatternMatcher
4848
import org.apache.ivy.plugins.repository.file.FileRepository
4949
import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver}
5050

@@ -366,7 +366,6 @@ private[spark] class SparkSubmit extends Logging {
366366
args.pyFiles = Option(args.pyFiles).map(resolveGlobPaths(_, hadoopConf)).orNull
367367
args.archives = Option(args.archives).map(resolveGlobPaths(_, hadoopConf)).orNull
368368

369-
lazy val secMgr = new SecurityManager(sparkConf)
370369

371370
// In client mode, download remote files.
372371
var localPrimaryResource: String = null
@@ -1153,8 +1152,6 @@ private[spark] object SparkSubmitUtils extends Logging {
11531152
// We need a chain resolver if we want to check multiple repositories
11541153
val cr = new ChainResolver
11551154
cr.setName("spark-list")
1156-
cr.setChangingMatcher(PatternMatcher.REGEXP)
1157-
cr.setChangingPattern(".*-SNAPSHOT")
11581155

11591156
val localM2 = new IBiblioResolver
11601157
localM2.setM2compatible(true)
@@ -1314,8 +1311,6 @@ private[spark] object SparkSubmitUtils extends Logging {
13141311
remoteRepos.filterNot(_.trim.isEmpty).map(_.split(",")).foreach { repositoryList =>
13151312
val cr = new ChainResolver
13161313
cr.setName("user-list")
1317-
cr.setChangingMatcher(PatternMatcher.REGEXP)
1318-
cr.setChangingPattern(".*-SNAPSHOT")
13191314

13201315
// add current default resolver, if any
13211316
Option(ivySettings.getDefaultResolver).foreach(cr.add)

‎core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ object DriverWrapper extends Logging {
7474

7575
private def setupDependencies(loader: MutableURLClassLoader, userJar: String): Unit = {
7676
val sparkConf = new SparkConf()
77-
val secMgr = new SecurityManager(sparkConf)
7877
val hadoopConf = SparkHadoopUtil.newConfiguration(sparkConf)
7978

8079
val ivyProperties = DependencyUtils.getIvyProperties()

‎core/src/main/scala/org/apache/spark/internal/config/package.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1037,7 +1037,7 @@ package object config {
10371037
.doc("When true, HadoopRDD/NewHadoopRDD will not create partitions for empty input splits.")
10381038
.version("2.3.0")
10391039
.booleanConf
1040-
.createWithDefault(false)
1040+
.createWithDefault(true)
10411041

10421042
private[spark] val SECRET_REDACTION_PATTERN =
10431043
ConfigBuilder("spark.redaction.regex")
@@ -1047,7 +1047,7 @@ package object config {
10471047
"like YARN and event logs.")
10481048
.version("2.1.2")
10491049
.regexConf
1050-
.createWithDefault("(?i)secret|password|token".r)
1050+
.createWithDefault("(?i)secret|password|token|access[.]key".r)
10511051

10521052
private[spark] val STRING_REDACTION_PATTERN =
10531053
ConfigBuilder("spark.redaction.string.regex")

‎core/src/main/scala/org/apache/spark/shuffle/BlockStoreShuffleReader.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,15 +51,17 @@ private[spark] class BlockStoreShuffleReader[K, C](
5151
true
5252
}
5353
val useOldFetchProtocol = conf.get(config.SHUFFLE_USE_OLD_FETCH_PROTOCOL)
54+
// SPARK-34790: Fetching continuous blocks in batch is incompatible with io encryption.
55+
val ioEncryption = conf.get(config.IO_ENCRYPTION_ENABLED)
5456

5557
val doBatchFetch = shouldBatchFetch && serializerRelocatable &&
56-
(!compressed || codecConcatenation) && !useOldFetchProtocol
58+
(!compressed || codecConcatenation) && !useOldFetchProtocol && !ioEncryption
5759
if (shouldBatchFetch && !doBatchFetch) {
5860
logDebug("The feature tag of continuous shuffle block fetching is set to true, but " +
5961
"we can not enable the feature because other conditions are not satisfied. " +
6062
s"Shuffle compress: $compressed, serializer relocatable: $serializerRelocatable, " +
6163
s"codec concatenation: $codecConcatenation, use old shuffle fetch protocol: " +
62-
s"$useOldFetchProtocol.")
64+
s"$useOldFetchProtocol, io encryption: $ioEncryption.")
6365
}
6466
doBatchFetch
6567
}

‎core/src/main/scala/org/apache/spark/util/Utils.scala

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2063,6 +2063,17 @@ private[spark] object Utils extends Logging {
20632063
}
20642064
}
20652065

2066+
/** Check whether a path is an absolute URI. */
2067+
def isAbsoluteURI(path: String): Boolean = {
2068+
try {
2069+
val uri = new URI(path: String)
2070+
uri.isAbsolute
2071+
} catch {
2072+
case _: URISyntaxException =>
2073+
false
2074+
}
2075+
}
2076+
20662077
/** Return all non-local paths from a comma-separated list of paths. */
20672078
def nonLocalPaths(paths: String, testWindows: Boolean = false): Array[String] = {
20682079
val windows = isWindows || testWindows

‎core/src/main/scala/org/apache/spark/util/collection/Spillable.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager}
2727
* has been exceeded.
2828
*/
2929
private[spark] abstract class Spillable[C](taskMemoryManager: TaskMemoryManager)
30-
extends MemoryConsumer(taskMemoryManager) with Logging {
30+
extends MemoryConsumer(taskMemoryManager, MemoryMode.ON_HEAP) with Logging {
3131
/**
3232
* Spills the current in-memory collection to disk, and releases the memory.
3333
*

‎core/src/test/scala/org/apache/spark/SparkContextSuite.scala

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1197,6 +1197,46 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
11971197
assert(sc.hadoopConfiguration.get(bufferKey).toInt === 65536,
11981198
"spark configs have higher priority than spark.hadoop configs")
11991199
}
1200+
1201+
test("SPARK-34225: addFile/addJar shouldn't further encode URI if a URI form string is passed") {
1202+
withTempDir { dir =>
1203+
val jar1 = File.createTempFile("testprefix", "test jar.jar", dir)
1204+
val jarUrl1 = jar1.toURI.toString
1205+
val file1 = File.createTempFile("testprefix", "test file.txt", dir)
1206+
val fileUrl1 = file1.toURI.toString
1207+
val jar2 = File.createTempFile("testprefix", "test %20jar.jar", dir)
1208+
val file2 = File.createTempFile("testprefix", "test %20file.txt", dir)
1209+
1210+
try {
1211+
sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
1212+
sc.addJar(jarUrl1)
1213+
sc.addFile(fileUrl1)
1214+
sc.addJar(jar2.toString)
1215+
sc.addFile(file2.toString)
1216+
sc.parallelize(Array(1), 1).map { x =>
1217+
val gottenJar1 = new File(SparkFiles.get(jar1.getName))
1218+
if (!gottenJar1.exists()) {
1219+
throw new SparkException("file doesn't exist : " + jar1)
1220+
}
1221+
val gottenFile1 = new File(SparkFiles.get(file1.getName))
1222+
if (!gottenFile1.exists()) {
1223+
throw new SparkException("file doesn't exist : " + file1)
1224+
}
1225+
val gottenJar2 = new File(SparkFiles.get(jar2.getName))
1226+
if (!gottenJar2.exists()) {
1227+
throw new SparkException("file doesn't exist : " + jar2)
1228+
}
1229+
val gottenFile2 = new File(SparkFiles.get(file2.getName))
1230+
if (!gottenFile2.exists()) {
1231+
throw new SparkException("file doesn't exist : " + file2)
1232+
}
1233+
x
1234+
}.collect()
1235+
} finally {
1236+
sc.stop()
1237+
}
1238+
}
1239+
}
12001240
}
12011241

12021242
object SparkContextSuite {

‎core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -304,22 +304,4 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
304304
s" Resolved jars are: $jarPath")
305305
}
306306
}
307-
308-
test("SPARK-34757: should ignore cache for SNAPSHOT dependencies") {
309-
val main = new MavenCoordinate("my.great.lib", "mylib", "0.1-SNAPSHOT")
310-
IvyTestUtils.withRepository(main, None, None) { repo =>
311-
val ivySettings = SparkSubmitUtils.buildIvySettings(Some(repo), Some(tempIvyPath))
312-
// set isTest to false since we need to check the resolved jar file
313-
val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
314-
main.toString, ivySettings, transitive = true, isTest = false)
315-
val modifiedTimestamp = Files.getLastModifiedTime(Paths.get(jarPath.head))
316-
// update the artifact and resolve again
317-
IvyTestUtils.createLocalRepositoryForTests(main, None, Some(new File(new URI(repo))))
318-
SparkSubmitUtils.resolveMavenCoordinates(
319-
main.toString, ivySettings, transitive = true, isTest = false)
320-
// check that the artifact is updated
321-
assert(
322-
modifiedTimestamp.compareTo(Files.getLastModifiedTime(Paths.get(jarPath.head))) != 0)
323-
}
324-
}
325307
}

‎core/src/test/scala/org/apache/spark/util/UtilsSuite.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1024,11 +1024,13 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
10241024
// Set some secret keys
10251025
val secretKeys = Seq(
10261026
"spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
1027+
"spark.hadoop.fs.s3a.access.key",
10271028
"spark.my.password",
10281029
"spark.my.sECreT")
10291030
secretKeys.foreach { key => sparkConf.set(key, "sensitive_value") }
10301031
// Set a non-secret key
10311032
sparkConf.set("spark.regular.property", "regular_value")
1033+
sparkConf.set("spark.hadoop.fs.s3a.access_key", "regular_value")
10321034
// Set a property with a regular key but secret in the value
10331035
sparkConf.set("spark.sensitive.property", "has_secret_in_value")
10341036

@@ -1039,7 +1041,8 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
10391041
secretKeys.foreach { key => assert(redactedConf(key) === Utils.REDACTION_REPLACEMENT_TEXT) }
10401042
assert(redactedConf("spark.regular.property") === "regular_value")
10411043
assert(redactedConf("spark.sensitive.property") === Utils.REDACTION_REPLACEMENT_TEXT)
1042-
1044+
assert(redactedConf("spark.hadoop.fs.s3a.access.key") === Utils.REDACTION_REPLACEMENT_TEXT)
1045+
assert(redactedConf("spark.hadoop.fs.s3a.access_key") === "regular_value")
10431046
}
10441047

10451048
test("redact sensitive information in command line args") {

‎dev/deps/spark-deps-hadoop-2.7-hive-2.3

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar
2222
arrow-vector/2.0.0//arrow-vector-2.0.0.jar
2323
audience-annotations/0.5.0//audience-annotations-0.5.0.jar
2424
automaton/1.11-8//automaton-1.11-8.jar
25-
avro-ipc/1.10.1//avro-ipc-1.10.1.jar
26-
avro-mapred/1.10.1//avro-mapred-1.10.1.jar
27-
avro/1.10.1//avro-1.10.1.jar
25+
avro-ipc/1.10.2//avro-ipc-1.10.2.jar
26+
avro-mapred/1.10.2//avro-mapred-1.10.2.jar
27+
avro/1.10.2//avro-1.10.2.jar
2828
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
2929
breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
3030
breeze_2.12/1.0//breeze_2.12-1.0.jar
@@ -104,17 +104,16 @@ httpclient/4.5.13//httpclient-4.5.13.jar
104104
httpcore/4.4.12//httpcore-4.4.12.jar
105105
istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
106106
ivy/2.4.0//ivy-2.4.0.jar
107-
jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar
107+
jackson-annotations/2.12.2//jackson-annotations-2.12.2.jar
108108
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
109-
jackson-core/2.11.4//jackson-core-2.11.4.jar
110-
jackson-databind/2.11.4//jackson-databind-2.11.4.jar
111-
jackson-dataformat-yaml/2.11.4//jackson-dataformat-yaml-2.11.4.jar
109+
jackson-core/2.12.2//jackson-core-2.12.2.jar
110+
jackson-databind/2.12.2//jackson-databind-2.12.2.jar
111+
jackson-dataformat-yaml/2.12.2//jackson-dataformat-yaml-2.12.2.jar
112112
jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
113113
jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
114114
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
115-
jackson-module-jaxb-annotations/2.11.4//jackson-module-jaxb-annotations-2.11.4.jar
116-
jackson-module-paranamer/2.11.4//jackson-module-paranamer-2.11.4.jar
117-
jackson-module-scala_2.12/2.11.4//jackson-module-scala_2.12-2.11.4.jar
115+
jackson-module-jaxb-annotations/2.12.2//jackson-module-jaxb-annotations-2.12.2.jar
116+
jackson-module-scala_2.12/2.12.2//jackson-module-scala_2.12-2.12.2.jar
118117
jackson-xc/1.9.13//jackson-xc-1.9.13.jar
119118
jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
120119
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
@@ -222,7 +221,7 @@ shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
222221
shims/0.9.0//shims-0.9.0.jar
223222
slf4j-api/1.7.30//slf4j-api-1.7.30.jar
224223
slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
225-
snakeyaml/1.26//snakeyaml-1.26.jar
224+
snakeyaml/1.27//snakeyaml-1.27.jar
226225
snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
227226
spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
228227
spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar

‎dev/deps/spark-deps-hadoop-3.2-hive-2.3

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar
1717
arrow-vector/2.0.0//arrow-vector-2.0.0.jar
1818
audience-annotations/0.5.0//audience-annotations-0.5.0.jar
1919
automaton/1.11-8//automaton-1.11-8.jar
20-
avro-ipc/1.10.1//avro-ipc-1.10.1.jar
21-
avro-mapred/1.10.1//avro-mapred-1.10.1.jar
22-
avro/1.10.1//avro-1.10.1.jar
20+
avro-ipc/1.10.2//avro-ipc-1.10.2.jar
21+
avro-mapred/1.10.2//avro-mapred-1.10.2.jar
22+
avro/1.10.2//avro-1.10.2.jar
2323
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
2424
breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
2525
breeze_2.12/1.0//breeze_2.12-1.0.jar
@@ -82,16 +82,15 @@ httpclient/4.5.13//httpclient-4.5.13.jar
8282
httpcore/4.4.12//httpcore-4.4.12.jar
8383
istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
8484
ivy/2.4.0//ivy-2.4.0.jar
85-
jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar
85+
jackson-annotations/2.12.2//jackson-annotations-2.12.2.jar
8686
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
87-
jackson-core/2.11.4//jackson-core-2.11.4.jar
88-
jackson-databind/2.11.4//jackson-databind-2.11.4.jar
89-
jackson-dataformat-yaml/2.11.4//jackson-dataformat-yaml-2.11.4.jar
87+
jackson-core/2.12.2//jackson-core-2.12.2.jar
88+
jackson-databind/2.12.2//jackson-databind-2.12.2.jar
89+
jackson-dataformat-yaml/2.12.2//jackson-dataformat-yaml-2.12.2.jar
9090
jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
9191
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
92-
jackson-module-jaxb-annotations/2.11.4//jackson-module-jaxb-annotations-2.11.4.jar
93-
jackson-module-paranamer/2.11.4//jackson-module-paranamer-2.11.4.jar
94-
jackson-module-scala_2.12/2.11.4//jackson-module-scala_2.12-2.11.4.jar
92+
jackson-module-jaxb-annotations/2.12.2//jackson-module-jaxb-annotations-2.12.2.jar
93+
jackson-module-scala_2.12/2.12.2//jackson-module-scala_2.12-2.12.2.jar
9594
jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
9695
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
9796
jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
@@ -193,7 +192,7 @@ shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
193192
shims/0.9.0//shims-0.9.0.jar
194193
slf4j-api/1.7.30//slf4j-api-1.7.30.jar
195194
slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
196-
snakeyaml/1.26//snakeyaml-1.26.jar
195+
snakeyaml/1.27//snakeyaml-1.27.jar
197196
snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
198197
spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
199198
spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar

‎docs/cloud-integration.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,8 @@ under-reported with Hadoop versions before 3.3.1.
276276
Here is the documentation on the standard connectors both from Apache and the cloud providers.
277277

278278
* [OpenStack Swift](https://hadoop.apache.org/docs/current/hadoop-openstack/index.html).
279-
* [Azure Blob Storage and Azure Datalake Gen 2](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
279+
* [Azure Blob Storage](https://hadoop.apache.org/docs/current/hadoop-azure/index.html).
280+
* [Azure Blob Filesystem (ABFS) and Azure Datalake Gen 2](https://hadoop.apache.org/docs/current/hadoop-azure/abfs.html).
280281
* [Azure Data Lake Gen 1](https://hadoop.apache.org/docs/current/hadoop-azure-datalake/index.html).
281282
* [Amazon S3 Strong Consistency](https://aws.amazon.com/s3/consistency/)
282283
* [Hadoop-AWS module (Hadoop 3.x)](https://hadoop.apache.org/docs/current3/hadoop-aws/tools/hadoop-aws/index.html).

‎docs/core-migration-guide.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ license: |
2424

2525
## Upgrading from Core 3.1 to 3.2
2626

27+
- Since Spark 3.2, `spark.hadoopRDD.ignoreEmptySplits` is set to `true` by default which means Spark will not create empty partitions for empty input splits. To restore the behavior before Spark 3.2, you can set `spark.hadoopRDD.ignoreEmptySplits` to `false`.
28+
2729
- Since Spark 3.2, `spark.eventLog.compression.codec` is set to `zstd` by default which means Spark will not fallback to use `spark.io.compression.codec` anymore.
2830

2931
- Since Spark 3.2, `spark.storage.replication.proactive` is enabled by default which means Spark tries to replenish in case of the loss of cached RDD block replicas due to executor failures. To restore the behavior before Spark 3.2, you can set `spark.storage.replication.proactive` to `false`.

0 commit comments

Comments
 (0)