Skip to content

feat: backport Hadoop 3.4.1 vulnerability fixes #1184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: backport Hadoop 3.4.1 vulnerability fixes #1184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0811c93
feat: backport Hadoop 3.4.1 vulnerability fixes
dervoeti Jun 19, 2025
72d104b
chore: changelog
dervoeti Jun 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ All notable changes to this project will be documented in this file.
- git-sync: Bump version to 4.4.1 ([#1151]).
- zookeeper: bump jetty version for CVE-2024-13009 in 3.9.3 ([#1179])
- zookeeper: bump netty version for CVE-2025-24970 in 3.9.3 ([#1180])
- hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184])

### Changed

Expand Down Expand Up @@ -200,6 +201,7 @@ All notable changes to this project will be documented in this file.
[#1174]: https://github.com/stackabletech/docker-images/pull/1174
[#1179]: https://github.com/stackabletech/docker-images/pull/1179
[#1180]: https://github.com/stackabletech/docker-images/pull/1180
[#1184]: https://github.com/stackabletech/docker-images/pull/1184

## [25.3.0] - 2025-03-21

Expand Down
247 changes: 247 additions & 0 deletions ...p/stackable/patches/3.4.1/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
From 699f329ba20d938004fc2b983b7b225de36ecf88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=91=A8=E7=BF=B1?= <zhouaoe@163.com>
Date: Tue, 18 Feb 2025 14:16:37 +0800
Subject: HADOOP-19352. Hadoop OSS Connector adds support for V4 signatures.
(#7205)

* Aliyun oss connector support v4 signature
---
hadoop-project/pom.xml | 2 +-
hadoop-tools/hadoop-aliyun/pom.xml | 26 +++++
.../aliyun/oss/AliyunOSSFileSystemStore.java | 16 +++
.../hadoop/fs/aliyun/oss/Constants.java | 15 +++
.../fs/aliyun/oss/ITAliyunOSSSignatureV4.java | 98 +++++++++++++++++++
.../src/test/resources/log4j.properties | 3 +
6 files changed, 159 insertions(+), 1 deletion(-)
create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3426a76c12..3a812ebc64 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1620,7 +1620,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
- <version>3.13.2</version>
+ <version>3.18.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index 5678cff642..3db04ce913 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -165,5 +165,31 @@
<scope>test</scope>
<type>jar</type>
</dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.platform</groupId>
+ <artifactId>junit-platform-launcher</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index 6e0c7dc7e4..dba267b04c 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -73,6 +73,7 @@ import java.util.List;
import java.util.ListIterator;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
+import com.aliyun.oss.common.comm.SignVersion;

import static org.apache.hadoop.fs.aliyun.oss.Constants.*;

@@ -113,6 +114,16 @@ public class AliyunOSSFileSystemStore {
conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/"
+ VersionInfo.getVersion());

+ String region = conf.get(REGION_KEY, "");
+ String signatureVersion = conf.get(SIGNATURE_VERSION_KEY, SIGNATURE_VERSION_DEFAULT);
+ if ("V4".equalsIgnoreCase(signatureVersion)) {
+ clientConf.setSignatureVersion(SignVersion.V4);
+ if (StringUtils.isEmpty(region)) {
+ LOG.error("Signature version is V4 ,but region is empty.");
+ throw new IOException("SignVersion is V4 but region is empty");
+ }
+ }
+
String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
if (StringUtils.isNotEmpty(proxyHost)) {
@@ -171,6 +182,11 @@ public class AliyunOSSFileSystemStore {
statistics.incrementWriteOps(1);
}

+ if (StringUtils.isNotEmpty(region)) {
+ ossClient.setRegion(region);
+ LOG.debug("ossClient setRegion {}", region);
+ }
+
maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION);
if (listVersion < 1 || listVersion > 2) {
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index baeb919937..176669ed15 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -211,4 +211,19 @@ public final class Constants {
public static final String LIST_VERSION = "fs.oss.list.version";

public static final int DEFAULT_LIST_VERSION = 2;
+
+ /**
+ * OSS signature version.
+ */
+ public static final String SIGNATURE_VERSION_KEY = "fs.oss.signatureversion";
+
+ /**
+ * OSS signature version DEFAULT {@value}.
+ */
+ public static final String SIGNATURE_VERSION_DEFAULT = "V1";
+
+ /**
+ * OSS region {@value}.
+ */
+ public static final String REGION_KEY = "fs.oss.region";
}
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
new file mode 100644
index 0000000000..5070f2a581
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.REGION_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SIGNATURE_VERSION_KEY;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.junit.Assert.*;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Tests Aliyun OSS system.
+ */
+public class ITAliyunOSSSignatureV4 {
+ private static final Logger LOG = LoggerFactory.getLogger(ITAliyunOSSSignatureV4.class);
+ private Configuration conf;
+ private URI testURI;
+ private Path testFile = new Path("ITAliyunOSSSignatureV4/atestr");
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ String bucketUri = conf.get("test.fs.oss.name");
+ LOG.debug("bucketUri={}", bucketUri);
+ testURI = URI.create(bucketUri);
+ }
+
+ @Test
+ public void testV4() throws IOException {
+ conf.set(SIGNATURE_VERSION_KEY, "V4");
+ conf.set(REGION_KEY, "cn-hongkong");
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+ fs.initialize(testURI, conf);
+ assumeNotNull(fs);
+
+ createFile(fs, testFile, true, dataset(256, 0, 255));
+ FileStatus status = fs.getFileStatus(testFile);
+ fs.delete(testFile);
+ fs.close();
+ }
+
+ @Test
+ public void testDefaultSignatureVersion() throws IOException {
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+ fs.initialize(testURI, conf);
+ assumeNotNull(fs);
+
+ Path testFile2 = new Path("/test/atestr");
+ createFile(fs, testFile2, true, dataset(256, 0, 255));
+ FileStatus status = fs.getFileStatus(testFile2);
+ fs.delete(testFile2);
+ fs.close();
+ }
+
+ @Test
+ public void testV4WithoutRegion() throws IOException {
+ conf.set(SIGNATURE_VERSION_KEY, "V4");
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+ IOException expectedException = null;
+ try {
+ fs.initialize(testURI, conf);
+ } catch (IOException e) {
+ LOG.warn("use V4 , but do not set region, get exception={}", e);
+ expectedException = e;
+ assertEquals("use V4 , but do not set region", e.getMessage(),
+ "SignVersion is V4 but region is empty");
+ }
+ assertNotNull(expectedException);
+ }
+}
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
index bb5cbe5ec3..2167f68811 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
@@ -21,3 +21,6 @@ log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# Log all oss classes
+log4j.logger.org.apache.hadoop.fs.aliyun.oss=DEBUG
\ No newline at end of file
134 changes: 134 additions & 0 deletions ...p/stackable/patches/3.4.1/0007-HADOOP-19335.-Bump-netty-to-4.1.116-due-to-CVE-2024-.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
From 147d927da55853b2d1f4ab29f5cb010fc339ad35 Mon Sep 17 00:00:00 2001
From: PJ Fanning <pjfanning@users.noreply.github.com>
Date: Thu, 2 Jan 2025 09:26:20 +0100
Subject: HADOOP-19335. Bump netty to 4.1.116 due to CVE-2024-47535 (#7158).
Contributed by PJ Fanning (#7259)

---
LICENSE-binary | 76 +++++++++----------
hadoop-project/pom.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-csi/pom.xml | 4 +-
3 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/LICENSE-binary b/LICENSE-binary
index b064b6a15d..3bcc589734 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -250,43 +250,43 @@ commons-daemon:commons-daemon:1.0.13
commons-io:commons-io:2.16.1
commons-net:commons-net:3.9.0
de.ruedigermoeller:fst:2.50
-io.grpc:grpc-api:1.53.0
-io.grpc:grpc-context:1.53.0
-io.grpc:grpc-core:1.53.0
-io.grpc:grpc-netty:1.53.0
-io.grpc:grpc-protobuf:1.53.0
-io.grpc:grpc-protobuf-lite:1.53.0
-io.grpc:grpc-stub:1.53.0
-io.netty:netty-all:4.1.100.Final
-io.netty:netty-buffer:4.1.100.Final
-io.netty:netty-codec:4.1.100.Final
-io.netty:netty-codec-dns:4.1.100.Final
-io.netty:netty-codec-haproxy:4.1.100.Final
-io.netty:netty-codec-http:4.1.100.Final
-io.netty:netty-codec-http2:4.1.100.Final
-io.netty:netty-codec-memcache:4.1.100.Final
-io.netty:netty-codec-mqtt:4.1.100.Final
-io.netty:netty-codec-redis:4.1.100.Final
-io.netty:netty-codec-smtp:4.1.100.Final
-io.netty:netty-codec-socks:4.1.100.Final
-io.netty:netty-codec-stomp:4.1.100.Final
-io.netty:netty-codec-xml:4.1.100.Final
-io.netty:netty-common:4.1.100.Final
-io.netty:netty-handler:4.1.100.Final
-io.netty:netty-handler-proxy:4.1.100.Final
-io.netty:netty-resolver:4.1.100.Final
-io.netty:netty-resolver-dns:4.1.100.Final
-io.netty:netty-transport:4.1.100.Final
-io.netty:netty-transport-rxtx:4.1.100.Final
-io.netty:netty-transport-sctp:4.1.100.Final
-io.netty:netty-transport-udt:4.1.100.Final
-io.netty:netty-transport-classes-epoll:4.1.100.Final
-io.netty:netty-transport-native-unix-common:4.1.100.Final
-io.netty:netty-transport-classes-kqueue:4.1.100.Final
-io.netty:netty-resolver-dns-classes-macos:4.1.100.Final
-io.netty:netty-transport-native-epoll:4.1.100.Final
-io.netty:netty-transport-native-kqueue:4.1.100.Final
-io.netty:netty-resolver-dns-native-macos:4.1.100.Final
+io.grpc:grpc-api:1.69.0
+io.grpc:grpc-context:1.69.0
+io.grpc:grpc-core:1.69.0
+io.grpc:grpc-netty:1.69.0
+io.grpc:grpc-protobuf:1.69.0
+io.grpc:grpc-protobuf-lite:1.69.0
+io.grpc:grpc-stub:1.69.0
+io.netty:netty-all:4.1.116.Final
+io.netty:netty-buffer:4.1.116.Final
+io.netty:netty-codec:4.1.116.Final
+io.netty:netty-codec-dns:4.1.116.Final
+io.netty:netty-codec-haproxy:4.1.116.Final
+io.netty:netty-codec-http:4.1.116.Final
+io.netty:netty-codec-http2:4.1.116.Final
+io.netty:netty-codec-memcache:4.1.116.Final
+io.netty:netty-codec-mqtt:4.1.116.Final
+io.netty:netty-codec-redis:4.1.116.Final
+io.netty:netty-codec-smtp:4.1.116.Final
+io.netty:netty-codec-socks:4.1.116.Final
+io.netty:netty-codec-stomp:4.1.116.Final
+io.netty:netty-codec-xml:4.1.116.Final
+io.netty:netty-common:4.1.116.Final
+io.netty:netty-handler:4.1.116.Final
+io.netty:netty-handler-proxy:4.1.116.Final
+io.netty:netty-resolver:4.1.116.Final
+io.netty:netty-resolver-dns:4.1.116.Final
+io.netty:netty-transport:4.1.116.Final
+io.netty:netty-transport-rxtx:4.1.116.Final
+io.netty:netty-transport-sctp:4.1.116.Final
+io.netty:netty-transport-udt:4.1.116.Final
+io.netty:netty-transport-classes-epoll:4.1.116.Final
+io.netty:netty-transport-native-unix-common:4.1.116.Final
+io.netty:netty-transport-classes-kqueue:4.1.116.Final
+io.netty:netty-resolver-dns-classes-macos:4.1.116.Final
+io.netty:netty-transport-native-epoll:4.1.116.Final
+io.netty:netty-transport-native-kqueue:4.1.116.Final
+io.netty:netty-resolver-dns-native-macos:4.1.116.Final
io.opencensus:opencensus-api:0.12.3
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
io.reactivex:rxjava:1.3.8
@@ -486,7 +486,7 @@ org.bouncycastle:bcpkix-jdk18on:1.78.1
org.bouncycastle:bcprov-jdk18on:1.78.1
org.bouncycastle:bcutil-jdk18on:1.78.1
org.checkerframework:checker-qual:3.8.0
-org.codehaus.mojo:animal-sniffer-annotations:1.21
+org.codehaus.mojo:animal-sniffer-annotations:1.24
org.jruby.jcodings:jcodings:1.0.13
org.jruby.joni:joni:2.1.2
org.ojalgo:ojalgo:43.0
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3a812ebc64..f921df0209 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -143,7 +143,7 @@
<jna.version>5.2.0</jna.version>
<gson.version>2.9.0</gson.version>
<metrics.version>3.2.4</metrics.version>
- <netty4.version>4.1.100.Final</netty4.version>
+ <netty4.version>4.1.116.Final</netty4.version>
<snappy-java.version>1.1.10.4</snappy-java.version>
<lz4-java.version>1.7.1</lz4-java.version>

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
index c66120798d..909b59d317 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
@@ -26,8 +26,8 @@
<packaging>jar</packaging>

<properties>
- <grpc.version>1.53.0</grpc.version>
- <animal-sniffer.version>1.21</animal-sniffer.version>
+ <grpc.version>1.69.0</grpc.version>
+ <animal-sniffer.version>1.24</animal-sniffer.version>
</properties>

<dependencies>
Loading