ODP-2013: ODP-1095 Critical CVE fixes

acceldata-io · Aug 12, 2024 · 4095c5c · 4095c5c
1 parent 47e5980
commit 4095c5c
Show file tree

Hide file tree

Showing 74 changed files with 713 additions and 297 deletions.
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -130,6 +130,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.curator</groupId>
           <artifactId>curator-client</artifactId>
@@ -417,29 +421,25 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
       <optional>true</optional>
       <exclusions>
         <exclusion>
-          <groupId>javax.xml.bind</groupId>
-          <artifactId>jaxb-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-jaxrs</artifactId>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-xc</artifactId>
+          <groupId>com.sun.xml.bind</groupId>
+          <artifactId>jaxb-impl</artifactId>
         </exclusion>
       </exclusions>
     </dependency>
@@ -452,6 +452,20 @@
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-servlet</artifactId>
       <optional>true</optional>
+      exclusions>
+      <exclusion>
+        <groupId>javax.servlet</groupId>
+        <artifactId>servlet-api</artifactId>
+      </exclusion>
+      <exclusion>
+        <groupId>javax.enterprise</groupId>
+        <artifactId>cdi-api</artifactId>
+      </exclusion>
+      <exclusion>
+        <groupId>ch.qos.cal10n</groupId>
+        <artifactId>cal10n-api</artifactId>
+      </exclusion>
+    </exclusions>
     </dependency>
     <!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
     <dependency>
@@ -729,6 +743,12 @@
                         <exclude>META-INF/versions/9/module-info.class</exclude>
                       </excludes>
                     </filter>
+                    <filter>
+                      <artifact>com.google.code.gson:gson</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/9/module-info.class</exclude>
+                      </excludes>
+                    </filter>
                     <!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
                     <filter>
                       <artifact>org.mockito:mockito-all</artifact>

diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
@@ -65,6 +65,14 @@
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.github.pjfanning</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jettison</groupId>
+          <artifactId>jettison</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
@@ -110,6 +118,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
@@ -162,14 +174,22 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jettison</groupId>
+          <artifactId>jettison</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>io.netty</groupId>
           <artifactId>netty-all</artifactId>

diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
@@ -133,12 +133,32 @@
       <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-servlet</artifactId>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.enterprise</groupId>
+          <artifactId>cdi-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>ch.qos.cal10n</groupId>
+          <artifactId>cal10n-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <!-- Used, even though 'mvn dependency:analyze' doesn't find it -->
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.codehaus.jettison</groupId>
+          <artifactId>jettison</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.sun.jersey</groupId>
@@ -149,7 +169,6 @@
       <groupId>commons-logging</groupId>
       <artifactId>commons-logging</artifactId>
       <scope>compile</scope>
-
       <exclusions>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
@@ -196,6 +215,14 @@
       <artifactId>commons-lang3</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <!--
+      adding jettison as direct dependency (as jersey-json's jettison dependency is vulnerable with verison 1.1),
+      so those who depends on hadoop-common externally will get the non-vulnerable jettison
+      -->
+      <groupId>org.codehaus.jettison</groupId>
+      <artifactId>jettison</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-text</artifactId>
@@ -217,8 +244,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro</artifactId>
+      <groupId>org.apache.hadoop.thirdparty</groupId>
+      <artifactId>hadoop-shaded-avro_1_11</artifactId>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -330,6 +357,10 @@
       <groupId>org.apache.kerby</groupId>
       <artifactId>kerb-simplekdc</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
@@ -485,6 +516,7 @@
                 <copy toDir="${project.build.directory}/test-classes">
                   <fileset dir="${basedir}/src/main/conf"/>
                 </copy>
+                <replace dir="${project.build.directory}/generated-test-sources/java" token="org.apache.avro" value="org.apache.hadoop.thirdparty.avro"/>
               </target>
             </configuration>
           </execution>

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
@@ -21,7 +21,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 
-import org.apache.avro.file.SeekableInput;
+import org.apache.hadoop.thirdparty.avro.file.SeekableInput;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
@@ -26,7 +26,7 @@
 import java.net.URISyntaxException;
 import java.util.regex.Pattern;
 
-import org.apache.avro.reflect.Stringable;
+import org.apache.hadoop.thirdparty.avro.reflect.Stringable;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -25,14 +25,14 @@
 import java.util.LinkedList;
 import java.util.zip.GZIPInputStream;
 
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.file.FileReader;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.io.JsonEncoder;
+import org.apache.hadoop.thirdparty.avro.Schema;
+import org.apache.hadoop.thirdparty.avro.file.DataFileReader;
+import org.apache.hadoop.thirdparty.avro.file.FileReader;
+import org.apache.hadoop.thirdparty.avro.generic.GenericDatumReader;
+import org.apache.hadoop.thirdparty.avro.generic.GenericDatumWriter;
+import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
+import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
+import org.apache.hadoop.thirdparty.avro.io.JsonEncoder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -33,7 +33,7 @@
 import java.text.StringCharacterIterator;
 import java.util.Arrays;
 
-import org.apache.avro.reflect.Stringable;
+import org.apache.hadoop.thirdparty.avro.reflect.Stringable;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;

diff --git a/...p-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java b/...p-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
@@ -21,12 +21,12 @@
 import java.util.HashSet;
 import java.util.Set;
 
-import org.apache.avro.Schema;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.reflect.ReflectData;
-import org.apache.avro.reflect.ReflectDatumReader;
-import org.apache.avro.reflect.ReflectDatumWriter;
+import org.apache.hadoop.thirdparty.avro.Schema;
+import org.apache.hadoop.thirdparty.avro.io.DatumReader;
+import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
+import org.apache.hadoop.thirdparty.avro.reflect.ReflectData;
+import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader;
+import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 

diff --git a/...t/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java b/...t/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
@@ -22,13 +22,13 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 
-import org.apache.avro.Schema;
-import org.apache.avro.io.BinaryDecoder;
-import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.io.DecoderFactory;
-import org.apache.avro.io.EncoderFactory;
+import org.apache.hadoop.thirdparty.avro.Schema;
+import org.apache.hadoop.thirdparty.avro.io.BinaryDecoder;
+import org.apache.hadoop.thirdparty.avro.io.BinaryEncoder;
+import org.apache.hadoop.thirdparty.avro.io.DatumReader;
+import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
+import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;
+import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configured;

diff --git a/...-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java b/...-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java
@@ -18,12 +18,12 @@
 
 package org.apache.hadoop.io.serializer.avro;
 
-import org.apache.avro.Schema;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.specific.SpecificRecord;
+import org.apache.hadoop.thirdparty.avro.Schema;
+import org.apache.hadoop.thirdparty.avro.io.DatumReader;
+import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
+import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader;
+import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter;
+import org.apache.hadoop.thirdparty.avro.specific.SpecificRecord;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 

diff --git a/...on-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java b/...on-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
@@ -21,8 +21,10 @@
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectWriter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

diff --git a/...adoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java b/...adoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java
@@ -25,7 +25,7 @@
 
 import javax.crypto.SecretKey;
 
-import org.apache.avro.reflect.Nullable;
+import org.apache.hadoop.thirdparty.avro.reflect.Nullable;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.Writable;

diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java
@@ -21,12 +21,12 @@
 import java.io.ByteArrayOutputStream;
 import java.lang.reflect.Type;
 
-import org.apache.avro.Schema;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.reflect.ReflectData;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.avro.reflect.ReflectDatumReader;
-import org.apache.avro.io.DecoderFactory;
+import org.apache.hadoop.thirdparty.avro.Schema;
+import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
+import org.apache.hadoop.thirdparty.avro.reflect.ReflectData;
+import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter;
+import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader;
+import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;
 
 import static org.junit.Assert.assertEquals;
 

diff --git a/...-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java b/...-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java
@@ -118,7 +118,8 @@ public void testSerializeAndDeserializeNull() throws IOException {
   @Test
   public void testAvroReflect() throws Exception {
     String schema = "{\"type\":\"array\",\"items\":{\"type\":\"enum\","
-        + "\"name\":\"TestEnumSet\","
+        +   "\"name\":\"TestEnumSet\","
+        + "\"namespace\":\"org.apache.hadoop.io.TestEnumSetWritable\","
         + "\"namespace\":\"org.apache.hadoop.io.TestEnumSetWritable\","
         + "\"symbols\":[\"CREATE\",\"OVERWRITE\",\"APPEND\"]},"
         + "\"java-class\":\"org.apache.hadoop.io.EnumSetWritable\"}";

diff --git a/...doop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java b/...doop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
@@ -33,7 +33,7 @@ public class TestAvroSerialization {
   @Test
   public void testSpecific() throws Exception {
     AvroRecord before = new AvroRecord();
-    before.intField = 5;
+    before.setIntField(5);
     AvroRecord after = SerializationTestUtil.testSerialization(conf, before);
     assertEquals(before, after);
   }