Skip to content

Commit 2f75bb7

Browse files
steveloughranTom McCormick
authored andcommitted
HADOOP-15033. Use java.util.zip.CRC32C for Java 9 and above
Contributed by Dmitry Chuyko, (cherry picked from commit 2e0a451)
1 parent 728e8af commit 2f75bb7

File tree

5 files changed

+96
-2
lines changed

5 files changed

+96
-2
lines changed

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@
2828
import org.apache.hadoop.classification.InterfaceAudience;
2929
import org.apache.hadoop.classification.InterfaceStability;
3030
import org.apache.hadoop.fs.ChecksumException;
31+
import org.slf4j.Logger;
32+
import org.slf4j.LoggerFactory;
33+
34+
import java.lang.invoke.MethodHandle;
35+
import java.lang.invoke.MethodHandles;
36+
import java.lang.invoke.MethodType;
3137

3238
/**
3339
* This class provides interface and utilities for processing checksums for
@@ -43,6 +49,9 @@ public class DataChecksum implements Checksum {
4349
public static final int CHECKSUM_CRC32C = 2;
4450
public static final int CHECKSUM_DEFAULT = 3;
4551
public static final int CHECKSUM_MIXED = 4;
52+
53+
private static final Logger LOG = LoggerFactory.getLogger(DataChecksum.class);
54+
private static volatile boolean useJava9Crc32C = Shell.isJavaVersionAtLeast(9);
4655

4756
/** The checksum types */
4857
public static enum Type {
@@ -78,6 +87,23 @@ public static Checksum newCrc32() {
7887
return new CRC32();
7988
}
8089

90+
91+
/**
92+
* The flag is volatile to avoid synchronization here.
93+
* Re-entrancy is unlikely except in failure mode (and inexpensive).
94+
*/
95+
static Checksum newCrc32C() {
96+
try {
97+
return useJava9Crc32C ? Java9Crc32CFactory.createChecksum()
98+
: new PureJavaCrc32C();
99+
} catch (ExceptionInInitializerError | RuntimeException e) {
100+
// should not happen
101+
LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e);
102+
useJava9Crc32C = false;
103+
return new PureJavaCrc32C();
104+
}
105+
}
106+
81107
public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
82108
if ( bytesPerChecksum <= 0 ) {
83109
return null;
@@ -89,7 +115,7 @@ public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
89115
case CRC32 :
90116
return new DataChecksum(type, newCrc32(), bytesPerChecksum );
91117
case CRC32C:
92-
return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
118+
return new DataChecksum(type, newCrc32C(), bytesPerChecksum);
93119
default:
94120
return null;
95121
}
@@ -520,4 +546,36 @@ public void update(byte[] b, int off, int len) {}
520546
@Override
521547
public void update(int b) {}
522548
};
549+
550+
/**
551+
* Holds constructor handle to let it be initialized on demand.
552+
*/
553+
private static class Java9Crc32CFactory {
554+
private static final MethodHandle NEW_CRC32C_MH;
555+
556+
static {
557+
MethodHandle newCRC32C = null;
558+
try {
559+
newCRC32C = MethodHandles.publicLookup()
560+
.findConstructor(
561+
Class.forName("java.util.zip.CRC32C"),
562+
MethodType.methodType(void.class)
563+
);
564+
} catch (ReflectiveOperationException e) {
565+
// Should not reach here.
566+
throw new RuntimeException(e);
567+
}
568+
NEW_CRC32C_MH = newCRC32C;
569+
}
570+
571+
public static Checksum createChecksum() {
572+
try {
573+
// Should throw nothing
574+
return (Checksum) NEW_CRC32C_MH.invoke();
575+
} catch (Throwable t) {
576+
throw (t instanceof RuntimeException) ? (RuntimeException) t
577+
: new RuntimeException(t);
578+
}
579+
}
580+
};
523581
}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,21 @@ public static boolean isJava7OrAbove() {
8989
return true;
9090
}
9191

92+
// "1.8"->8, "9"->9, "10"->10
93+
private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt(
94+
System.getProperty("java.specification.version").split("\\.")[0]));
95+
96+
/**
97+
* Query to see if major version of Java specification of the system
98+
* is equal or greater than the parameter.
99+
*
100+
* @param version 8, 9, 10 etc.
101+
* @return comparison with system property, always true for 8
102+
*/
103+
public static boolean isJavaVersionAtLeast(int version) {
104+
return JAVA_SPEC_VER >= version;
105+
}
106+
92107
/**
93108
* Maximum command line length in Windows
94109
* KB830473 documents this as 8191

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,18 @@ public DataChecksum.Type crcType() {
129129
}
130130
}
131131

132+
final class ZipC extends AbstractCrc32<Checksum> {
133+
@Override
134+
public Checksum newAlgorithm() {
135+
return DataChecksum.newCrc32C();
136+
}
137+
138+
@Override
139+
public DataChecksum.Type crcType() {
140+
return DataChecksum.Type.CRC32C;
141+
}
142+
}
143+
132144
final class PureJava extends AbstractCrc32<PureJavaCrc32> {
133145
@Override
134146
public PureJavaCrc32 newAlgorithm() {
@@ -169,6 +181,9 @@ public DataChecksum.Type crcType() {
169181
this.direct = direct;
170182

171183
crcs.add(Crc32.Zip.class);
184+
if (Shell.isJavaVersionAtLeast(9)) {
185+
crcs.add(Crc32.ZipC.class);
186+
}
172187
crcs.add(Crc32.PureJava.class);
173188
crcs.add(Crc32.PureJavaC.class);
174189

@@ -435,4 +450,4 @@ static void printSystemProperties(PrintStream outCrc) {
435450
outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n));
436451
}
437452
}
438-
}
453+
}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -528,4 +528,9 @@ public Boolean get() {
528528
shexc1.getProcess().waitFor();
529529
shexc2.getProcess().waitFor();
530530
}
531+
532+
@Test
533+
public void testIsJavaVersionAtLeast() {
534+
assertTrue(Shell.isJavaVersionAtLeast(8));
535+
}
531536
}

hadoop-project/pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1538,6 +1538,7 @@
15381538
<ignore>sun.nio.ch.*</ignore>
15391539
<ignore>com.sun.javadoc.*</ignore>
15401540
<ignore>com.sun.tools.*</ignore>
1541+
<ignore>java.lang.invoke.*</ignore>
15411542
</ignores>
15421543
</configuration>
15431544
</plugin>

0 commit comments

Comments
 (0)