Skip to content

Commit

Permalink
Merge branch 'apache:trunk' into HADOOP-18302
Browse files Browse the repository at this point in the history
  • Loading branch information
@slfan1989
slfan1989 authored Sep 11, 2022
2 parents 27fd624 + 21bae31 commit 7421ba8
Show file tree
Hide file tree
Showing 206 changed files with 8,074 additions and 1,299 deletions.
4 changes: 2 additions & 2 deletions LICENSE-binary
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ commons-collections:commons-collections:3.2.2
commons-daemon:commons-daemon:1.0.13
commons-io:commons-io:2.8.0
commons-logging:commons-logging:1.1.3
commons-net:commons-net:3.6
commons-net:commons-net:3.8.0
de.ruedigermoeller:fst:2.50
io.grpc:grpc-api:1.26.0
io.grpc:grpc-context:1.26.0
Expand Down Expand Up @@ -362,7 +362,7 @@ org.ehcache:ehcache:3.3.1
org.lz4:lz4-java:1.7.1
org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.0.5
org.yaml:snakeyaml:1.16:
org.yaml:snakeyaml:1.31:
org.wildfly.openssl:wildfly-openssl:1.0.7.Final


Expand Down
10 changes: 1 addition & 9 deletions hadoop-client-modules/hadoop-client-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,6 @@
<createSourcesJar>true</createSourcesJar>
<shadeSourcesContent>true</shadeSourcesContent>
</configuration>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
Expand Down Expand Up @@ -254,8 +247,7 @@
</relocation>
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resource>NOTICE.txt</resource>
Expand Down
10 changes: 1 addition & 9 deletions hadoop-client-modules/hadoop-client-minicluster/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -671,13 +671,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
Expand Down Expand Up @@ -1052,8 +1045,7 @@
</relocation>
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
Expand Down
10 changes: 1 addition & 9 deletions hadoop-client-modules/hadoop-client-runtime/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
Expand Down Expand Up @@ -397,8 +390,7 @@
-->
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
Expand Down
84 changes: 84 additions & 0 deletions hadoop-common-project/hadoop-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,36 @@
</execution>
</executions>
</plugin>
<plugin>
<!--Sets the skip.platformToolsetDetection to true if use.platformToolsetVersion is specified.
This implies that the automatic detection of which platform toolset to use will be skipped
and the one specified with use.platformToolsetVersion will be used.-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target>
<condition property="skip.platformToolsetDetection" value="true" else="false">
<isset property="use.platformToolsetVersion"/>
</condition>
<!--Unfortunately, Maven doesn't have a way to negate a flag, thus we declare a
property which holds the negated value of skip.platformToolsetDetection.-->
<condition property="skip.platformToolsetDetection.negated" value="false" else="true">
<isset property="use.platformToolsetVersion"/>
</condition>
<echo>Skip platform toolset version detection = ${skip.platformToolsetDetection}</echo>
</target>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
Expand All @@ -852,6 +882,7 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
<arguments>
<argument>${basedir}\src\main\winutils</argument>
Expand All @@ -866,6 +897,27 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
<argument>/nologo</argument>
<argument>/p:Configuration=Release</argument>
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>compile-ms-winutils-using-build-tools</id>
<phase>compile</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection.negated}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
Expand All @@ -875,6 +927,7 @@
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
</arguments>
</configuration>
</execution>
Expand All @@ -885,6 +938,7 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
<arguments>
<argument>${basedir}\src\main\native</argument>
Expand All @@ -899,6 +953,35 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/native/native.sln</argument>
<argument>/nologo</argument>
<argument>/p:Configuration=Release</argument>
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
<argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
<argument>/p:CustomZstdLib=${zstd.lib}</argument>
<argument>/p:CustomZstdInclude=${zstd.include}</argument>
<argument>/p:RequireZstd=${require.zstd}</argument>
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
<argument>/p:RequireOpenssl=${require.openssl}</argument>
<argument>/p:RequireIsal=${require.isal}</argument>
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
<argument>/p:CustomIsalLib=${isal.lib}</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>compile-ms-native-dll-using-build-tools</id>
<phase>compile</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection.negated}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/native/native.sln</argument>
Expand All @@ -916,6 +999,7 @@
<argument>/p:RequireIsal=${require.isal}</argument>
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
<argument>/p:CustomIsalLib=${isal.lib}</argument>
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
</arguments>
</configuration>
</execution>
Expand Down
74 changes: 64 additions & 10 deletions ...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ private static class ChecksumFSInputChecker extends FSInputChecker implements
private static final int HEADER_LENGTH = 8;

private int bytesPerSum = 1;
private long fileLen = -1L;

public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
throws IOException {
Expand Down Expand Up @@ -320,6 +321,18 @@ public static long findChecksumOffset(long dataOffset,
return HEADER_LENGTH + (dataOffset/bytesPerSum) * FSInputChecker.CHECKSUM_SIZE;
}

/**
* Calculate length of file if not already cached.
* @return file length.
* @throws IOException any IOE.
*/
private long getFileLength() throws IOException {
if (fileLen == -1L) {
fileLen = fs.getFileStatus(file).getLen();
}
return fileLen;
}

/**
* Find the checksum ranges that correspond to the given data ranges.
* @param dataRanges the input data ranges, which are assumed to be sorted
Expand Down Expand Up @@ -371,13 +384,28 @@ static ByteBuffer checkBytes(ByteBuffer sumsBytes,
IntBuffer sums = sumsBytes.asIntBuffer();
sums.position(offset / FSInputChecker.CHECKSUM_SIZE);
ByteBuffer current = data.duplicate();
int numChunks = data.remaining() / bytesPerSum;
int numFullChunks = data.remaining() / bytesPerSum;
boolean partialChunk = ((data.remaining() % bytesPerSum) != 0);
int totalChunks = numFullChunks;
if (partialChunk) {
totalChunks++;
}
CRC32 crc = new CRC32();
// check each chunk to ensure they match
for(int c = 0; c < numChunks; ++c) {
// set the buffer position and the limit
current.limit((c + 1) * bytesPerSum);
for(int c = 0; c < totalChunks; ++c) {
// set the buffer position to the start of every chunk.
current.position(c * bytesPerSum);

if (c == numFullChunks) {
// During last chunk, there may be less than chunk size
// data preset, so setting the limit accordingly.
int lastIncompleteChunk = data.remaining() % bytesPerSum;
current.limit((c * bytesPerSum) + lastIncompleteChunk);
} else {
// set the buffer limit to end of every chunk.
current.limit((c + 1) * bytesPerSum);
}

// compute the crc
crc.reset();
crc.update(current);
Expand All @@ -396,11 +424,34 @@ static ByteBuffer checkBytes(ByteBuffer sumsBytes,
return data;
}

/**
* Validates range parameters.
* In case of CheckSum FS, we already have calculated
* fileLength so failing fast here.
* @param ranges requested ranges.
* @param fileLength length of file.
* @throws EOFException end of file exception.
*/
private void validateRangeRequest(List<? extends FileRange> ranges,
final long fileLength) throws EOFException {
for (FileRange range : ranges) {
VectoredReadUtils.validateRangeRequest(range);
if (range.getOffset() + range.getLength() > fileLength) {
final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s",
range.getOffset(), range.getLength(), file);
LOG.warn(errMsg);
throw new EOFException(errMsg);
}
}
}

@Override
public void readVectored(List<? extends FileRange> ranges,
IntFunction<ByteBuffer> allocate) throws IOException {
final long length = getFileLength();
validateRangeRequest(ranges, length);

// If the stream doesn't have checksums, just delegate.
VectoredReadUtils.validateVectoredReadRanges(ranges);
if (sums == null) {
datas.readVectored(ranges, allocate);
return;
Expand All @@ -410,15 +461,18 @@ public void readVectored(List<? extends FileRange> ranges,
List<CombinedFileRange> dataRanges =
VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(ranges)), bytesPerSum,
minSeek, maxReadSizeForVectorReads());
// While merging the ranges above, they are rounded up based on the value of bytesPerSum
// which leads to some ranges crossing the EOF thus they need to be fixed else it will
// cause EOFException during actual reads.
for (CombinedFileRange range : dataRanges) {
if (range.getOffset() + range.getLength() > length) {
range.setLength((int) (length - range.getOffset()));
}
}
List<CombinedFileRange> checksumRanges = findChecksumRanges(dataRanges,
bytesPerSum, minSeek, maxSize);
sums.readVectored(checksumRanges, allocate);
datas.readVectored(dataRanges, allocate);
// Data read is correct. I have verified content of dataRanges.
// There is some bug below here as test (testVectoredReadMultipleRanges)
// is failing, should be
// somewhere while slicing the merged data into smaller user ranges.
// Spend some time figuring out but it is a complex code.
for(CombinedFileRange checksumRange: checksumRanges) {
for(FileRange dataRange: checksumRange.getUnderlying()) {
// when we have both the ranges, validate the checksum
Expand Down
Loading

0 comments on commit 7421ba8

Please sign in to comment.