add UT

Author: Apache9

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufWALReader.java

@@ -522,7 +522,8 @@ protected final boolean isEOF(InvalidProtocolBufferException e) {
    */
   protected final boolean isWALTrailer(long startPosition) throws IOException {
     // We have nothing in the WALTrailer PB message now so its size is just a int length size and a
-    // magic at the end, in TestWALTrailer we have a test to make sure its serialized size is 0
+    // magic at the end, in TestParsePartialWALFile we have a test to make sure its serialized size
+    // is 0
     int trailerSize = PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT;
     if (fileLength - startPosition >= trailerSize) {
       // We still have more than trailerSize bytes before reaching the EOF so this is not a trailer.

hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestParsePartialWALFile.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell.Type;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestParsePartialWALFile {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+    HBaseClassTestRule.forClass(TestParsePartialWALFile.class);
+
+  private static final HBaseCommonTestingUtil UTIL = new HBaseCommonTestingUtil();
+
+  @BeforeClass
+  public static void setUp() {
+    UTIL.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    UTIL.cleanupTestDir();
+  }
+
+  @Test
+  public void testTrailerSize() {
+    // make sure that the trailer size is currently still zero, as when parsing broken WAL file
+    // without a valid trailer, we need this assumption to determine whether there is a broken
+    // trailer
+    WALTrailer trailer = WALTrailer.newBuilder().build();
+    assertEquals(0, trailer.getSerializedSize());
+  }
+
+  private Path generateBrokenWALFile(FileSystem fs, byte[] content, int length) throws IOException {
+    Path walFile = UTIL.getDataTestDir("wal-" + length);
+    try (FSDataOutputStream out = fs.create(walFile)) {
+      out.write(content, 0, length);
+    }
+    return walFile;
+  }
+
+  // TODO: can not pass yet, need modify the way on how we parse PB message
+  @Ignore
+  @Test
+  public void testPartialParse() throws Exception {
+    FileSystem fs = FileSystem.getLocal(UTIL.getConfiguration());
+    if (!fs.mkdirs(UTIL.getDataTestDir())) {
+      throw new IOException("can not create " + UTIL.getDataTestDir());
+    }
+    TableName tn = TableName.valueOf("test");
+    RegionInfo ri = RegionInfoBuilder.newBuilder(tn).build();
+    byte[] row = Bytes.toBytes("row");
+    byte[] family = Bytes.toBytes("family");
+    byte[] qual = Bytes.toBytes("qualifier");
+    byte[] value = Bytes.toBytes("value");
+    Path walFile = UTIL.getDataTestDir("wal");
+
+    long headerLength;
+    List<Long> endOffsets = new ArrayList<>();
+    try (WALProvider.Writer writer =
+      WALFactory.createWALWriter(fs, walFile, UTIL.getConfiguration())) {
+      headerLength = writer.getLength();
+      for (int i = 0; i < 2; i++) {
+        WALKeyImpl key = new WALKeyImpl(ri.getEncodedNameAsBytes(), tn, i,
+          EnvironmentEdgeManager.currentTime(), HConstants.DEFAULT_CLUSTER_ID);
+        WALEdit edit = new WALEdit();
+        edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Type.Put)
+          .setRow(row).setFamily(family).setQualifier(qual).setValue(value).build());
+        writer.append(new WAL.Entry(key, edit));
+        writer.sync(true);
+        endOffsets.add(writer.getLength());
+      }
+    }
+    long fileLength = fs.getFileStatus(walFile).getLen();
+    byte[] content = new byte[(int) fileLength];
+    try (FSDataInputStream in = fs.open(walFile)) {
+      in.readFully(content);
+    }
+    // partial header, should throw WALHeaderEOFException
+    for (int i = 0; i < headerLength; i++) {
+      Path brokenFile = generateBrokenWALFile(fs, content, i);
+      assertThrows(WALHeaderEOFException.class,
+        () -> WALFactory.createStreamReader(fs, brokenFile, UTIL.getConfiguration()));
+      assertThrows(WALHeaderEOFException.class,
+        () -> WALFactory.createTailingReader(fs, brokenFile, UTIL.getConfiguration(), -1));
+      fs.delete(brokenFile, false);
+    }
+  }
+}