HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 (apache#209)

SnapshotFileCache depends on getting the last modified time of the
snapshot directory, however, S3 FileSystem's do not update the
last modified time of the top 'folder' when objects are added/removed.
This commit adds a test for the previously fixed SnapshotFileCache.

Author: z-york

hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java

@@ -68,14 +68,18 @@ public class TestSnapshotFileCache {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class);
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  // don't refresh the cache unless we tell it to
+  private static final long PERIOD = Long.MAX_VALUE;
   private static FileSystem fs;
   private static Path rootDir;
+  private static Path snapshotDir;
 
   @BeforeClass
   public static void startCluster() throws Exception {
     UTIL.startMiniDFSCluster(1);
     fs = UTIL.getDFSCluster().getFileSystem();
     rootDir = UTIL.getDefaultRootDirPath();
+    snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
   }
 
   @AfterClass
@@ -86,56 +90,59 @@ public static void stopCluster() throws Exception {
   @After
   public void cleanupFiles() throws Exception {
     // cleanup the snapshot directory
-    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
     fs.delete(snapshotDir, true);
   }
 
   @Test
   public void testLoadAndDelete() throws IOException {
-    // don't refresh the cache unless we tell it to
-    long period = Long.MAX_VALUE;
-    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
         "test-snapshot-file-cache-refresh", new SnapshotFiles());
 
-    createAndTestSnapshotV1(cache, "snapshot1a", false, true);
-    createAndTestSnapshotV1(cache, "snapshot1b", true, true);
+    createAndTestSnapshotV1(cache, "snapshot1a", false, true, false);
+    createAndTestSnapshotV1(cache, "snapshot1b", true, true, false);
 
-    createAndTestSnapshotV2(cache, "snapshot2a", false, true);
-    createAndTestSnapshotV2(cache, "snapshot2b", true, true);
+    createAndTestSnapshotV2(cache, "snapshot2a", false, true, false);
+    createAndTestSnapshotV2(cache, "snapshot2b", true, true, false);
   }
 
   @Test
   public void testReloadModifiedDirectory() throws IOException {
-    // don't refresh the cache unless we tell it to
-    long period = Long.MAX_VALUE;
-    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
         "test-snapshot-file-cache-refresh", new SnapshotFiles());
 
-    createAndTestSnapshotV1(cache, "snapshot1", false, true);
+    createAndTestSnapshotV1(cache, "snapshot1", false, true, false);
     // now delete the snapshot and add a file with a different name
-    createAndTestSnapshotV1(cache, "snapshot1", false, false);
+    createAndTestSnapshotV1(cache, "snapshot1", false, false, false);
 
-    createAndTestSnapshotV2(cache, "snapshot2", false, true);
+    createAndTestSnapshotV2(cache, "snapshot2", false, true, false);
     // now delete the snapshot and add a file with a different name
-    createAndTestSnapshotV2(cache, "snapshot2", false, false);
+    createAndTestSnapshotV2(cache, "snapshot2", false, false, false);
   }
 
   @Test
   public void testSnapshotTempDirReload() throws IOException {
-    long period = Long.MAX_VALUE;
-    // This doesn't refresh cache until we invoke it explicitly
-    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+    SnapshotFileCache cache =
+        new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    // Add a new non-tmp snapshot
+    createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false);
+    createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false);
+  }
+
+  @Test
+  public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException {
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
         "test-snapshot-file-cache-refresh", new SnapshotFiles());
 
     // Add a new non-tmp snapshot
-    createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
-    createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
+    createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true);
+    createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true);
 
     // Add a new tmp snapshot
-    createAndTestSnapshotV2(cache, "snapshot1", true, false);
+    createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true);
 
     // Add another tmp snapshot
-    createAndTestSnapshotV2(cache, "snapshot2", true, false);
+    createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true);
   }
 
   @Test
@@ -160,7 +167,7 @@ List<String> getSnapshotsInProgress()
     };
 
     SnapshotMock.SnapshotBuilder complete =
-        createAndTestSnapshotV1(cache, "snapshot", false, false);
+        createAndTestSnapshotV1(cache, "snapshot", false, false, false);
 
     int countBeforeCheck = count.get();
 
@@ -214,23 +221,24 @@ public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOEx
   };
 
   private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
-      final String name, final boolean tmp, final boolean removeOnExit) throws IOException {
+      final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime)
+      throws IOException {
     SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
     SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
-    createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+    createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
     return builder;
   }
 
   private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
-      final boolean tmp, final boolean removeOnExit) throws IOException {
+      final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
     SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
     SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
-    createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+    createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
   }
 
   private void createAndTestSnapshot(final SnapshotFileCache cache,
       final SnapshotMock.SnapshotBuilder builder,
-      final boolean tmp, final boolean removeOnExit) throws IOException {
+      final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
     List<Path> files = new ArrayList<>();
     for (int i = 0; i < 3; ++i) {
       for (Path filePath: builder.addRegion()) {
@@ -249,6 +257,10 @@ private void createAndTestSnapshot(final SnapshotFileCache cache,
       builder.commit();
     }
 
+    if (setFolderTime) {
+      fs.setTimes(snapshotDir, 0, -1);
+    }
+
     // Make sure that all files are still present
     for (Path path: files) {
       assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path));