HDDS-1949. Missing or error-prone test cleanup.

Contributed by Doroszlai, Attila.

Author: anuengineer

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

@@ -240,7 +240,7 @@ abstract class Builder {
     protected static final int ACTIVE_OMS_NOT_SET = -1;
 
     protected final OzoneConfiguration conf;
-    protected final String path;
+    protected String path;
 
     protected String clusterId;
     protected String omServiceId;
@@ -269,9 +269,7 @@ abstract class Builder {
 
     protected Builder(OzoneConfiguration conf) {
       this.conf = conf;
-      this.clusterId = UUID.randomUUID().toString();
-      this.path = GenericTestUtils.getTempPath(
-          MiniOzoneClusterImpl.class.getSimpleName() + "-" + clusterId);
+      setClusterId(UUID.randomUUID().toString());
     }
 
     /**
@@ -283,6 +281,8 @@ protected Builder(OzoneConfiguration conf) {
      */
     public Builder setClusterId(String id) {
       clusterId = id;
+      path = GenericTestUtils.getTempPath(
+          MiniOzoneClusterImpl.class.getSimpleName() + "-" + clusterId);
       return this;
     }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -19,6 +19,8 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import org.apache.commons.io.FileUtils;
@@ -317,6 +319,7 @@ public void shutdown() {
       stop();
       FileUtils.deleteDirectory(baseDir);
       ContainerCache.getInstance(conf).shutdownCache();
+      DefaultMetricsSystem.shutdown();
     } catch (IOException e) {
       LOG.error("Exception while shutting down the cluster.", e);
     }
@@ -325,26 +328,9 @@ public void shutdown() {
   @Override
   public void stop() {
     LOG.info("Stopping the Mini Ozone Cluster");
-    if (ozoneManager != null) {
-      LOG.info("Stopping the OzoneManager");
-      ozoneManager.stop();
-      ozoneManager.join();
-    }
-
-    if (!hddsDatanodes.isEmpty()) {
-      LOG.info("Shutting the HddsDatanodes");
-      hddsDatanodes.parallelStream()
-          .forEach(dn -> {
-            dn.stop();
-            dn.join();
-          });
-    }
-
-    if (scm != null) {
-      LOG.info("Stopping the StorageContainerManager");
-      scm.stop();
-      scm.join();
-    }
+    stopOM(ozoneManager);
+    stopDatanodes(hddsDatanodes);
+    stopSCM(scm);
   }
 
   /**
@@ -385,6 +371,37 @@ private void setCAClient(CertificateClient client) {
     this.caClient = client;
   }
 
+  private static void stopDatanodes(
+      Collection<HddsDatanodeService> hddsDatanodes) {
+    if (!hddsDatanodes.isEmpty()) {
+      LOG.info("Stopping the HddsDatanodes");
+      hddsDatanodes.parallelStream()
+          .forEach(MiniOzoneClusterImpl::stopDatanode);
+    }
+  }
+
+  private static void stopDatanode(HddsDatanodeService dn) {
+    if (dn != null) {
+      dn.stop();
+      dn.join();
+    }
+  }
+
+  private static void stopSCM(StorageContainerManager scm) {
+    if (scm != null) {
+      LOG.info("Stopping the StorageContainerManager");
+      scm.stop();
+      scm.join();
+    }
+  }
+
+  private static void stopOM(OzoneManager om) {
+    if (om != null) {
+      LOG.info("Stopping the OzoneManager");
+      om.stop();
+      om.join();
+    }
+  }
 
   /**
    * Builder for configuring the MiniOzoneCluster to run.
@@ -404,28 +421,42 @@ public Builder(OzoneConfiguration conf) {
     public MiniOzoneCluster build() throws IOException {
       DefaultMetricsSystem.setMiniClusterMode(true);
       initializeConfiguration();
-      StorageContainerManager scm;
-      OzoneManager om;
+      StorageContainerManager scm = null;
+      OzoneManager om = null;
+      List<HddsDatanodeService> hddsDatanodes = Collections.emptyList();
       try {
         scm = createSCM();
         scm.start();
         om = createOM();
         if(certClient != null) {
           om.setCertClient(certClient);
         }
-      } catch (AuthenticationException ex) {
-        throw new IOException("Unable to build MiniOzoneCluster. ", ex);
-      }
+        om.start();
+
+        hddsDatanodes = createHddsDatanodes(scm);
+        MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
+            hddsDatanodes);
+        cluster.setCAClient(certClient);
+        if (startDataNodes) {
+          cluster.startHddsDatanodes();
+        }
+        return cluster;
+      } catch (Exception ex) {
+        stopOM(om);
+        if (startDataNodes) {
+          stopDatanodes(hddsDatanodes);
+        }
+        stopSCM(scm);
+        removeConfiguration();
 
-      om.start();
-      final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(scm);
-      MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
-          hddsDatanodes);
-      cluster.setCAClient(certClient);
-      if (startDataNodes) {
-        cluster.startHddsDatanodes();
+        if (ex instanceof IOException) {
+          throw (IOException) ex;
+        }
+        if (ex instanceof RuntimeException) {
+          throw (RuntimeException) ex;
+        }
+        throw new IOException("Unable to build MiniOzoneCluster. ", ex);
       }
-      return cluster;
     }
 
     /**
@@ -466,6 +497,10 @@ void initializeConfiguration() throws IOException {
       configureTrace();
     }
 
+    void removeConfiguration() {
+      FileUtils.deleteQuietly(new File(path));
+    }
+
     /**
      * Creates a new StorageContainerManager instance.
      *

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java

@@ -35,6 +35,7 @@
 import java.util.HashSet;
 import java.util.List;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -53,6 +54,7 @@
 import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
 import org.apache.hadoop.test.PathUtils;
 import org.apache.hadoop.test.TestGenericTestUtils;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -64,7 +66,7 @@
  */
 public class TestMiniOzoneCluster {
 
-  private static MiniOzoneCluster cluster;
+  private MiniOzoneCluster cluster;
   private static OzoneConfiguration conf;
 
   private final static File TEST_ROOT = TestGenericTestUtils.getTestDir();
@@ -78,17 +80,21 @@ public static void setup() {
     conf.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true);
     WRITE_TMP.mkdirs();
     READ_TMP.mkdirs();
-    WRITE_TMP.deleteOnExit();
-    READ_TMP.deleteOnExit();
   }
 
-  @AfterClass
-  public static void cleanup() {
+  @After
+  public void cleanup() {
     if (cluster != null) {
       cluster.shutdown();
     }
   }
 
+  @AfterClass
+  public static void afterClass() {
+    FileUtils.deleteQuietly(WRITE_TMP);
+    FileUtils.deleteQuietly(READ_TMP);
+  }
+
   @Test(timeout = 30000)
   public void testStartMultipleDatanodes() throws Exception {
     final int numberOfNodes = 3;