comments, remove some unnecessary changes

Author: squito

network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java

@@ -51,8 +51,7 @@ public class ExternalShuffleBlockHandler extends RpcHandler {
   final ExternalShuffleBlockResolver blockManager;
   private final OneForOneStreamManager streamManager;
 
-  public ExternalShuffleBlockHandler(TransportConf conf, File registeredExecutorFile)
-      throws IOException, ClassNotFoundException {
+  public ExternalShuffleBlockHandler(TransportConf conf, File registeredExecutorFile) {
     this(new OneForOneStreamManager(),
       new ExternalShuffleBlockResolver(conf, registeredExecutorFile));
   }
@@ -104,6 +103,16 @@ public void applicationRemoved(String appId, boolean cleanupLocalDirs) {
     blockManager.applicationRemoved(appId, cleanupLocalDirs);
   }
 
+  /**
+   * Register an (application, executor) with the given shuffle info.
+   *
+   * The "re-" is meant to highlight the intended use of this method -- when this service is
+   * restarted, this is used to restore the state of executors from before the restart.  Normal
+   * registration will happen via a message handled in receive()
+   *
+   * @param appExecId
+   * @param executorInfo
+   */
   public void reregisterExecutor(AppExecId appExecId, ExecutorShuffleInfo executorInfo) {
     blockManager.registerExecutor(appExecId.appId, appExecId.execId, executorInfo);
   }

network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java

@@ -61,8 +61,7 @@ public class ExternalShuffleBlockResolver {
   @VisibleForTesting
   final File registeredExecutorFile;
 
-  public ExternalShuffleBlockResolver(TransportConf conf, File registeredExecutorFile)
-      throws IOException, ClassNotFoundException {
+  public ExternalShuffleBlockResolver(TransportConf conf, File registeredExecutorFile) {
     this(conf, registeredExecutorFile, Executors.newSingleThreadExecutor(
         // Add `spark` prefix because it will run in NM in Yarn mode.
         NettyUtils.createThreadFactory("spark-shuffle-directory-cleaner")));
@@ -73,7 +72,7 @@ public ExternalShuffleBlockResolver(TransportConf conf, File registeredExecutorF
   ExternalShuffleBlockResolver(
       TransportConf conf,
       File registeredExecutorFile,
-      Executor directoryCleaner) throws IOException, ClassNotFoundException {
+      Executor directoryCleaner) {
     this.conf = conf;
     this.registeredExecutorFile = registeredExecutorFile;
     this.executors = Maps.newConcurrentMap();

network/shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java

@@ -59,7 +59,7 @@ public static void afterAll() {
   }
 
   @Test
-  public void testBadRequests() throws IOException, ClassNotFoundException {
+  public void testBadRequests() {
     ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null);
     // Unregistered executor
     try {
@@ -90,7 +90,7 @@ public void testBadRequests() throws IOException, ClassNotFoundException {
   }
 
   @Test
-  public void testSortShuffleBlocks() throws IOException, ClassNotFoundException {
+  public void testSortShuffleBlocks() throws IOException {
     ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null);
     resolver.registerExecutor("app0", "exec0",
       dataContext.createExecutorInfo("org.apache.spark.shuffle.sort.SortShuffleManager"));
@@ -109,7 +109,7 @@ public void testSortShuffleBlocks() throws IOException, ClassNotFoundException {
   }
 
   @Test
-  public void testHashShuffleBlocks() throws IOException, ClassNotFoundException {
+  public void testHashShuffleBlocks() throws IOException {
     ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null);
     resolver.registerExecutor("app0", "exec0",
       dataContext.createExecutorInfo("org.apache.spark.shuffle.hash.HashShuffleManager"));

network/shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleCleanupSuite.java

@@ -38,7 +38,7 @@ public class ExternalShuffleCleanupSuite {
   TransportConf conf = new TransportConf(new SystemPropertyConfigProvider());
 
   @Test
-  public void noCleanupAndCleanup() throws IOException, ClassNotFoundException {
+  public void noCleanupAndCleanup() throws IOException {
     TestShuffleDataContext dataContext = createSomeData();
 
     ExternalShuffleBlockResolver resolver =
@@ -55,7 +55,7 @@ public void noCleanupAndCleanup() throws IOException, ClassNotFoundException {
   }
 
   @Test
-  public void cleanupUsesExecutor() throws IOException, ClassNotFoundException {
+  public void cleanupUsesExecutor() throws IOException {
     TestShuffleDataContext dataContext = createSomeData();
 
     final AtomicBoolean cleanupCalled = new AtomicBoolean(false);
@@ -79,7 +79,7 @@ public void cleanupUsesExecutor() throws IOException, ClassNotFoundException {
   }
 
   @Test
-  public void cleanupMultipleExecutors() throws IOException, ClassNotFoundException {
+  public void cleanupMultipleExecutors() throws IOException {
     TestShuffleDataContext dataContext0 = createSomeData();
     TestShuffleDataContext dataContext1 = createSomeData();
 
@@ -95,7 +95,7 @@ public void cleanupMultipleExecutors() throws IOException, ClassNotFoundExceptio
   }
 
   @Test
-  public void cleanupOnlyRemovedApp() throws IOException, ClassNotFoundException {
+  public void cleanupOnlyRemovedApp() throws IOException {
     TestShuffleDataContext dataContext0 = createSomeData();
     TestShuffleDataContext dataContext1 = createSomeData();
 

network/shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java

@@ -73,7 +73,7 @@ public class ExternalShuffleIntegrationSuite {
   };
 
   @BeforeClass
-  public static void beforeAll() throws IOException, ClassNotFoundException {
+  public static void beforeAll() throws IOException {
     Random rand = new Random();
 
     for (byte[] block : exec0Blocks) {

network/shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleSecuritySuite.java

@@ -43,7 +43,7 @@ public class ExternalShuffleSecuritySuite {
   TransportServer server;
 
   @Before
-  public void beforeEach() throws IOException, ClassNotFoundException {
+  public void beforeEach() {
     TransportContext context =
       new TransportContext(conf, new ExternalShuffleBlockHandler(conf, null));
     TransportServerBootstrap bootstrap = new SaslServerBootstrap(conf,

yarn/src/test/resources/log4j.properties

@@ -25,4 +25,4 @@ log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{
 
 # Ignore messages below warning level from Jetty, because it's a bit verbose
 log4j.logger.org.spark-project.jetty=WARN
-log4j.logger.org.apache.hadoop=INFO
+log4j.logger.org.apache.hadoop=WARN