YARN-9524. Fixed TestAHSWebService and TestLogsCLI unit tests.

            Contributed by Prabhu Joseph

Author: macroadster

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java

@@ -407,7 +407,7 @@ public void testFetchFinishedApplictionLogs() throws Exception {
 
     Path path =
         new Path(remoteLogRootDir + ugi.getShortUserName()
-            + "/logs/application_0_0001");
+        + "/bucket_logs/0001/application_0_0001");
     if (fs.exists(path)) {
       fs.delete(path, true);
     }
@@ -925,7 +925,6 @@ public void testFetchRunningApplicationLogs() throws Exception {
   public void testFetchApplictionLogsAsAnotherUser() throws Exception {
     String remoteLogRootDir = "target/logs/";
     String rootLogDir = "target/LocalLogs";
-
     String testUser = "test";
     UserGroupInformation testUgi = UserGroupInformation
         .createRemoteUser(testUser);
@@ -966,9 +965,9 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
       // create container logs in localLogDir for app
       createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
 
-      // create the remote app dir for app
-      // but for a different user testUser"
-      Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
+      // create the remote app dir for app but for a different user testUser
+      Path path = new Path(remoteLogRootDir + testUser + "/bucket_logs/0001/"
+          + appId);
       if (fs.exists(path)) {
         fs.delete(path, true);
       }
@@ -1016,6 +1015,22 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
           logMessage(containerId, "syslog")));
       sysOutStream.reset();
 
+      // Verify appOwner guessed correctly with older log dir dtructure
+      path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
+      if (fs.exists(path)) {
+        fs.delete(path, true);
+      }
+      assertTrue(fs.mkdirs(path));
+      uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
+          nodeId, containerId, path, fs);
+
+      exitCode = cli.run(new String[] {
+          "-applicationId", appId.toString()});
+      assertTrue(exitCode == 0);
+      assertTrue(sysOutStream.toString().contains(
+          logMessage(containerId, "syslog")));
+      sysOutStream.reset();
+
       // Verify that we could get the err message "Can not find the appOwner"
       // if we do not specify the appOwner, can not get appReport, and
       // the app does not exist in remote dir.
@@ -1034,7 +1049,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
           System.currentTimeMillis(), 1000);
       String priorityUser = "priority";
       Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
-          + "/logs/" + appTest);
+          + "/bucket_logs/1000/" + appTest);
       if (fs.exists(pathWithoutPerm)) {
         fs.delete(pathWithoutPerm, true);
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java

@@ -83,6 +83,30 @@ public static Path getOlderRemoteAppLogDir(ApplicationId appId,
          suffix), appId.toString());
   }
 
+  public static Path getOlderRemoteAppLogDir(Configuration conf,
+      ApplicationId appId, String user, Path remoteRootLogDir, String suffix)
+      throws IOException {
+    org.apache.hadoop.fs.Path remoteAppDir = null;
+    if (user == null) {
+      org.apache.hadoop.fs.Path qualifiedRemoteRootLogDir =
+          FileContext.getFileContext(conf).makeQualified(remoteRootLogDir);
+      FileContext fc = FileContext.getFileContext(
+          qualifiedRemoteRootLogDir.toUri(), conf);
+      org.apache.hadoop.fs.Path toMatch = LogAggregationUtils
+          .getOlderRemoteAppLogDir(appId, "*", remoteRootLogDir, suffix);
+      FileStatus[] matching  = fc.util().globStatus(toMatch);
+      if (matching == null || matching.length != 1) {
+        throw new IOException("Can not find remote application directory for "
+            + "the application:" + appId);
+      }
+      remoteAppDir = matching[0].getPath();
+    } else {
+      remoteAppDir = LogAggregationUtils.getOlderRemoteAppLogDir(
+          appId, user, remoteRootLogDir, suffix);
+    }
+    return remoteAppDir;
+  }
+
   /**
    * Gets the remote suffixed log dir for the user.
    * @param remoteRootLogDir the aggregated log remote root log dir
@@ -274,19 +298,19 @@ public static RemoteIterator<FileStatus> getRemoteNodeFileDir(
     StringBuilder diagnosticsMsg = new StringBuilder();
 
     // Get Node Files from new app log dir
-    Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
-        remoteRootLogDir, suffix);
     try {
+      Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
+          remoteRootLogDir, suffix);
       nodeFilesCur = getNodeFiles(conf, remoteAppLogDir, appId, appOwner);
     } catch (IOException ex) {
       diagnosticsMsg.append(ex.getMessage() + "\n");
     }
 
     // Get Node Files from old app log dir
     if (isOlderPathEnabled(conf)) {
-      remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner,
-              remoteRootLogDir, suffix);
       try {
+        Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner,
+            remoteRootLogDir, suffix);
         nodeFilesPrev = getNodeFiles(conf,
                 remoteAppLogDir, appId, appOwner);
       } catch (IOException ex) {
@@ -338,14 +362,14 @@ public static List<FileStatus> getRemoteNodeFileList(
       org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
       throws IOException {
     StringBuilder diagnosticsMsg = new StringBuilder();
-    Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
-        remoteRootLogDir, suffix);
     List<FileStatus> nodeFiles = new ArrayList<>();
-    Path qualifiedLogDir =
-        FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
 
     // Get Node Files from new app log dir
     try {
+      Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
+          remoteRootLogDir, suffix);
+      Path qualifiedLogDir =
+          FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
       nodeFiles.addAll(Arrays.asList(FileContext.getFileContext(
           qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir)));
     } catch (IOException ex) {
@@ -354,11 +378,11 @@ public static List<FileStatus> getRemoteNodeFileList(
 
     // Get Node Files from old app log dir
     if (isOlderPathEnabled(conf)) {
-      remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner,
-          remoteRootLogDir, suffix);
-      qualifiedLogDir = FileContext.getFileContext(conf).
-          makeQualified(remoteAppLogDir);
       try {
+        Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner,
+            remoteRootLogDir, suffix);
+        Path qualifiedLogDir = FileContext.getFileContext(conf).
+            makeQualified(remoteAppLogDir);
         nodeFiles.addAll(Arrays.asList(FileContext.getFileContext(
             qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir)));
       } catch (IOException ex) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java

@@ -88,26 +88,58 @@ public static String getOwnerForAppIdOrNull(
     String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
     Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
         appId, bestGuess, suffix);
+
     FileContext fc =
         FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
     String pathAccess = fullPath.toString();
     try {
       if (fc.util().exists(fullPath)) {
         return bestGuess;
       }
+
+      boolean scanOldPath = LogAggregationUtils.isOlderPathEnabled(conf);
+      if (scanOldPath) {
+        Path olderAppPath = LogAggregationUtils.getOlderRemoteAppLogDir(appId,
+            bestGuess, remoteRootLogDir, suffix);
+        if (fc.util().exists(olderAppPath)) {
+          return bestGuess;
+        }
+      }
+
       Path toMatch = LogAggregationUtils.
           getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
+
       pathAccess = toMatch.toString();
       FileStatus[] matching  = fc.util().globStatus(toMatch);
       if (matching == null || matching.length != 1) {
+        if (scanOldPath) {
+          toMatch = LogAggregationUtils.getOlderRemoteAppLogDir(appId, "*",
+              remoteRootLogDir, suffix);
+          try {
+            matching = fc.util().globStatus(toMatch);
+            if (matching != null && matching.length == 1) {
+              //fetch the user from the old path /app-logs/user[/suffix]/app_id
+              Path parent = matching[0].getPath().getParent();
+              //skip the suffix too
+              if (suffix != null && !StringUtils.isEmpty(suffix)) {
+                parent = parent.getParent();
+              }
+              return parent.getName();
+            }
+          } catch (IOException e) {
+            // Ignore IOException from accessing older app log dir
+          }
+        }
         return null;
       }
-      //fetch the user from the full path /app-logs/user[/suffix]/app_id
+      //fetch the user from the full path /app-logs/user[/suffix]/bucket/app_id
       Path parent = matching[0].getPath().getParent();
       //skip the suffix too
       if (suffix != null && !StringUtils.isEmpty(suffix)) {
         parent = parent.getParent();
       }
+      //skip the bucket
+      parent = parent.getParent();
       return parent.getName();
     } catch (AccessControlException | AccessDeniedException ex) {
       logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java

@@ -470,7 +470,7 @@ public Path getRemoteAppLogDir(ApplicationId appId, String appOwner)
    */
   public Path getOlderRemoteAppLogDir(ApplicationId appId, String appOwner)
       throws IOException {
-    return LogAggregationUtils.getOlderRemoteAppLogDir(appId, appOwner,
+    return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, appOwner,
         this.remoteRootLogDir, this.remoteRootLogDirSuffix);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java

@@ -822,7 +822,7 @@ public Path getRemoteAppLogDir(ApplicationId appId, String user)
   @Override
   public Path getOlderRemoteAppLogDir(ApplicationId appId, String user)
       throws IOException {
-    return LogAggregationUtils.getOlderRemoteAppLogDir(appId, user,
+    return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, user,
         this.remoteRootLogDir, this.remoteRootLogDirSuffix);
   }