diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 59fac03057a28..0d69032976509 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1919,10 +1919,14 @@ public String listStatus(final Path path, final String startFrom, if (path.isRoot()) { prefix = null; } + + TreeMap fileMetadata = new TreeMap<>(); do { - // List Blob calls will be made with delimiter "/". This will ensure - // that all the children of a folder not listed out separately. Instead, - // a single entry corresponding to the directory name will be returned as BlobPrefix. + /* + * List Blob calls will be made with delimiter "/". This will ensure + * that all the children of a folder not listed out separately. Instead, + * a single entry corresponding to the directory name will be returned as BlobPrefix. + */ try (AbfsPerfInfo perfInfo = startTracking("listStatus", "getListBlobs")) { AbfsRestOperation op = client.getListBlobs( continuation, prefix, delimiter, abfsConfiguration.getListMaxResults(), @@ -1932,7 +1936,7 @@ public String listStatus(final Path path, final String startFrom, BlobList blobList = op.getResult().getBlobList(); continuation = blobList.getNextMarker(); - addBlobListAsFileStatus(blobList, fileStatuses); + addBlobListAsFileStatus(blobList, fileMetadata); perfInfo.registerSuccess(true); countAggregate++; @@ -1945,6 +1949,7 @@ public String listStatus(final Path path, final String startFrom, } } while (shouldContinue); + fileStatuses.addAll(fileMetadata.values()); return continuation; } @@ -2015,21 +2020,22 @@ public String listStatus(final Path path, final String startFrom, } private void addBlobListAsFileStatus(final BlobList blobList, - List fileStatuses) throws IOException { - - // Here before adding the data we might have to remove the duplicates. - // List Blobs call over blob endpoint returns two types of entries: Blob - // and BlobPrefix. In the case where ABFS generated the data, - // there will be a marker blob for each "directory" created by driver, - // and we will receive them as a Blob. If there are also files within this - // "directory", we will also receive a BlobPrefix. To further - // complicate matters, the data may not be generated by ABFS Driver, in - // which case we may not have a marker blob for each "directory". In this - // the only way to know there is a directory is using BlobPrefix entry. - // So, sometimes we receive both a Blob and a BlobPrefix for directories, - // and sometimes we receive only BlobPrefix as directory. We remove duplicates - // but prefer Blob over BlobPrefix. - TreeMap fileMetadata = new TreeMap<>(); + TreeMap fileMetadata) throws IOException { + + /* + * Here before adding the data we might have to remove the duplicates. + * List Blobs call over blob endpoint returns two types of entries: Blob + * and BlobPrefix. In the case where ABFS generated the data, + * there will be a marker blob for each "directory" created by driver, + * and we will receive them as a Blob. If there are also files within this + * "directory", we will also receive a BlobPrefix. To further + * complicate matters, the data may not be generated by ABFS Driver, in + * which case we may not have a marker blob for each "directory". In this + * the only way to know there is a directory is using BlobPrefix entry. + * So, sometimes we receive both a Blob and a BlobPrefix for directories, + * and sometimes we receive only BlobPrefix as directory. We remove duplicates + * but prefer Blob over BlobPrefix. + */ List blobProperties = blobList.getBlobPropertyList(); for (BlobProperty entry: blobProperties) { @@ -2064,8 +2070,7 @@ private void addBlobListAsFileStatus(final BlobList blobList, // This is a blob entry. It is either a file or a marker blob. // In both cases we will add this. fileMetadata.put(blobKey, fileStatus); - } - else { + } else { // This is a BlobPrefix entry. It is a directory with file inside // This might have already been added as a marker blob. if (!fileMetadata.containsKey(blobKey)) { @@ -2073,7 +2078,6 @@ private void addBlobListAsFileStatus(final BlobList blobList, } } } - fileStatuses.addAll(fileMetadata.values()); } // generate continuation token for xns account