Skip to content

Commit

Permalink
Addressed Comments
Browse files Browse the repository at this point in the history
  • Loading branch information
anujmodi2021 committed Jun 13, 2023
1 parent ba21593 commit 73d05ca
Showing 1 changed file with 26 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1919,10 +1919,14 @@ public String listStatus(final Path path, final String startFrom,
if (path.isRoot()) {
prefix = null;
}

TreeMap<String, FileStatus> fileMetadata = new TreeMap<>();
do {
// List Blob calls will be made with delimiter "/". This will ensure
// that all the children of a folder not listed out separately. Instead,
// a single entry corresponding to the directory name will be returned as BlobPrefix.
/*
* List Blob calls will be made with delimiter "/". This will ensure
* that all the children of a folder not listed out separately. Instead,
* a single entry corresponding to the directory name will be returned as BlobPrefix.
*/
try (AbfsPerfInfo perfInfo = startTracking("listStatus", "getListBlobs")) {
AbfsRestOperation op = client.getListBlobs(
continuation, prefix, delimiter, abfsConfiguration.getListMaxResults(),
Expand All @@ -1932,7 +1936,7 @@ public String listStatus(final Path path, final String startFrom,
BlobList blobList = op.getResult().getBlobList();
continuation = blobList.getNextMarker();

addBlobListAsFileStatus(blobList, fileStatuses);
addBlobListAsFileStatus(blobList, fileMetadata);

perfInfo.registerSuccess(true);
countAggregate++;
Expand All @@ -1945,6 +1949,7 @@ public String listStatus(final Path path, final String startFrom,
}
} while (shouldContinue);

fileStatuses.addAll(fileMetadata.values());
return continuation;
}

Expand Down Expand Up @@ -2015,21 +2020,22 @@ public String listStatus(final Path path, final String startFrom,
}

private void addBlobListAsFileStatus(final BlobList blobList,
List<FileStatus> fileStatuses) throws IOException {

// Here before adding the data we might have to remove the duplicates.
// List Blobs call over blob endpoint returns two types of entries: Blob
// and BlobPrefix. In the case where ABFS generated the data,
// there will be a marker blob for each "directory" created by driver,
// and we will receive them as a Blob. If there are also files within this
// "directory", we will also receive a BlobPrefix. To further
// complicate matters, the data may not be generated by ABFS Driver, in
// which case we may not have a marker blob for each "directory". In this
// the only way to know there is a directory is using BlobPrefix entry.
// So, sometimes we receive both a Blob and a BlobPrefix for directories,
// and sometimes we receive only BlobPrefix as directory. We remove duplicates
// but prefer Blob over BlobPrefix.
TreeMap<String, FileStatus> fileMetadata = new TreeMap<>();
TreeMap<String, FileStatus> fileMetadata) throws IOException {

/*
* Here before adding the data we might have to remove the duplicates.
* List Blobs call over blob endpoint returns two types of entries: Blob
* and BlobPrefix. In the case where ABFS generated the data,
* there will be a marker blob for each "directory" created by driver,
* and we will receive them as a Blob. If there are also files within this
* "directory", we will also receive a BlobPrefix. To further
* complicate matters, the data may not be generated by ABFS Driver, in
* which case we may not have a marker blob for each "directory". In this
* the only way to know there is a directory is using BlobPrefix entry.
* So, sometimes we receive both a Blob and a BlobPrefix for directories,
* and sometimes we receive only BlobPrefix as directory. We remove duplicates
* but prefer Blob over BlobPrefix.
*/
List<BlobProperty> blobProperties = blobList.getBlobPropertyList();

for (BlobProperty entry: blobProperties) {
Expand Down Expand Up @@ -2064,16 +2070,14 @@ private void addBlobListAsFileStatus(final BlobList blobList,
// This is a blob entry. It is either a file or a marker blob.
// In both cases we will add this.
fileMetadata.put(blobKey, fileStatus);
}
else {
} else {
// This is a BlobPrefix entry. It is a directory with file inside
// This might have already been added as a marker blob.
if (!fileMetadata.containsKey(blobKey)) {
fileMetadata.put(blobKey, fileStatus);
}
}
}
fileStatuses.addAll(fileMetadata.values());
}

// generate continuation token for xns account
Expand Down

0 comments on commit 73d05ca

Please sign in to comment.