Skip to content

Commit

Permalink
Parse blob names with hierarchical namespace (microsoft#24903)
Browse files Browse the repository at this point in the history 
Resolves microsoft#24891 

Proposed solution. more tests to come.

---------

Co-authored-by: Jesper Schulz-Wedde <JesperSchulz@users.noreply.github.com>
  • Loading branch information
@adrogin @JesperSchulz
adrogin and JesperSchulz authored Oct 7, 2023
1 parent a10bfe5 commit 18cfab2
Show file tree
Hide file tree
Showing 3 changed files with 222 additions and 9 deletions.
74 changes: 68 additions & 6 deletions Modules/System Tests/Azure Blob Services API/src/ABSBlobClientTest.Codeunit.al
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ codeunit 132920 "ABS Blob Client Test"
Response: Codeunit "ABS Operation Response";
ContainerName, BlobName, BlobContent, NewBlobContent : Text;
begin
// [Scenarion] Given a storage account and a container, PutBlobBlockBlob operation succeeds and GetBlobAsText returns the content
// [Scenario] Given a storage account and a container, PutBlobBlockBlob operation succeeds and GetBlobAsText returns the content

SharedKeyAuthorization := StorageServiceAuthorization.CreateSharedKey(AzuriteTestLibrary.GetAccessKey());

Expand Down Expand Up @@ -53,7 +53,7 @@ codeunit 132920 "ABS Blob Client Test"
Response: Codeunit "ABS Operation Response";
ContainerName, FirstBlobName, SecondBlobName, BlobContent : Text;
begin
// [Scenarion] Given a storage account and a container with BLOBs, ListBlobs operation succeeds.
// [Scenario] Given a storage account and a container with BLOBs, ListBlobs operation succeeds.

SharedKeyAuthorization := StorageServiceAuthorization.CreateSharedKey(AzuriteTestLibrary.GetAccessKey());

Expand Down Expand Up @@ -101,7 +101,7 @@ codeunit 132920 "ABS Blob Client Test"
BlobList: Dictionary of [Text, XmlNode];
Blobs: List of [Text];
begin
// [Scenarion] Given a storage account and a container with BLOBs, ListBlobs operation succeeds.
// [Scenario] Given a storage account and a container with BLOBs, ListBlobs operation succeeds.

SharedKeyAuthorization := StorageServiceAuthorization.CreateSharedKey(AzuriteTestLibrary.GetAccessKey());

Expand Down Expand Up @@ -200,7 +200,7 @@ codeunit 132920 "ABS Blob Client Test"
ABSContainerContent: Record "ABS Container Content";
ContainerName: Text;
begin
// [Scenarion] When listing blobs, the levels, parent directories etc. are set correctly
// [Scenario] When listing blobs, the levels, parent directories etc. are set correctly

SharedKeyAuthorization := StorageServiceAuthorization.CreateSharedKey(AzuriteTestLibrary.GetAccessKey());

Expand Down Expand Up @@ -538,7 +538,7 @@ codeunit 132920 "ABS Blob Client Test"
LeaseId: Guid;
ProposedLeaseId: Guid;
begin
// [Scenarion] Given a storage account and a container, PutBlobBlockBlob operation succeeds and subsequent lease-operations
// [Scenario] Given a storage account and a container, PutBlobBlockBlob operation succeeds and subsequent lease-operations
// (1) create a lease, (2) renew a lease, [(3) change a lease], (4) break a lease and (5) release the lease

SharedKeyAuthorization := StorageServiceAuthorization.CreateSharedKey(AzuriteTestLibrary.GetAccessKey());
Expand Down Expand Up @@ -665,6 +665,68 @@ codeunit 132920 "ABS Blob Client Test"
ABSContainerClient.DeleteContainer(ContainerName);
end;

[Test]
procedure ParseResponseWithHierarchicalBlobName()
var
ABSContainerContent: Record "ABS Container Content";
ABSHelperLibrary: Codeunit "ABS Helper Library";
NodeList: XmlNodeList;
NextMarker: Text;
begin
// [SCENARIO] Parse the BLOB storage API response listing BLOBs in a container without the hierarchical namespace

// [GIVEN] Prepare an XML response from the BLOB Storage API, listing all BLOBs in a container.
// [GIVEN] Container does not have hierarchical namespace and contains a single BLOB with the name like "folder/blob"
NodeList := ABSHelperLibrary.CreateBlobNodeListFromResponse(ABSTestLibrary.GetServiceResponseBlobWithHierarchicalName(), NextMarker);

// [WHEN] Invoke BlobNodeListToTempRecord
ABSHelperLibrary.BlobNodeListToTempRecord(NodeList, ABSContainerContent);

// [THEN] "ABS Container Content" contains one record "folder" and one record "blob"
Assert.RecordCount(ABSContainerContent, 2);

ABSContainerContent.SetRange(Name, ABSTestLibrary.GetSampleResponseRootDirName());
Assert.RecordCount(ABSContainerContent, 1);

ABSContainerContent.SetRange(Name, ABSTestLibrary.GetSampleResponseFileName());
Assert.RecordCount(ABSContainerContent, 1);
end;

[Test]
procedure ParseResponseFromStorageHierarachicalNamespace()
var
ABSContainerContent: Record "ABS Container Content";
ABSHelperLibrary: Codeunit "ABS Helper Library";
NodeList: XmlNodeList;
NextMarker: Text;
begin
// [SCENARIO] Parse the BLOB storage API response listing BLOBs in a container with the hierarchical namespace enabled

// [GIVEN] Prepare an XML response from the BLOB Storage API, listing all BLOBs in a container.
// [GIVEN] Container has the hierarchical namespace enabled, and contains a root folder named "rootdir".
// [GIVEN] There is a subdirectory "subdir" in the root, and one blob named "blob" in the subdirectory.
NodeList := ABSHelperLibrary.CreateBlobNodeListFromResponse(ABSTestLibrary.GetServiceResponseHierarchicalNamespace(), NextMarker);

// [WHEN] Invoke BlobNodeListToTempRecord
ABSHelperLibrary.BlobNodeListToTempRecord(NodeList, ABSContainerContent);

// [THEN] "ABS Container Content" contains two records with resource type "folder" and one record with resource type = "blob"
Assert.RecordCount(ABSContainerContent, 3);

VerifyContainerContentType(ABSContainerContent, CopyStr(ABSTestLibrary.GetSampleResponseRootDirName(), 1, MaxStrLen(ABSContainerContent.Name)), Enum::"ABS Blob Resource Type"::Directory);
VerifyContainerContentType(ABSContainerContent, CopyStr(ABSTestLibrary.GetSampleResponseSubdirName(), 1, MaxStrLen(ABSContainerContent.Name)), Enum::"ABS Blob Resource Type"::Directory);
VerifyContainerContentType(ABSContainerContent, CopyStr(ABSTestLibrary.GetSampleResponseFileName(), 1, MaxStrLen(ABSContainerContent.Name)), Enum::"ABS Blob Resource Type"::File);
end;

local procedure VerifyContainerContentType(var ABSContainerContent: Record "ABS Container Content"; BlobName: Text[2048]; ExpectedResourceType: Enum "ABS Blob Resource Type")
var
IncorrectBlobPropertyErr: Label 'BLOB property is assigned incorrectly';
begin
ABSContainerContent.SetRange(Name, BlobName);
ABSContainerContent.FindFirst();
Assert.AreEqual(ExpectedResourceType, ABSContainerContent."Resource Type", IncorrectBlobPropertyErr);
end;

local procedure GetBlobTagsFromABSContainerBlobList(BlobName: Text; BlobList: Dictionary of [Text, XmlNode]): Dictionary of [Text, Text]
var
BlobNode: XmlNode;
Expand Down Expand Up @@ -715,4 +777,4 @@ codeunit 132920 "ABS Blob Client Test"
ABSTestLibrary: Codeunit "ABS Test Library";
AzuriteTestLibrary: Codeunit "Azurite Test Library";
SharedKeyAuthorization: Interface "Storage Service Authorization";
}
}
131 changes: 131 additions & 0 deletions Modules/System Tests/Azure Blob Services API/src/ABSTestLibrary.Codeunit.al
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,137 @@ codeunit 132921 "ABS Test Library"
exit(Document);
end;

procedure GetServiceResponseBlobWithHierarchicalName(): Text;
var
Builder: TextBuilder;
begin
Builder.Append('<?xml version="1.0" encoding="utf-8"?>');
Builder.Append('<EnumerationResults ContainerName="https://myaccount.blob.core.windows.net/mycontainer">');
Builder.Append('<Blobs>');
Builder.Append('<Blob>');
Builder.Append('<Name>rootdir/filename.txt</Name>');
Builder.Append('<Url>https://myaccount.blob.core.windows.net/mycontainer/rootdir/filename.txt</Url>');
Builder.Append('<Properties>');
Builder.Append('<Last-Modified>Sat, 23 Sep 2023 21:32:55 GMT</Last-Modified>');
Builder.Append('<Etag>0x8DBBC7CA6253661</Etag>');
Builder.Append('<Content-Length>1</Content-Length>');
Builder.Append('<Content-Type>text/plain</Content-Type>');
Builder.Append('<Content-Encoding />');
Builder.Append('<Content-Language />');
Builder.Append('<Content-MD5>dpT0pmMW5TyM3Z2ZVL1hHQ==</Content-MD5>');
Builder.Append('<Cache-Control />');
Builder.Append('<BlobType>BlockBlob</BlobType>');
Builder.Append('<LeaseStatus>unlocked</LeaseStatus>');
Builder.Append('</Properties>');
Builder.Append('</Blob>');
Builder.Append('</Blobs>');
Builder.Append('<NextMarker />');
Builder.Append('</EnumerationResults>');

exit(Builder.ToText());
end;

procedure GetServiceResponseHierarchicalNamespace(): Text;
var
Builder: TextBuilder;
begin
Builder.Append('<?xml version="1.0" encoding="utf-8"?>');
Builder.Append('<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net/" ContainerName="mycontainer">');
Builder.Append('<Blobs>');
Builder.Append('<Blob>');
Builder.Append('<Name>rootdir</Name>');
Builder.Append('<Properties>');
Builder.Append('<Creation-Time>Sat, 23 Sep 2023 21:04:18 GMT</Creation-Time>');
Builder.Append('<Last-Modified>Sat, 23 Sep 2023 21:04:18 GMT</Last-Modified>');
Builder.Append('<Etag>0x8DBBC78A6E95AF7</Etag>');
Builder.Append('<ResourceType>directory</ResourceType>');
Builder.Append('<Content-Length>0</Content-Length>');
Builder.Append('<Content-Type>application/octet-stream</Content-Type>');
Builder.Append('<Content-Encoding />');
Builder.Append('<Content-Language />');
Builder.Append('<Content-CRC64>AAAAAAAAAAA=</Content-CRC64>');
Builder.Append('<Content-MD5 />');
Builder.Append('<Cache-Control />');
Builder.Append('<Content-Disposition />');
Builder.Append('<BlobType>BlockBlob</BlobType>');
Builder.Append('<AccessTier>Hot</AccessTier>');
Builder.Append('<AccessTierInferred>true</AccessTierInferred>');
Builder.Append('<LeaseStatus>unlocked</LeaseStatus>');
Builder.Append('<LeaseState>available</LeaseState>');
Builder.Append('<ServerEncrypted>true</ServerEncrypted>');
Builder.Append('</Properties>');
Builder.Append('<OrMetadata />');
Builder.Append('</Blob>');
Builder.Append('<Blob>');
Builder.Append('<Name>rootdir/subdirectory</Name>');
Builder.Append('<Properties>');
Builder.Append('<Creation-Time>Tue, 26 Sep 2023 21:47:48 GMT</Creation-Time>');
Builder.Append('<Last-Modified>Tue, 26 Sep 2023 21:47:48 GMT</Last-Modified>');
Builder.Append('<Etag>0x8DBBEDA39F9C41D</Etag>');
Builder.Append('<ResourceType>directory</ResourceType>');
Builder.Append('<Content-Length>0</Content-Length>');
Builder.Append('<Content-Type>application/octet-stream</Content-Type>');
Builder.Append('<Content-Encoding />');
Builder.Append('<Content-Language />');
Builder.Append('<Content-CRC64>AAAAAAAAAAA=</Content-CRC64>');
Builder.Append('<Content-MD5 />');
Builder.Append('<Cache-Control />');
Builder.Append('<Content-Disposition />');
Builder.Append('<BlobType>BlockBlob</BlobType>');
Builder.Append('<AccessTier>Hot</AccessTier>');
Builder.Append('<AccessTierInferred>true</AccessTierInferred>');
Builder.Append('<LeaseStatus>unlocked</LeaseStatus>');
Builder.Append('<LeaseState>available</LeaseState>');
Builder.Append('<ServerEncrypted>true</ServerEncrypted>');
Builder.Append('</Properties>');
Builder.Append('<OrMetadata />');
Builder.Append('</Blob>');
Builder.Append('<Blob>');
Builder.Append('<Name>rootdir/subdirectory/filename.txt</Name>');
Builder.Append('<Properties>');
Builder.Append('<Creation-Time>Wed, 27 Sep 2023 20:24:18 GMT</Creation-Time>');
Builder.Append('<Last-Modified>Wed, 27 Sep 2023 20:24:18 GMT</Last-Modified>');
Builder.Append('<Etag>0x8DBBF97BA4A9839</Etag>');
Builder.Append('<ResourceType>file</ResourceType>');
Builder.Append('<Content-Length>1</Content-Length>');
Builder.Append('<Content-Type>text/plain</Content-Type>');
Builder.Append('<Content-Encoding />');
Builder.Append('<Content-Language />');
Builder.Append('<Content-CRC64 />');
Builder.Append('<Content-MD5>dpT0pmMW5TyM3Z2ZVL1hHQ==</Content-MD5>');
Builder.Append('<Cache-Control />');
Builder.Append('<Content-Disposition />');
Builder.Append('<BlobType>BlockBlob</BlobType>');
Builder.Append('<AccessTier>Hot</AccessTier>');
Builder.Append('<AccessTierInferred>true</AccessTierInferred>');
Builder.Append('<LeaseStatus>unlocked</LeaseStatus>');
Builder.Append('<LeaseState>available</LeaseState>');
Builder.Append('<ServerEncrypted>true</ServerEncrypted>');
Builder.Append('</Properties>');
Builder.Append('<OrMetadata />');
Builder.Append('</Blob>');
Builder.Append('</Blobs>');
Builder.Append('<NextMarker />');
Builder.Append('</EnumerationResults>');

exit(Builder.ToText());
end;

procedure GetSampleResponseRootDirName(): Text
begin
exit('rootdir');
end;

procedure GetSampleResponseSubdirName(): Text
begin
exit('subdirectory');
end;

procedure GetSampleResponseFileName(): Text
begin
exit('filename.txt');
end;

local procedure GetNewLineCharacter(): Text
var
LF: Char;
Expand Down
26 changes: 23 additions & 3 deletions Modules/System/Azure Blob Services API/src/Helper/ABSContainerContentHelper.Codeunit.al
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,19 @@ codeunit 9054 "ABS Container Content Helper"
Node.SelectSingleNode('.//Properties', PropertiesNode);
ChildNodes := PropertiesNode.AsXmlElement().GetChildNodes();

AddNewEntry(ABSContainerContent, NameFromXml, OuterXml, ChildNodes, EntryNo);
AddNewEntry(ABSContainerContent, NameFromXml, OuterXml, ChildNodes, EntryNo, GetBlobResourceType(Node));
end;

[NonDebuggable]
procedure AddNewEntry(var ABSContainerContent: Record "ABS Container Content"; NameFromXml: Text; OuterXml: Text; ChildNodes: XmlNodeList; var EntryNo: Integer)
procedure AddNewEntry(
var ABSContainerContent: Record "ABS Container Content"; NameFromXml: Text; OuterXml: Text; ChildNodes: XmlNodeList; var EntryNo: Integer; ResourceType: Enum "ABS Blob Resource Type")
var
OutStream: OutStream;
begin
AddParentEntries(NameFromXml, ABSContainerContent, EntryNo);
if ResourceType = ResourceType::Directory then
ParentEntryFullNameList.Add(NameFromXml)
else
AddParentEntries(NameFromXml, ABSContainerContent, EntryNo);

ABSContainerContent.Init();
ABSContainerContent.Level := GetLevel(NameFromXml);
Expand Down Expand Up @@ -199,6 +203,22 @@ codeunit 9054 "ABS Container Content Helper"
Node := Document.AsXmlNode();
end;

local procedure GetBlobResourceType(BlobXmlNode: XmlNode): Enum "ABS Blob Resource Type"
var
ResourceTypeNode: XmlNode;
begin
if not BlobXmlNode.SelectSingleNode('.//ResourceType', ResourceTypeNode) then
exit(Enum::"ABS Blob Resource Type"::File);

case ResourceTypeNode.AsXmlElement().InnerText().ToLower() of
Format(Enum::"ABS Blob Resource Type"::File).ToLower():
exit(Enum::"ABS Blob Resource Type"::File);

Format(Enum::"ABS Blob Resource Type"::Directory).ToLower():
exit(Enum::"ABS Blob Resource Type"::Directory)
end;
end;

var
ParentEntryFullNameList: List of [Text];
}

0 comments on commit 18cfab2

Please sign in to comment.