Create the blob container if it doesn't exist when uploading (#267)

Author: hydraxman

.gitignore

@@ -64,4 +64,5 @@ application-local.yml
 /application.yml
 /application-*.yml
 
-/local_debug
+/local_debug
+.env

azure-pipelines-ci.yml

@@ -36,6 +36,13 @@ stages:
         command: 'custom'
         workingDir: 'react'
         customCommand: 'run pub'
+    - task: PowerShell@2
+      displayName: Generate Env File
+      inputs:
+        targetType: 'inline'
+        script: |
+          New-Item -Path common/src/test/resources -Name ".env" -ItemType "file" -Value "BLOB_CONNECTION_STRING = $(BLOB_CONNECTION_STRING)"
+        workingDirectory: '$(Build.Repository.LocalPath)'
     - task: PowerShell@2
       displayName: Set center/agent version
       inputs:
@@ -65,6 +72,13 @@ stages:
         jdkVersionOption: '1.11'
         sonarQubeRunAnalysis: false
         spotBugsAnalysis: false
+    - task: PowerShell@2
+      displayName: Delete Env File
+      inputs:
+        targetType: 'inline'
+        script: |
+          Remove-Item -Path common/src/test/resources/.env -Force
+        workingDirectory: '$(Build.Repository.LocalPath)'
     - task: PublishCodeCoverageResults@1
       displayName: Publich Code Coverage
       inputs:

common/build.gradle

@@ -23,6 +23,7 @@ dependencies {
     compile 'com.android.tools.ddms:ddmlib:27.0.2'
     compile group: 'junit', name: 'junit', version: '4.12'
     testCompile 'com.github.stefanbirkner:system-rules:1.19.0'
+    testCompile 'io.github.cdimascio:java-dotenv:5.1.3'
 
     compile project(":sdk")
     compile project(":taps_to_cases:runner")

common/src/main/java/com/microsoft/hydralab/common/util/blob/BlobStorageClient.java

@@ -18,7 +18,6 @@
 import com.azure.storage.common.sas.AccountSasSignatureValues;
 import com.google.common.net.MediaType;
 import com.microsoft.hydralab.common.entity.center.BlobProperty;
-import com.microsoft.hydralab.common.entity.common.EntityFileRelation.EntityType;
 import com.microsoft.hydralab.common.entity.common.SASData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -52,7 +51,6 @@ public BlobStorageClient(BlobProperty blobProperty) {
         blobServiceClient = new BlobServiceClientBuilder().connectionString(blobProperty.getConnection()).buildClient();
         fileLimitDay = blobProperty.getFileLimitDay();
         cdnUrl = blobProperty.getCDNUrl();
-        initContainer();
         isAuthedBySAS = false;
         isConnected = true;
     }
@@ -70,7 +68,6 @@ private void buildClientBySAS(SASData sasData) {
         blobServiceClient = new BlobServiceClientBuilder().endpoint(sasData.getEndpoint()).credential(azureSasCredential).buildClient();
         fileLimitDay = sasData.getFileLimitDay();
         cdnUrl = sasData.getCdnUrl();
-        initContainer();
         isConnected = true;
         sasDataInUse = sasData;
     }
@@ -82,18 +79,20 @@ private void checkBlobStorageClientUpdate() {
         }
     }
 
-    private void initContainer() {
-        EntityType[] entityTypes = EntityType.values();
-        for (EntityType entityType : entityTypes) {
-            String containerName = entityType.blobConstant;
-            try {
-                blobServiceClient.getBlobContainerClient(containerName);
-                classLogger.info("Get a BlobContainerClient for container {}", containerName);
-            } catch (BlobStorageException e) {
-                classLogger.info("Can't connect to container for {}. Try to create one!", containerName);
-                blobServiceClient.createBlobContainerWithResponse(containerName, null, PublicAccessType.CONTAINER, Context.NONE);
+    private BlobContainerClient getContainer(String containerName) {
+        BlobContainerClient blobContainerClient;
+        try {
+            blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
+            classLogger.info("Get a BlobContainerClient for container {}", containerName);
+            if (!blobContainerClient.exists()) {
+                classLogger.info("Container {} doesn't exist, will try to create it.", containerName);
+                blobContainerClient.create();
             }
+        } catch (BlobStorageException e) {
+            classLogger.info("Can't connect to container for {}. Try to create one!", containerName);
+            blobContainerClient = blobServiceClient.createBlobContainerWithResponse(containerName, null, PublicAccessType.CONTAINER, Context.NONE).getValue();
         }
+        return blobContainerClient;
     }
 
     public SASData generateSAS(SASData.SASPermission sasPermission) {
@@ -139,10 +138,8 @@ public String uploadBlobFromFile(File uploadFile, String containerName, String b
         if (logger == null) {
             logger = classLogger;
         }
-        BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
-        logger.info("Get a BlobContainerClient for container {} for file {}", containerName, uploadFile.getAbsoluteFile());
 
-        BlobClient blobClient = blobContainerClient.getBlobClient(blobFilePath);
+        BlobClient blobClient = getContainer(containerName).getBlobClient(blobFilePath);
         if (uploadFile.getName().endsWith(MediaType.MP4_VIDEO.subtype())) {
             BlobHttpHeaders headers = new BlobHttpHeaders();
             headers.setContentType(MediaType.MP4_VIDEO.toString());
@@ -172,8 +169,7 @@ public BlobProperties downloadFileFromBlob(File downloadToFile, String container
         if (!saveDir.exists()) {
             cn.hutool.core.lang.Assert.isTrue(saveDir.mkdirs(), "mkdirs fail in downloadFileFromUrl");
         }
-        BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
-        BlobClient blobClient = blobContainerClient.getBlobClient(blobFilePath);
+        BlobClient blobClient = getContainer(containerName).getBlobClient(blobFilePath);
         return blobClient.downloadToFile(downloadToFile.getAbsolutePath(), true);
     }
 }

common/src/test/java/com/microsoft/hydralab/common/util/blob/BlobStorageClientTest.java

@@ -5,6 +5,7 @@
 import com.microsoft.hydralab.common.entity.common.SASData;
 import com.microsoft.hydralab.common.test.BaseTest;
 import com.microsoft.hydralab.common.util.ThreadUtils;
+import io.github.cdimascio.dotenv.Dotenv;
 import org.junit.jupiter.api.*;
 import org.junit.platform.commons.util.StringUtils;
 
@@ -13,13 +14,21 @@
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class BlobStorageClientTest extends BaseTest {
-    String connectionString = "";
     BlobStorageClient blobStorageClient;
     File sampleFile = new File("src/test/resources/uitestsample.ipa");
     BlobProperty property = new BlobProperty();
 
     @BeforeAll
     void initBlob() {
+        String connectionString = null;
+        try {
+            Dotenv dotenv = Dotenv.load();
+            connectionString = dotenv.get("BLOB_CONNECTION_STRING");
+            logger.info("Get connectionString from env file successfully!");
+        } catch (Exception e) {
+            logger.error("Get connectionString from env file failed!", e);
+        }
+
         property.setConnection(connectionString);
         property.setFileLimitDay(6);
         property.setSASExpiryTimeAgent(30);
@@ -50,9 +59,8 @@ void downloadFileFromBlob() {
             BlobProperties properties = blobStorageClient.downloadFileFromBlob(sampleFile_copy, DeviceNetworkBlobConstants.PKG_BLOB_NAME, "test/unit/" + sampleFile.getName());
             logger.info("Download sample file finished, properties: " + properties);
             Assertions.assertNotNull(properties, "Download File Failed!");
-            if (sampleFile_copy.exists()) {
-                sampleFile_copy.delete();
-            }
+            Assertions.assertTrue(sampleFile_copy.exists(), "Download File Failed!");
+            sampleFile_copy.delete();
         }
     }