Blobs Sync Stack Migration (#40812)

eng/versioning/version_client.txt

@@ -490,6 +490,9 @@ unreleased_com.azure:azure-json;1.3.0-beta.1
 
 unreleased_com.azure:azure-core-amqp;2.10.0-beta.1
 
+unreleased_com.azure:azure-core;1.52.0-beta.1
+unreleased_com.azure:azure-core-http-okhttp;1.13.0-beta.1
+
 # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current
 # version and set the version to the released beta. Released beta dependencies are only valid
 # for dependency versions. These entries are specifically for when we've released a beta for

sdk/storage/azure-storage-blob/checkstyle-suppressions.xml

@@ -35,7 +35,14 @@
   <suppress files="com.azure.storage.blob.implementation.util.ModelHelper.java" checks="com.azure.tools.checkstyle.checks.JavadocThrowsChecks" />
   <suppress files="com.azure.storage.blob.implementation.AzureBlobStorageImplBuilder.java" checks="com.azure.tools.checkstyle.checks.ServiceClientBuilderCheck" />
   <suppress files="com.azure.storage.blob.BlobClient.java" checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" />
+  <suppress files="com.azure.storage.blob.specialized.BlobLeaseClient.java" checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" />
   <suppress files="com.azure.storage.blob.specialized.BlobLeaseAsyncClient.java" checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" />
   <suppress files="com.azure.storage.blob.specialized.BlobInputStream.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
   <suppress files="com.azure.storage.blob.specialized.BlobOutputStream.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
+  <suppress files="com.azure.storage.blob.implementation.AppendBlobsImpl.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
+  <suppress files="com.azure.storage.blob.implementation.PageBlobsImpl.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
+  <suppress files="com.azure.storage.blob.implementation.ServicesImpl.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
+  <suppress files="com.azure.storage.blob.implementation.BlobsImpl.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
+  <suppress files="com.azure.storage.blob.implementation.BlockBlobsImpl.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
+  <suppress files="com.azure.storage.blob.implementation.ContainersImpl.java" checks="com.azure.tools.checkstyle.checks.ThrowFromClientLoggerCheck" />
 </suppressions>

sdk/storage/azure-storage-blob/pom.xml

@@ -70,7 +70,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core</artifactId>
-      <version>1.51.0</version> <!-- {x-version-update;com.azure:azure-core;dependency} -->
+      <version>1.52.0-beta.1</version><!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
@@ -182,7 +182,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-http-okhttp</artifactId>
-      <version>1.12.2</version> <!-- {x-version-update;com.azure:azure-core-http-okhttp;dependency} -->
+      <version>1.13.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-http-okhttp;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/storage/azure-storage-blob/spotbugs-exclude.xml

@@ -8,6 +8,7 @@
       <Class name="com.azure.storage.blob.implementation.util.BlobSasImplUtil" />
       <Class name="com.azure.storage.blob.models.CustomerProvidedKey" />
       <Class name="com.azure.storage.blob.specialized.BlobAsyncClientBase" />
+      <Class name="com.azure.storage.blob.specialized.BlobClientBase" />
     </Or>
   </Match>
   <Match>
@@ -185,4 +186,8 @@
       <Class name="com.azure.storage.blob.sas.BlobServiceSasSignatureValues" />
     </Or>
   </Match>
+  <Match>
+    <Bug pattern="MC_OVERRIDABLE_METHOD_CALL_IN_CONSTRUCTOR" />
+    <Class name="com.azure.storage.blob.specialized.BlobClientBase" />
+  </Match>
 </FindBugsFilter>

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobClient.java

@@ -6,13 +6,16 @@
 import com.azure.core.annotation.ReturnType;
 import com.azure.core.annotation.ServiceClient;
 import com.azure.core.annotation.ServiceMethod;
+import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.rest.Response;
 import com.azure.core.util.BinaryData;
 import com.azure.core.util.Context;
 import com.azure.core.util.FluxUtil;
 import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.implementation.models.EncryptionScope;
 import com.azure.storage.blob.implementation.util.ModelHelper;
 import com.azure.storage.blob.models.AccessTier;
+import com.azure.storage.blob.models.CpkInfo;
 import com.azure.storage.blob.models.CustomerProvidedKey;
 import com.azure.storage.blob.options.BlobParallelUploadOptions;
 import com.azure.storage.blob.models.BlobRequestConditions;
@@ -32,6 +35,8 @@
 
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.file.Path;
 import java.time.Duration;
 import java.util.Map;
 import java.util.Objects;
@@ -61,19 +66,26 @@ public class BlobClient extends BlobClientBase {
     /**
      * The block size to use if none is specified in parallel operations.
      */
-    public static final int BLOB_DEFAULT_UPLOAD_BLOCK_SIZE = BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE;
+    public static final int BLOB_DEFAULT_UPLOAD_BLOCK_SIZE = 4 * Constants.MB;
 
     /**
      * The number of buffers to use if none is specied on the buffered upload method.
      */
-    public static final int BLOB_DEFAULT_NUMBER_OF_BUFFERS = BlobAsyncClient.BLOB_DEFAULT_NUMBER_OF_BUFFERS;
+    public static final int BLOB_DEFAULT_NUMBER_OF_BUFFERS = 8;
     /**
      * If a blob  is known to be greater than 100MB, using a larger block size will trigger some server-side
      * optimizations. If the block size is not set and the size of the blob is known to be greater than 100MB, this
      * value will be used.
      */
-    public static final int BLOB_DEFAULT_HTBB_UPLOAD_BLOCK_SIZE = BlobAsyncClient.BLOB_DEFAULT_HTBB_UPLOAD_BLOCK_SIZE;
+    public static final int BLOB_DEFAULT_HTBB_UPLOAD_BLOCK_SIZE = 8 * Constants.MB;
 
+    /**
+     * The default block size used in {@link FluxUtil#readFile(AsynchronousFileChannel)}.
+     * This is to make sure we're using same size when using {@link BinaryData#fromFile(Path, int)}
+     * and {@link BinaryData#fromFile(Path, Long, Long, int)}
+     * to represent the content.
+     */
+    private static final int DEFAULT_FILE_READ_CHUNK_SIZE = 1024 * 64;
     private final BlobAsyncClient client;
 
     private BlockBlobClient blockBlobClient;
@@ -89,6 +101,31 @@ protected BlobClient(BlobAsyncClient client) {
         this.client = client;
     }
 
+    /**
+     * Protected constructor for use by {@link BlobClientBuilder}.
+     *
+     * @param client the async blob client
+     * @param pipeline The pipeline used to send and receive service requests.
+     * @param url The endpoint where to send service requests.
+     * @param serviceVersion The version of the service to receive requests.
+     * @param accountName The storage account name.
+     * @param containerName The container name.
+     * @param blobName The blob name.
+     * @param snapshot The snapshot identifier for the blob, pass {@code null} to interact with the blob directly.
+     * @param customerProvidedKey Customer provided key used during encryption of the blob's data on the server, pass
+     * {@code null} to allow the service to use its own encryption.
+     * @param encryptionScope Encryption scope used during encryption of the blob's data on the server, pass
+     * {@code null} to allow the service to use its own encryption.
+     * @param versionId The version identifier for the blob, pass {@code null} to interact with the latest blob version.
+     */
+    protected BlobClient(BlobAsyncClient client, HttpPipeline pipeline, String url, BlobServiceVersion serviceVersion,
+        String accountName, String containerName, String blobName, String snapshot, CpkInfo customerProvidedKey,
+        EncryptionScope encryptionScope, String versionId) {
+        super(client, pipeline, url, serviceVersion, accountName, containerName, blobName, snapshot, customerProvidedKey,
+            encryptionScope, versionId);
+        this.client = client;
+    }
+
     /**
      * Creates a new {@link BlobClient} linked to the {@code snapshot} of this blob resource.
      *
@@ -97,7 +134,11 @@ protected BlobClient(BlobAsyncClient client) {
      */
     @Override
     public BlobClient getSnapshotClient(String snapshot) {
-        return new BlobClient(client.getSnapshotClient(snapshot));
+        BlobAsyncClient asyncClient = new BlobAsyncClient(getHttpPipeline(), getAccountUrl(), getServiceVersion(),
+            getAccountName(), getContainerName(), getBlobName(), snapshot, getCustomerProvidedKey(),
+            encryptionScope, getVersionId());
+        return new BlobClient(asyncClient, getHttpPipeline(), getAccountUrl(), getServiceVersion(), getAccountName(),
+            getContainerName(), getBlobName(), snapshot, getCustomerProvidedKey(), encryptionScope, getVersionId());
     }
 
     /**
@@ -109,7 +150,11 @@ public BlobClient getSnapshotClient(String snapshot) {
      */
     @Override
     public BlobClient getVersionClient(String versionId) {
-        return new BlobClient(client.getVersionClient(versionId));
+        BlobAsyncClient asyncClient = new BlobAsyncClient(getHttpPipeline(), getAccountUrl(), getServiceVersion(),
+            getAccountName(), getContainerName(), getBlobName(), getSnapshotId(), getCustomerProvidedKey(),
+            encryptionScope, versionId);
+        return new BlobClient(asyncClient, getHttpPipeline(), getAccountUrl(), getServiceVersion(), getAccountName(),
+            getContainerName(), getBlobName(), getSnapshotId(), getCustomerProvidedKey(), encryptionScope, versionId);
     }
 
     /**
@@ -120,7 +165,13 @@ public BlobClient getVersionClient(String versionId) {
      */
     @Override
     public BlobClient getEncryptionScopeClient(String encryptionScope) {
-        return new BlobClient(client.getEncryptionScopeAsyncClient(encryptionScope));
+        EncryptionScope finalEncryptionScope = null;
+        if (encryptionScope != null) {
+            finalEncryptionScope = new EncryptionScope().setEncryptionScope(encryptionScope);
+        }
+        return new BlobClient(this.client.getEncryptionScopeAsyncClient(encryptionScope), getHttpPipeline(),
+            getAccountUrl(), getServiceVersion(), getAccountName(), getContainerName(), getBlobName(), getSnapshotId(),
+            getCustomerProvidedKey(), finalEncryptionScope, getVersionId());
     }
 
     /**
@@ -132,7 +183,16 @@ public BlobClient getEncryptionScopeClient(String encryptionScope) {
      */
     @Override
     public BlobClient getCustomerProvidedKeyClient(CustomerProvidedKey customerProvidedKey) {
-        return new BlobClient(client.getCustomerProvidedKeyAsyncClient(customerProvidedKey));
+        CpkInfo finalCustomerProvidedKey = null;
+        if (customerProvidedKey != null) {
+            finalCustomerProvidedKey = new CpkInfo()
+                .setEncryptionKey(customerProvidedKey.getKey())
+                .setEncryptionKeySha256(customerProvidedKey.getKeySha256())
+                .setEncryptionAlgorithm(customerProvidedKey.getEncryptionAlgorithm());
+        }
+        return new BlobClient(this.client.getCustomerProvidedKeyAsyncClient(customerProvidedKey), getHttpPipeline(),
+            getAccountUrl(), getServiceVersion(), getAccountName(), getContainerName(), getBlobName(), getSnapshotId(),
+            finalCustomerProvidedKey, encryptionScope, getVersionId());
     }
 
     /**

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobClientBuilder.java

@@ -127,7 +127,29 @@ public BlobClientBuilder() {
      * and {@link #retryOptions(RequestRetryOptions)} have been set.
      */
     public BlobClient buildClient() {
-        return new BlobClient(buildAsyncClient());
+        Objects.requireNonNull(blobName, "'blobName' cannot be null.");
+        Objects.requireNonNull(endpoint, "'endpoint' cannot be null");
+
+        BuilderHelper.httpsValidation(customerProvidedKey, "customer provided key", endpoint, LOGGER);
+
+        if (Objects.nonNull(customerProvidedKey) && Objects.nonNull(encryptionScope)) {
+            throw LOGGER.logExceptionAsError(new IllegalArgumentException("Customer provided key and encryption "
+                + "scope cannot both be set"));
+        }
+
+        /*
+        Implicit and explicit root container access are functionally equivalent, but explicit references are easier
+        to read and debug.
+         */
+        String blobContainerName = CoreUtils.isNullOrEmpty(containerName) ? BlobContainerClient.ROOT_CONTAINER_NAME
+            : containerName;
+
+        BlobServiceVersion serviceVersion = version != null ? version : BlobServiceVersion.getLatest();
+
+        BlobAsyncClient asyncClient = buildAsyncClient();
+
+        return new BlobClient(asyncClient, asyncClient.getHttpPipeline(), endpoint, serviceVersion, accountName,
+            blobContainerName, blobName, snapshot, customerProvidedKey, encryptionScope, versionId);
     }
 
     /**
@@ -171,15 +193,19 @@ public BlobAsyncClient buildAsyncClient() {
 
         BlobServiceVersion serviceVersion = version != null ? version : BlobServiceVersion.getLatest();
 
-        HttpPipeline pipeline = (httpPipeline != null) ? httpPipeline : BuilderHelper.buildPipeline(
-            storageSharedKeyCredential, tokenCredential, azureSasCredential, sasToken,
-            endpoint, retryOptions, coreRetryOptions, logOptions,
-            clientOptions, httpClient, perCallPolicies, perRetryPolicies, configuration, audience, LOGGER);
+        HttpPipeline pipeline = constructPipeline();
 
         return new BlobAsyncClient(pipeline, endpoint, serviceVersion, accountName, blobContainerName, blobName,
             snapshot, customerProvidedKey, encryptionScope, versionId);
     }
 
+    private HttpPipeline constructPipeline() {
+        return (httpPipeline != null) ? httpPipeline : BuilderHelper.buildPipeline(
+            storageSharedKeyCredential, tokenCredential, azureSasCredential, sasToken,
+            endpoint, retryOptions, coreRetryOptions, logOptions,
+            clientOptions, httpClient, perCallPolicies, perRetryPolicies, configuration, audience, LOGGER);
+    }
+
     /**
      * Sets the {@link CustomerProvidedKey customer provided key} that is used to encrypt blob contents on the server.
      *

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java

@@ -56,7 +56,6 @@
 import java.net.URI;
 import java.time.Duration;
 import java.time.OffsetDateTime;
-import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -315,6 +314,19 @@ public String getEncryptionScope() {
         return encryptionScope.getEncryptionScope();
     }
 
+    /**
+     * Gets the {@link EncryptionScope} used to encrypt this blob's content on the server.
+     *
+     * @return the encryption scope used for encryption.
+     */
+    BlobContainerEncryptionScope getBlobContainerEncryptionScope() {
+        if (blobContainerEncryptionScope == null) {
+            return null;
+        }
+        return blobContainerEncryptionScope;
+    }
+
+
     /**
      * Gets if the container this client represents exists in the cloud.
      *
@@ -565,7 +577,7 @@ public Mono<Response<Void>> deleteWithResponse(BlobRequestConditions requestCond
     Mono<Response<Void>> deleteWithResponse(BlobRequestConditions requestConditions, Context context) {
         requestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions;
 
-        if (!validateNoETag(requestConditions)) {
+        if (!ModelHelper.validateNoETag(requestConditions)) {
             // Throwing is preferred to Mono.error because this will error out immediately instead of waiting until
             // subscription.
             throw LOGGER.logExceptionAsError(
@@ -791,7 +803,7 @@ Mono<Response<Void>> setMetadataWithResponse(Map<String, String> metadata,
         BlobRequestConditions requestConditions, Context context) {
         context = context == null ? Context.NONE : context;
         requestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions;
-        if (!validateNoETag(requestConditions) || requestConditions.getIfUnmodifiedSince() != null) {
+        if (!ModelHelper.validateNoETag(requestConditions) || requestConditions.getIfUnmodifiedSince() != null) {
             // Throwing is preferred to Mono.error because this will error out immediately instead of waiting until
             // subscription.
             throw LOGGER.logExceptionAsError(new UnsupportedOperationException(
@@ -961,36 +973,19 @@ Mono<Response<Void>> setAccessPolicyWithResponse(PublicAccessType accessType,
         List<BlobSignedIdentifier> identifiers, BlobRequestConditions requestConditions, Context context) {
         requestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions;
 
-        if (!validateNoETag(requestConditions)) {
+        if (!ModelHelper.validateNoETag(requestConditions)) {
             // Throwing is preferred to Mono.error because this will error out immediately instead of waiting until
             // subscription.
             throw LOGGER.logExceptionAsError(
                 new UnsupportedOperationException("ETag access conditions are not supported for this API."));
         }
 
-        /*
-        We truncate to seconds because the service only supports nanoseconds or seconds, but doing an
-        OffsetDateTime.now will only give back milliseconds (more precise fields are zeroed and not serialized). This
-        allows for proper serialization with no real detriment to users as sub-second precision on active time for
-        signed identifiers is not really necessary.
-         */
-        if (identifiers != null) {
-            for (BlobSignedIdentifier identifier : identifiers) {
-                if (identifier.getAccessPolicy() != null && identifier.getAccessPolicy().getStartsOn() != null) {
-                    identifier.getAccessPolicy().setStartsOn(
-                        identifier.getAccessPolicy().getStartsOn().truncatedTo(ChronoUnit.SECONDS));
-                }
-                if (identifier.getAccessPolicy() != null && identifier.getAccessPolicy().getExpiresOn() != null) {
-                    identifier.getAccessPolicy().setExpiresOn(
-                        identifier.getAccessPolicy().getExpiresOn().truncatedTo(ChronoUnit.SECONDS));
-                }
-            }
-        }
+        List<BlobSignedIdentifier> finalIdentifiers = ModelHelper.truncateTimeForBlobSignedIdentifier(identifiers);
         context = context == null ? Context.NONE : context;
 
         return this.azureBlobStorage.getContainers().setAccessPolicyNoCustomHeadersWithResponseAsync(containerName,
             null, requestConditions.getLeaseId(), accessType, requestConditions.getIfModifiedSince(),
-            requestConditions.getIfUnmodifiedSince(), null, identifiers, context);
+            requestConditions.getIfUnmodifiedSince(), null, finalIdentifiers, context);
     }
 
     /**
@@ -1725,13 +1720,6 @@ public String generateSas(BlobServiceSasSignatureValues blobServiceSasSignatureV
             .generateSas(SasImplUtils.extractSharedKeyCredential(getHttpPipeline()), stringToSignHandler, context);
     }
 
-    private static boolean validateNoETag(BlobRequestConditions modifiedRequestConditions) {
-        if (modifiedRequestConditions == null) {
-            return true;
-        }
-        return modifiedRequestConditions.getIfMatch() == null && modifiedRequestConditions.getIfNoneMatch() == null;
-    }
-
 //    private boolean validateNoTime(BlobRequestConditions modifiedRequestConditions) {
 //        if (modifiedRequestConditions == null) {
 //            return true;